diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q1.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q1.txt index 23544dbc5..fe3e108f5 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q1.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q1.txt @@ -11,9 +11,9 @@ AdaptiveSparkPlan (151) : +- NativeShuffleExchange (81) : +- NativeProject (80) : +- NativeSortMergeJoin Inner (79) - : :- ConvertToNative (69) - : : +- * Project (68) - : : +- * SortMergeJoin Inner (67) + : :- NativeProject (69) + : : +- NativeFilter (68) + : : +- NativeSortMergeJoin Inner (67) : : :- NativeSort (33) : : : +- InputAdapter (32) : : : +- AQEShuffleRead (31) @@ -440,18 +440,19 @@ Condition : isnotnull((avg(ctr_total_return) * 1.2)#27) Input [2]: [(avg(ctr_total_return) * 1.2)#27, ctr_store_sk#21] Arguments: [ctr_store_sk#21 ASC NULLS FIRST], false -(67) SortMergeJoin [codegen id : X] +(67) NativeSortMergeJoin Left keys [1]: [ctr_store_sk#12] Right keys [1]: [ctr_store_sk#21] Join type: Inner -Join condition: (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#27) +Join condition: None -(68) Project [codegen id : X] -Output [2]: [ctr_customer_sk#11, ctr_store_sk#12] +(68) NativeFilter Input [5]: [ctr_customer_sk#11, ctr_store_sk#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#21] +Condition : (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#27) -(69) ConvertToNative -Input [2]: [ctr_customer_sk#11, ctr_store_sk#12] +(69) NativeProject +Output [2]: [ctr_customer_sk#11, ctr_store_sk#12] +Input [5]: [ctr_customer_sk#11, ctr_store_sk#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#27, ctr_store_sk#21] (135) Scan parquet Output [2]: [s_store_sk#28, s_state#29] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q11.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q11.txt index d6c6fcbc7..22e889087 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q11.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q11.txt @@ -1,264 +1,265 @@ == Physical Plan == -AdaptiveSparkPlan (255) +AdaptiveSparkPlan (256) +- == Final Plan == - TakeOrderedAndProject (153) - +- * Project (152) - +- * SortMergeJoin Inner (151) - :- NativeProject (123) - : +- NativeSortMergeJoin Inner (122) - : :- NativeProject (78) - : : +- NativeSortMergeJoin Inner (77) - : : :- NativeSort (47) - : : : +- InputAdapter (46) - : : : +- AQEShuffleRead (45) - : : : +- ShuffleQueryStage (44), Statistics(X) - : : : +- NativeShuffleExchange (43) - : : : +- NativeFilter (42) - : : : +- NativeProject (41) - : : : +- NativeHashAggregate (40) - : : : +- InputAdapter (39) - : : : +- AQEShuffleRead (38) - : : : +- ShuffleQueryStage (37), Statistics(X) - : : : +- NativeShuffleExchange (36) - : : : +- NativeHashAggregate (35) - : : : +- NativeProject (34) - : : : +- NativeProject (33) - : : : +- NativeSortMergeJoin Inner (32) - : : : :- NativeSort (23) - : : : : +- InputAdapter (22) - : : : : +- AQEShuffleRead (21) - : : : : +- ShuffleQueryStage (20), Statistics(X) - : : : : +- NativeShuffleExchange (19) - : : : : +- NativeProject (18) - : : : : +- NativeSortMergeJoin Inner (17) - : : : : :- NativeSort (8) - : : : : : +- InputAdapter (7) - : : : : : +- AQEShuffleRead (6) - : : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : : +- NativeShuffleExchange (4) - : : : : : +- NativeFilter (3) - : : : : : +- InputAdapter (2) - : : : : : +- NativeParquetScan (1) - : : : : +- NativeSort (16) - : : : : +- InputAdapter (15) - : : : : +- AQEShuffleRead (14) - : : : : +- ShuffleQueryStage (13), Statistics(X) - : : : : +- NativeShuffleExchange (12) - : : : : +- NativeFilter (11) - : : : : +- InputAdapter (10) - : : : : +- NativeParquetScan (9) - : : : +- NativeSort (31) - : : : +- InputAdapter (30) - : : : +- AQEShuffleRead (29) - : : : +- ShuffleQueryStage (28), Statistics(X) - : : : +- NativeShuffleExchange (27) - : : : +- NativeFilter (26) - : : : +- InputAdapter (25) - : : : +- NativeParquetScan (24) - : : +- NativeSort (76) - : : +- InputAdapter (75) - : : +- AQEShuffleRead (74) - : : +- ShuffleQueryStage (73), Statistics(X) - : : +- NativeShuffleExchange (72) - : : +- NativeProject (71) - : : +- NativeHashAggregate (70) - : : +- InputAdapter (69) - : : +- AQEShuffleRead (68) - : : +- ShuffleQueryStage (67), Statistics(X) - : : +- NativeShuffleExchange (66) - : : +- NativeHashAggregate (65) - : : +- NativeProject (64) - : : +- NativeProject (63) - : : +- NativeSortMergeJoin Inner (62) - : : :- NativeSort (53) - : : : +- InputAdapter (52) - : : : +- InputAdapter (51) - : : : +- AQEShuffleRead (50) - : : : +- ShuffleQueryStage (49), Statistics(X) - : : : +- ReusedExchange (48) - : : +- NativeSort (61) - : : +- InputAdapter (60) - : : +- AQEShuffleRead (59) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- NativeShuffleExchange (57) - : : +- NativeFilter (56) - : : +- InputAdapter (55) - : : +- NativeParquetScan (54) - : +- NativeSort (121) - : +- InputAdapter (120) - : +- AQEShuffleRead (119) - : +- ShuffleQueryStage (118), Statistics(X) - : +- NativeShuffleExchange (117) - : +- NativeFilter (116) - : +- NativeProject (115) - : +- NativeHashAggregate (114) - : +- InputAdapter (113) - : +- AQEShuffleRead (112) - : +- ShuffleQueryStage (111), Statistics(X) - : +- NativeShuffleExchange (110) - : +- NativeHashAggregate (109) - : +- NativeProject (108) - : +- NativeProject (107) - : +- NativeSortMergeJoin Inner (106) - : :- NativeSort (99) - : : +- InputAdapter (98) - : : +- AQEShuffleRead (97) - : : +- ShuffleQueryStage (96), Statistics(X) - : : +- NativeShuffleExchange (95) - : : +- NativeProject (94) - : : +- NativeSortMergeJoin Inner (93) - : : :- NativeSort (84) - : : : +- InputAdapter (83) - : : : +- InputAdapter (82) - : : : +- AQEShuffleRead (81) - : : : +- ShuffleQueryStage (80), Statistics(X) - : : : +- ReusedExchange (79) - : : +- NativeSort (92) - : : +- InputAdapter (91) - : : +- AQEShuffleRead (90) - : : +- ShuffleQueryStage (89), Statistics(X) - : : +- NativeShuffleExchange (88) - : : +- NativeFilter (87) - : : +- InputAdapter (86) - : : +- NativeParquetScan (85) - : +- NativeSort (105) - : +- InputAdapter (104) - : +- InputAdapter (103) - : +- AQEShuffleRead (102) - : +- ShuffleQueryStage (101), Statistics(X) - : +- ReusedExchange (100) - +- NativeSort (150) - +- InputAdapter (149) - +- AQEShuffleRead (148) - +- ShuffleQueryStage (147), Statistics(X) - +- NativeShuffleExchange (146) - +- NativeProject (145) - +- NativeHashAggregate (144) - +- InputAdapter (143) - +- AQEShuffleRead (142) - +- ShuffleQueryStage (141), Statistics(X) - +- NativeShuffleExchange (140) - +- NativeHashAggregate (139) - +- NativeProject (138) - +- NativeProject (137) - +- NativeSortMergeJoin Inner (136) - :- NativeSort (129) - : +- InputAdapter (128) - : +- InputAdapter (127) - : +- AQEShuffleRead (126) - : +- ShuffleQueryStage (125), Statistics(X) - : +- ReusedExchange (124) - +- NativeSort (135) - +- InputAdapter (134) - +- InputAdapter (133) - +- AQEShuffleRead (132) - +- ShuffleQueryStage (131), Statistics(X) - +- ReusedExchange (130) + NativeTakeOrdered (154) + +- NativeProject (153) + +- NativeFilter (152) + +- NativeSortMergeJoin Inner (151) + :- NativeProject (123) + : +- NativeSortMergeJoin Inner (122) + : :- NativeProject (78) + : : +- NativeSortMergeJoin Inner (77) + : : :- NativeSort (47) + : : : +- InputAdapter (46) + : : : +- AQEShuffleRead (45) + : : : +- ShuffleQueryStage (44), Statistics(X) + : : : +- NativeShuffleExchange (43) + : : : +- NativeFilter (42) + : : : +- NativeProject (41) + : : : +- NativeHashAggregate (40) + : : : +- InputAdapter (39) + : : : +- AQEShuffleRead (38) + : : : +- ShuffleQueryStage (37), Statistics(X) + : : : +- NativeShuffleExchange (36) + : : : +- NativeHashAggregate (35) + : : : +- NativeProject (34) + : : : +- NativeProject (33) + : : : +- NativeSortMergeJoin Inner (32) + : : : :- NativeSort (23) + : : : : +- InputAdapter (22) + : : : : +- AQEShuffleRead (21) + : : : : +- ShuffleQueryStage (20), Statistics(X) + : : : : +- NativeShuffleExchange (19) + : : : : +- NativeProject (18) + : : : : +- NativeSortMergeJoin Inner (17) + : : : : :- NativeSort (8) + : : : : : +- InputAdapter (7) + : : : : : +- AQEShuffleRead (6) + : : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : : +- NativeShuffleExchange (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (16) + : : : : +- InputAdapter (15) + : : : : +- AQEShuffleRead (14) + : : : : +- ShuffleQueryStage (13), Statistics(X) + : : : : +- NativeShuffleExchange (12) + : : : : +- NativeFilter (11) + : : : : +- InputAdapter (10) + : : : : +- NativeParquetScan (9) + : : : +- NativeSort (31) + : : : +- InputAdapter (30) + : : : +- AQEShuffleRead (29) + : : : +- ShuffleQueryStage (28), Statistics(X) + : : : +- NativeShuffleExchange (27) + : : : +- NativeFilter (26) + : : : +- InputAdapter (25) + : : : +- NativeParquetScan (24) + : : +- NativeSort (76) + : : +- InputAdapter (75) + : : +- AQEShuffleRead (74) + : : +- ShuffleQueryStage (73), Statistics(X) + : : +- NativeShuffleExchange (72) + : : +- NativeProject (71) + : : +- NativeHashAggregate (70) + : : +- InputAdapter (69) + : : +- AQEShuffleRead (68) + : : +- ShuffleQueryStage (67), Statistics(X) + : : +- NativeShuffleExchange (66) + : : +- NativeHashAggregate (65) + : : +- NativeProject (64) + : : +- NativeProject (63) + : : +- NativeSortMergeJoin Inner (62) + : : :- NativeSort (53) + : : : +- InputAdapter (52) + : : : +- InputAdapter (51) + : : : +- AQEShuffleRead (50) + : : : +- ShuffleQueryStage (49), Statistics(X) + : : : +- ReusedExchange (48) + : : +- NativeSort (61) + : : +- InputAdapter (60) + : : +- AQEShuffleRead (59) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- NativeShuffleExchange (57) + : : +- NativeFilter (56) + : : +- InputAdapter (55) + : : +- NativeParquetScan (54) + : +- NativeSort (121) + : +- InputAdapter (120) + : +- AQEShuffleRead (119) + : +- ShuffleQueryStage (118), Statistics(X) + : +- NativeShuffleExchange (117) + : +- NativeFilter (116) + : +- NativeProject (115) + : +- NativeHashAggregate (114) + : +- InputAdapter (113) + : +- AQEShuffleRead (112) + : +- ShuffleQueryStage (111), Statistics(X) + : +- NativeShuffleExchange (110) + : +- NativeHashAggregate (109) + : +- NativeProject (108) + : +- NativeProject (107) + : +- NativeSortMergeJoin Inner (106) + : :- NativeSort (99) + : : +- InputAdapter (98) + : : +- AQEShuffleRead (97) + : : +- ShuffleQueryStage (96), Statistics(X) + : : +- NativeShuffleExchange (95) + : : +- NativeProject (94) + : : +- NativeSortMergeJoin Inner (93) + : : :- NativeSort (84) + : : : +- InputAdapter (83) + : : : +- InputAdapter (82) + : : : +- AQEShuffleRead (81) + : : : +- ShuffleQueryStage (80), Statistics(X) + : : : +- ReusedExchange (79) + : : +- NativeSort (92) + : : +- InputAdapter (91) + : : +- AQEShuffleRead (90) + : : +- ShuffleQueryStage (89), Statistics(X) + : : +- NativeShuffleExchange (88) + : : +- NativeFilter (87) + : : +- InputAdapter (86) + : : +- NativeParquetScan (85) + : +- NativeSort (105) + : +- InputAdapter (104) + : +- InputAdapter (103) + : +- AQEShuffleRead (102) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ReusedExchange (100) + +- NativeSort (150) + +- InputAdapter (149) + +- AQEShuffleRead (148) + +- ShuffleQueryStage (147), Statistics(X) + +- NativeShuffleExchange (146) + +- NativeProject (145) + +- NativeHashAggregate (144) + +- InputAdapter (143) + +- AQEShuffleRead (142) + +- ShuffleQueryStage (141), Statistics(X) + +- NativeShuffleExchange (140) + +- NativeHashAggregate (139) + +- NativeProject (138) + +- NativeProject (137) + +- NativeSortMergeJoin Inner (136) + :- NativeSort (129) + : +- InputAdapter (128) + : +- InputAdapter (127) + : +- AQEShuffleRead (126) + : +- ShuffleQueryStage (125), Statistics(X) + : +- ReusedExchange (124) + +- NativeSort (135) + +- InputAdapter (134) + +- InputAdapter (133) + +- AQEShuffleRead (132) + +- ShuffleQueryStage (131), Statistics(X) + +- ReusedExchange (130) +- == Initial Plan == - TakeOrderedAndProject (254) - +- Project (253) - +- SortMergeJoin Inner (252) - :- Project (228) - : +- SortMergeJoin Inner (227) - : :- Project (202) - : : +- SortMergeJoin Inner (201) - : : :- Sort (177) - : : : +- Exchange (176) - : : : +- Filter (175) - : : : +- HashAggregate (174) - : : : +- Exchange (173) - : : : +- HashAggregate (172) - : : : +- Project (171) - : : : +- SortMergeJoin Inner (170) - : : : :- Sort (165) - : : : : +- Exchange (164) - : : : : +- Project (163) - : : : : +- SortMergeJoin Inner (162) - : : : : :- Sort (157) - : : : : : +- Exchange (156) - : : : : : +- Filter (155) - : : : : : +- Scan parquet (154) - : : : : +- Sort (161) - : : : : +- Exchange (160) - : : : : +- Filter (159) - : : : : +- Scan parquet (158) - : : : +- Sort (169) - : : : +- Exchange (168) - : : : +- Filter (167) - : : : +- Scan parquet (166) - : : +- Sort (200) - : : +- Exchange (199) - : : +- HashAggregate (198) - : : +- Exchange (197) - : : +- HashAggregate (196) - : : +- Project (195) - : : +- SortMergeJoin Inner (194) - : : :- Sort (189) - : : : +- Exchange (188) - : : : +- Project (187) - : : : +- SortMergeJoin Inner (186) - : : : :- Sort (181) - : : : : +- Exchange (180) - : : : : +- Filter (179) - : : : : +- Scan parquet (178) - : : : +- Sort (185) - : : : +- Exchange (184) - : : : +- Filter (183) - : : : +- Scan parquet (182) - : : +- Sort (193) - : : +- Exchange (192) - : : +- Filter (191) - : : +- Scan parquet (190) - : +- Sort (226) - : +- Exchange (225) - : +- Filter (224) - : +- HashAggregate (223) - : +- Exchange (222) - : +- HashAggregate (221) - : +- Project (220) - : +- SortMergeJoin Inner (219) - : :- Sort (214) - : : +- Exchange (213) - : : +- Project (212) - : : +- SortMergeJoin Inner (211) - : : :- Sort (206) - : : : +- Exchange (205) - : : : +- Filter (204) - : : : +- Scan parquet (203) - : : +- Sort (210) - : : +- Exchange (209) - : : +- Filter (208) - : : +- Scan parquet (207) - : +- Sort (218) - : +- Exchange (217) - : +- Filter (216) - : +- Scan parquet (215) - +- Sort (251) - +- Exchange (250) - +- HashAggregate (249) - +- Exchange (248) - +- HashAggregate (247) - +- Project (246) - +- SortMergeJoin Inner (245) - :- Sort (240) - : +- Exchange (239) - : +- Project (238) - : +- SortMergeJoin Inner (237) - : :- Sort (232) - : : +- Exchange (231) - : : +- Filter (230) - : : +- Scan parquet (229) - : +- Sort (236) - : +- Exchange (235) - : +- Filter (234) - : +- Scan parquet (233) - +- Sort (244) - +- Exchange (243) - +- Filter (242) - +- Scan parquet (241) - - -(154) Scan parquet + TakeOrderedAndProject (255) + +- Project (254) + +- SortMergeJoin Inner (253) + :- Project (229) + : +- SortMergeJoin Inner (228) + : :- Project (203) + : : +- SortMergeJoin Inner (202) + : : :- Sort (178) + : : : +- Exchange (177) + : : : +- Filter (176) + : : : +- HashAggregate (175) + : : : +- Exchange (174) + : : : +- HashAggregate (173) + : : : +- Project (172) + : : : +- SortMergeJoin Inner (171) + : : : :- Sort (166) + : : : : +- Exchange (165) + : : : : +- Project (164) + : : : : +- SortMergeJoin Inner (163) + : : : : :- Sort (158) + : : : : : +- Exchange (157) + : : : : : +- Filter (156) + : : : : : +- Scan parquet (155) + : : : : +- Sort (162) + : : : : +- Exchange (161) + : : : : +- Filter (160) + : : : : +- Scan parquet (159) + : : : +- Sort (170) + : : : +- Exchange (169) + : : : +- Filter (168) + : : : +- Scan parquet (167) + : : +- Sort (201) + : : +- Exchange (200) + : : +- HashAggregate (199) + : : +- Exchange (198) + : : +- HashAggregate (197) + : : +- Project (196) + : : +- SortMergeJoin Inner (195) + : : :- Sort (190) + : : : +- Exchange (189) + : : : +- Project (188) + : : : +- SortMergeJoin Inner (187) + : : : :- Sort (182) + : : : : +- Exchange (181) + : : : : +- Filter (180) + : : : : +- Scan parquet (179) + : : : +- Sort (186) + : : : +- Exchange (185) + : : : +- Filter (184) + : : : +- Scan parquet (183) + : : +- Sort (194) + : : +- Exchange (193) + : : +- Filter (192) + : : +- Scan parquet (191) + : +- Sort (227) + : +- Exchange (226) + : +- Filter (225) + : +- HashAggregate (224) + : +- Exchange (223) + : +- HashAggregate (222) + : +- Project (221) + : +- SortMergeJoin Inner (220) + : :- Sort (215) + : : +- Exchange (214) + : : +- Project (213) + : : +- SortMergeJoin Inner (212) + : : :- Sort (207) + : : : +- Exchange (206) + : : : +- Filter (205) + : : : +- Scan parquet (204) + : : +- Sort (211) + : : +- Exchange (210) + : : +- Filter (209) + : : +- Scan parquet (208) + : +- Sort (219) + : +- Exchange (218) + : +- Filter (217) + : +- Scan parquet (216) + +- Sort (252) + +- Exchange (251) + +- HashAggregate (250) + +- Exchange (249) + +- HashAggregate (248) + +- Project (247) + +- SortMergeJoin Inner (246) + :- Sort (241) + : +- Exchange (240) + : +- Project (239) + : +- SortMergeJoin Inner (238) + : :- Sort (233) + : : +- Exchange (232) + : : +- Filter (231) + : : +- Scan parquet (230) + : +- Sort (237) + : +- Exchange (236) + : +- Filter (235) + : +- Scan parquet (234) + +- Sort (245) + +- Exchange (244) + +- Filter (243) + +- Scan parquet (242) + + +(155) Scan parquet Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location: InMemoryFileIndex [file:/] @@ -292,7 +293,7 @@ Input [8]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8] Input [8]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8] Arguments: [c_customer_sk#1 ASC NULLS FIRST], false -(158) Scan parquet +(159) Scan parquet Output [4]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_list_price#12] Batched: true Location: InMemoryFileIndex [file:/] @@ -355,7 +356,7 @@ Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_fl Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_list_price#12] Arguments: [ss_sold_date_sk#9 ASC NULLS FIRST], false -(166) Scan parquet +(167) Scan parquet Output [2]: [d_date_sk#13, d_year#14] Batched: true Location: InMemoryFileIndex [file:/] @@ -481,7 +482,7 @@ Input [10]: [#21#21, #22#22, #23#23, #24#24, #25#25, #26#26, #27#27, #28#28, #29 Input [10]: [#21#21, #22#22, #23#23, #24#24, #25#25, #26#26, #27#27, #28#28, #29#29, #30#30] Arguments: [ss_sold_date_sk#28 ASC NULLS FIRST], false -(190) Scan parquet +(191) Scan parquet Output [2]: [d_date_sk#31, d_year#32] Batched: true Location: InMemoryFileIndex [file:/] @@ -613,7 +614,7 @@ Input [8]: [#38#38, #39#39, #40#40, #41#41, #42#42, #43#43, #44#44, #45#45] Input [8]: [#38#38, #39#39, #40#40, #41#41, #42#42, #43#43, #44#44, #45#45] Arguments: [c_customer_sk#38 ASC NULLS FIRST], false -(207) Scan parquet +(208) Scan parquet Output [4]: [ws_sold_date_sk#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49] Batched: true Location: InMemoryFileIndex [file:/] @@ -888,507 +889,511 @@ Input [2]: [customer_id#71, year_total#72] Input [2]: [customer_id#71, year_total#72] Arguments: [customer_id#71 ASC NULLS FIRST], false -(151) SortMergeJoin [codegen id : X] +(151) NativeSortMergeJoin Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#71] Join type: Inner -Join condition: (CASE WHEN (year_total#56 > 0.00) THEN (year_total#72 / year_total#56) END > CASE WHEN (year_total#20 > 0.00) THEN (year_total#37 / year_total#20) END) +Join condition: None -(152) Project [codegen id : X] +(152) NativeFilter +Input [7]: [customer_id#19, year_total#20, customer_preferred_cust_flag#36, year_total#37, year_total#56, customer_id#71, year_total#72] +Condition : (CASE WHEN (year_total#56 > 0.00) THEN (year_total#72 / year_total#56) END > CASE WHEN (year_total#20 > 0.00) THEN (year_total#37 / year_total#20) END) + +(153) NativeProject Output [1]: [customer_preferred_cust_flag#36] Input [7]: [customer_id#19, year_total#20, customer_preferred_cust_flag#36, year_total#37, year_total#56, customer_id#71, year_total#72] -(153) TakeOrderedAndProject +(154) NativeTakeOrdered Input [1]: [customer_preferred_cust_flag#36] -Arguments: X, [customer_preferred_cust_flag#36 ASC NULLS FIRST], [customer_preferred_cust_flag#36] +Arguments: X, X, [customer_preferred_cust_flag#36 ASC NULLS FIRST] -(154) Scan parquet +(155) Scan parquet Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(155) Filter +(156) Filter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) -(156) Exchange +(157) Exchange Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: hashpartitioning(c_customer_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=16] -(157) Sort +(158) Sort Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0 -(158) Scan parquet +(159) Scan parquet Output [4]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_list_price#12] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(159) Filter +(160) Filter Input [4]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_list_price#12] Condition : (isnotnull(ss_customer_sk#10) AND isnotnull(ss_sold_date_sk#9)) -(160) Exchange +(161) Exchange Input [4]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_list_price#12] Arguments: hashpartitioning(ss_customer_sk#10, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(161) Sort +(162) Sort Input [4]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_list_price#12] Arguments: [ss_customer_sk#10 ASC NULLS FIRST], false, 0 -(162) SortMergeJoin +(163) SortMergeJoin Left keys [1]: [c_customer_sk#1] Right keys [1]: [ss_customer_sk#10] Join type: Inner Join condition: None -(163) Project +(164) Project Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_list_price#12] Input [12]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_list_price#12] -(164) Exchange +(165) Exchange Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_list_price#12] Arguments: hashpartitioning(ss_sold_date_sk#9, 100), ENSURE_REQUIREMENTS, [plan_id=18] -(165) Sort +(166) Sort Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_list_price#12] Arguments: [ss_sold_date_sk#9 ASC NULLS FIRST], false, 0 -(166) Scan parquet +(167) Scan parquet Output [2]: [d_date_sk#13, d_year#14] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(167) Filter +(168) Filter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2001)) AND isnotnull(d_date_sk#13)) -(168) Exchange +(169) Exchange Input [2]: [d_date_sk#13, d_year#14] Arguments: hashpartitioning(d_date_sk#13, 100), ENSURE_REQUIREMENTS, [plan_id=19] -(169) Sort +(170) Sort Input [2]: [d_date_sk#13, d_year#14] Arguments: [d_date_sk#13 ASC NULLS FIRST], false, 0 -(170) SortMergeJoin +(171) SortMergeJoin Left keys [1]: [ss_sold_date_sk#9] Right keys [1]: [d_date_sk#13] Join type: Inner Join condition: None -(171) Project +(172) Project Output [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#11, ss_ext_list_price#12, d_year#14] Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_list_price#12, d_date_sk#13, d_year#14] -(172) HashAggregate +(173) HashAggregate Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#11, ss_ext_list_price#12, d_year#14] Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#12 - ss_ext_discount_amt#11)))] Aggregate Attributes [1]: [sum#16] Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#73] -(173) Exchange +(174) Exchange Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#73] Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 100), ENSURE_REQUIREMENTS, [plan_id=20] -(174) HashAggregate +(175) HashAggregate Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#73] Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#14, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#12 - ss_ext_discount_amt#11)))] Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#12 - ss_ext_discount_amt#11)))#18] Results [2]: [c_customer_id#2 AS customer_id#19, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#12 - ss_ext_discount_amt#11)))#18,18,2) AS year_total#20] -(175) Filter +(176) Filter Input [2]: [customer_id#19, year_total#20] Condition : (isnotnull(year_total#20) AND (year_total#20 > 0.00)) -(176) Exchange +(177) Exchange Input [2]: [customer_id#19, year_total#20] Arguments: hashpartitioning(customer_id#19, 100), ENSURE_REQUIREMENTS, [plan_id=21] -(177) Sort +(178) Sort Input [2]: [customer_id#19, year_total#20] Arguments: [customer_id#19 ASC NULLS FIRST], false, 0 -(178) Scan parquet +(179) Scan parquet Output [8]: [c_customer_sk#74, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(179) Filter +(180) Filter Input [8]: [c_customer_sk#74, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] Condition : (isnotnull(c_customer_sk#74) AND isnotnull(c_customer_id#21)) -(180) Exchange +(181) Exchange Input [8]: [c_customer_sk#74, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] Arguments: hashpartitioning(c_customer_sk#74, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(181) Sort +(182) Sort Input [8]: [c_customer_sk#74, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] Arguments: [c_customer_sk#74 ASC NULLS FIRST], false, 0 -(182) Scan parquet +(183) Scan parquet Output [4]: [ss_sold_date_sk#28, ss_customer_sk#75, ss_ext_discount_amt#29, ss_ext_list_price#30] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(183) Filter +(184) Filter Input [4]: [ss_sold_date_sk#28, ss_customer_sk#75, ss_ext_discount_amt#29, ss_ext_list_price#30] Condition : (isnotnull(ss_customer_sk#75) AND isnotnull(ss_sold_date_sk#28)) -(184) Exchange +(185) Exchange Input [4]: [ss_sold_date_sk#28, ss_customer_sk#75, ss_ext_discount_amt#29, ss_ext_list_price#30] Arguments: hashpartitioning(ss_customer_sk#75, 100), ENSURE_REQUIREMENTS, [plan_id=23] -(185) Sort +(186) Sort Input [4]: [ss_sold_date_sk#28, ss_customer_sk#75, ss_ext_discount_amt#29, ss_ext_list_price#30] Arguments: [ss_customer_sk#75 ASC NULLS FIRST], false, 0 -(186) SortMergeJoin +(187) SortMergeJoin Left keys [1]: [c_customer_sk#74] Right keys [1]: [ss_customer_sk#75] Join type: Inner Join condition: None -(187) Project +(188) Project Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_sold_date_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30] Input [12]: [c_customer_sk#74, c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_sold_date_sk#28, ss_customer_sk#75, ss_ext_discount_amt#29, ss_ext_list_price#30] -(188) Exchange +(189) Exchange Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_sold_date_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30] Arguments: hashpartitioning(ss_sold_date_sk#28, 100), ENSURE_REQUIREMENTS, [plan_id=24] -(189) Sort +(190) Sort Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_sold_date_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30] Arguments: [ss_sold_date_sk#28 ASC NULLS FIRST], false, 0 -(190) Scan parquet +(191) Scan parquet Output [2]: [d_date_sk#31, d_year#32] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(191) Filter +(192) Filter Input [2]: [d_date_sk#31, d_year#32] Condition : ((isnotnull(d_year#32) AND (d_year#32 = 2002)) AND isnotnull(d_date_sk#31)) -(192) Exchange +(193) Exchange Input [2]: [d_date_sk#31, d_year#32] Arguments: hashpartitioning(d_date_sk#31, 100), ENSURE_REQUIREMENTS, [plan_id=25] -(193) Sort +(194) Sort Input [2]: [d_date_sk#31, d_year#32] Arguments: [d_date_sk#31 ASC NULLS FIRST], false, 0 -(194) SortMergeJoin +(195) SortMergeJoin Left keys [1]: [ss_sold_date_sk#28] Right keys [1]: [d_date_sk#31] Join type: Inner Join condition: None -(195) Project +(196) Project Output [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#32] Input [12]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_sold_date_sk#28, ss_ext_discount_amt#29, ss_ext_list_price#30, d_date_sk#31, d_year#32] -(196) HashAggregate +(197) HashAggregate Input [10]: [c_customer_id#21, c_first_name#22, c_last_name#23, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, ss_ext_discount_amt#29, ss_ext_list_price#30, d_year#32] Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#32, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] Aggregate Attributes [1]: [sum#34] Results [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#32, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#76] -(197) Exchange +(198) Exchange Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#32, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#76] Arguments: hashpartitioning(c_customer_id#21, c_first_name#22, c_last_name#23, d_year#32, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, 100), ENSURE_REQUIREMENTS, [plan_id=26] -(198) HashAggregate +(199) HashAggregate Input [9]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#32, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27, sum#76] Keys [8]: [c_customer_id#21, c_first_name#22, c_last_name#23, d_year#32, c_preferred_cust_flag#24, c_birth_country#25, c_login#26, c_email_address#27] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))] Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#18] Results [3]: [c_customer_id#21 AS customer_id#35, c_preferred_cust_flag#24 AS customer_preferred_cust_flag#36, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#30 - ss_ext_discount_amt#29)))#18,18,2) AS year_total#37] -(199) Exchange +(200) Exchange Input [3]: [customer_id#35, customer_preferred_cust_flag#36, year_total#37] Arguments: hashpartitioning(customer_id#35, 100), ENSURE_REQUIREMENTS, [plan_id=27] -(200) Sort +(201) Sort Input [3]: [customer_id#35, customer_preferred_cust_flag#36, year_total#37] Arguments: [customer_id#35 ASC NULLS FIRST], false, 0 -(201) SortMergeJoin +(202) SortMergeJoin Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#35] Join type: Inner Join condition: None -(202) Project +(203) Project Output [4]: [customer_id#19, year_total#20, customer_preferred_cust_flag#36, year_total#37] Input [5]: [customer_id#19, year_total#20, customer_id#35, customer_preferred_cust_flag#36, year_total#37] -(203) Scan parquet +(204) Scan parquet Output [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(204) Filter +(205) Filter Input [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] Condition : (isnotnull(c_customer_sk#38) AND isnotnull(c_customer_id#39)) -(205) Exchange +(206) Exchange Input [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] Arguments: hashpartitioning(c_customer_sk#38, 100), ENSURE_REQUIREMENTS, [plan_id=28] -(206) Sort +(207) Sort Input [8]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45] Arguments: [c_customer_sk#38 ASC NULLS FIRST], false, 0 -(207) Scan parquet +(208) Scan parquet Output [4]: [ws_sold_date_sk#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(208) Filter +(209) Filter Input [4]: [ws_sold_date_sk#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49] Condition : (isnotnull(ws_bill_customer_sk#47) AND isnotnull(ws_sold_date_sk#46)) -(209) Exchange +(210) Exchange Input [4]: [ws_sold_date_sk#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49] Arguments: hashpartitioning(ws_bill_customer_sk#47, 100), ENSURE_REQUIREMENTS, [plan_id=29] -(210) Sort +(211) Sort Input [4]: [ws_sold_date_sk#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49] Arguments: [ws_bill_customer_sk#47 ASC NULLS FIRST], false, 0 -(211) SortMergeJoin +(212) SortMergeJoin Left keys [1]: [c_customer_sk#38] Right keys [1]: [ws_bill_customer_sk#47] Join type: Inner Join condition: None -(212) Project +(213) Project Output [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_sold_date_sk#46, ws_ext_discount_amt#48, ws_ext_list_price#49] Input [12]: [c_customer_sk#38, c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_sold_date_sk#46, ws_bill_customer_sk#47, ws_ext_discount_amt#48, ws_ext_list_price#49] -(213) Exchange +(214) Exchange Input [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_sold_date_sk#46, ws_ext_discount_amt#48, ws_ext_list_price#49] Arguments: hashpartitioning(ws_sold_date_sk#46, 100), ENSURE_REQUIREMENTS, [plan_id=30] -(214) Sort +(215) Sort Input [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_sold_date_sk#46, ws_ext_discount_amt#48, ws_ext_list_price#49] Arguments: [ws_sold_date_sk#46 ASC NULLS FIRST], false, 0 -(215) Scan parquet +(216) Scan parquet Output [2]: [d_date_sk#50, d_year#51] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(216) Filter +(217) Filter Input [2]: [d_date_sk#50, d_year#51] Condition : ((isnotnull(d_year#51) AND (d_year#51 = 2001)) AND isnotnull(d_date_sk#50)) -(217) Exchange +(218) Exchange Input [2]: [d_date_sk#50, d_year#51] Arguments: hashpartitioning(d_date_sk#50, 100), ENSURE_REQUIREMENTS, [plan_id=31] -(218) Sort +(219) Sort Input [2]: [d_date_sk#50, d_year#51] Arguments: [d_date_sk#50 ASC NULLS FIRST], false, 0 -(219) SortMergeJoin +(220) SortMergeJoin Left keys [1]: [ws_sold_date_sk#46] Right keys [1]: [d_date_sk#50] Join type: Inner Join condition: None -(220) Project +(221) Project Output [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#51] Input [12]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_sold_date_sk#46, ws_ext_discount_amt#48, ws_ext_list_price#49, d_date_sk#50, d_year#51] -(221) HashAggregate +(222) HashAggregate Input [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, ws_ext_discount_amt#48, ws_ext_list_price#49, d_year#51] Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] Aggregate Attributes [1]: [sum#53] Results [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#77] -(222) Exchange +(223) Exchange Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#77] Arguments: hashpartitioning(c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, 100), ENSURE_REQUIREMENTS, [plan_id=32] -(223) HashAggregate +(224) HashAggregate Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51, sum#77] Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#51] Functions [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))] Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#54] Results [2]: [c_customer_id#39 AS customer_id#55, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#49 - ws_ext_discount_amt#48)))#54,18,2) AS year_total#56] -(224) Filter +(225) Filter Input [2]: [customer_id#55, year_total#56] Condition : (isnotnull(year_total#56) AND (year_total#56 > 0.00)) -(225) Exchange +(226) Exchange Input [2]: [customer_id#55, year_total#56] Arguments: hashpartitioning(customer_id#55, 100), ENSURE_REQUIREMENTS, [plan_id=33] -(226) Sort +(227) Sort Input [2]: [customer_id#55, year_total#56] Arguments: [customer_id#55 ASC NULLS FIRST], false, 0 -(227) SortMergeJoin +(228) SortMergeJoin Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#55] Join type: Inner Join condition: None -(228) Project +(229) Project Output [5]: [customer_id#19, year_total#20, customer_preferred_cust_flag#36, year_total#37, year_total#56] Input [6]: [customer_id#19, year_total#20, customer_preferred_cust_flag#36, year_total#37, customer_id#55, year_total#56] -(229) Scan parquet +(230) Scan parquet Output [8]: [c_customer_sk#78, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(230) Filter +(231) Filter Input [8]: [c_customer_sk#78, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63] Condition : (isnotnull(c_customer_sk#78) AND isnotnull(c_customer_id#57)) -(231) Exchange +(232) Exchange Input [8]: [c_customer_sk#78, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63] Arguments: hashpartitioning(c_customer_sk#78, 100), ENSURE_REQUIREMENTS, [plan_id=34] -(232) Sort +(233) Sort Input [8]: [c_customer_sk#78, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63] Arguments: [c_customer_sk#78 ASC NULLS FIRST], false, 0 -(233) Scan parquet +(234) Scan parquet Output [4]: [ws_sold_date_sk#64, ws_bill_customer_sk#79, ws_ext_discount_amt#65, ws_ext_list_price#66] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(234) Filter +(235) Filter Input [4]: [ws_sold_date_sk#64, ws_bill_customer_sk#79, ws_ext_discount_amt#65, ws_ext_list_price#66] Condition : (isnotnull(ws_bill_customer_sk#79) AND isnotnull(ws_sold_date_sk#64)) -(235) Exchange +(236) Exchange Input [4]: [ws_sold_date_sk#64, ws_bill_customer_sk#79, ws_ext_discount_amt#65, ws_ext_list_price#66] Arguments: hashpartitioning(ws_bill_customer_sk#79, 100), ENSURE_REQUIREMENTS, [plan_id=35] -(236) Sort +(237) Sort Input [4]: [ws_sold_date_sk#64, ws_bill_customer_sk#79, ws_ext_discount_amt#65, ws_ext_list_price#66] Arguments: [ws_bill_customer_sk#79 ASC NULLS FIRST], false, 0 -(237) SortMergeJoin +(238) SortMergeJoin Left keys [1]: [c_customer_sk#78] Right keys [1]: [ws_bill_customer_sk#79] Join type: Inner Join condition: None -(238) Project +(239) Project Output [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, ws_sold_date_sk#64, ws_ext_discount_amt#65, ws_ext_list_price#66] Input [12]: [c_customer_sk#78, c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, ws_sold_date_sk#64, ws_bill_customer_sk#79, ws_ext_discount_amt#65, ws_ext_list_price#66] -(239) Exchange +(240) Exchange Input [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, ws_sold_date_sk#64, ws_ext_discount_amt#65, ws_ext_list_price#66] Arguments: hashpartitioning(ws_sold_date_sk#64, 100), ENSURE_REQUIREMENTS, [plan_id=36] -(240) Sort +(241) Sort Input [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, ws_sold_date_sk#64, ws_ext_discount_amt#65, ws_ext_list_price#66] Arguments: [ws_sold_date_sk#64 ASC NULLS FIRST], false, 0 -(241) Scan parquet +(242) Scan parquet Output [2]: [d_date_sk#67, d_year#68] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(242) Filter +(243) Filter Input [2]: [d_date_sk#67, d_year#68] Condition : ((isnotnull(d_year#68) AND (d_year#68 = 2002)) AND isnotnull(d_date_sk#67)) -(243) Exchange +(244) Exchange Input [2]: [d_date_sk#67, d_year#68] Arguments: hashpartitioning(d_date_sk#67, 100), ENSURE_REQUIREMENTS, [plan_id=37] -(244) Sort +(245) Sort Input [2]: [d_date_sk#67, d_year#68] Arguments: [d_date_sk#67 ASC NULLS FIRST], false, 0 -(245) SortMergeJoin +(246) SortMergeJoin Left keys [1]: [ws_sold_date_sk#64] Right keys [1]: [d_date_sk#67] Join type: Inner Join condition: None -(246) Project +(247) Project Output [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, ws_ext_discount_amt#65, ws_ext_list_price#66, d_year#68] Input [12]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, ws_sold_date_sk#64, ws_ext_discount_amt#65, ws_ext_list_price#66, d_date_sk#67, d_year#68] -(247) HashAggregate +(248) HashAggregate Input [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, ws_ext_discount_amt#65, ws_ext_list_price#66, d_year#68] Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#68] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#66 - ws_ext_discount_amt#65)))] Aggregate Attributes [1]: [sum#70] Results [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#68, sum#80] -(248) Exchange +(249) Exchange Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#68, sum#80] Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#68, 100), ENSURE_REQUIREMENTS, [plan_id=38] -(249) HashAggregate +(250) HashAggregate Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#68, sum#80] Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#68] Functions [1]: [sum(UnscaledValue((ws_ext_list_price#66 - ws_ext_discount_amt#65)))] Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#66 - ws_ext_discount_amt#65)))#54] Results [2]: [c_customer_id#57 AS customer_id#71, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#66 - ws_ext_discount_amt#65)))#54,18,2) AS year_total#72] -(250) Exchange +(251) Exchange Input [2]: [customer_id#71, year_total#72] Arguments: hashpartitioning(customer_id#71, 100), ENSURE_REQUIREMENTS, [plan_id=39] -(251) Sort +(252) Sort Input [2]: [customer_id#71, year_total#72] Arguments: [customer_id#71 ASC NULLS FIRST], false, 0 -(252) SortMergeJoin +(253) SortMergeJoin Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#71] Join type: Inner Join condition: (CASE WHEN (year_total#56 > 0.00) THEN (year_total#72 / year_total#56) END > CASE WHEN (year_total#20 > 0.00) THEN (year_total#37 / year_total#20) END) -(253) Project +(254) Project Output [1]: [customer_preferred_cust_flag#36] Input [7]: [customer_id#19, year_total#20, customer_preferred_cust_flag#36, year_total#37, year_total#56, customer_id#71, year_total#72] -(254) TakeOrderedAndProject +(255) TakeOrderedAndProject Input [1]: [customer_preferred_cust_flag#36] Arguments: X, [customer_preferred_cust_flag#36 ASC NULLS FIRST], [customer_preferred_cust_flag#36] -(255) AdaptiveSparkPlan +(256) AdaptiveSparkPlan Output [1]: [customer_preferred_cust_flag#36] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q13.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q13.txt index 3d7d4e40e..704c5bbdd 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q13.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q13.txt @@ -1,143 +1,146 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (137) +- == Final Plan == - * HashAggregate (86) - +- ShuffleQueryStage (85), Statistics(X) - +- Exchange (84) - +- * HashAggregate (83) - +- * Project (82) - +- * SortMergeJoin Inner (81) - :- NativeSort (72) - : +- InputAdapter (71) - : +- AQEShuffleRead (70) - : +- ShuffleQueryStage (69), Statistics(X) - : +- NativeShuffleExchange (68) - : +- ConvertToNative (67) - : +- * Project (66) - : +- * SortMergeJoin Inner (65) - : :- NativeSort (56) - : : +- InputAdapter (55) - : : +- AQEShuffleRead (54) - : : +- ShuffleQueryStage (53), Statistics(X) - : : +- NativeShuffleExchange (52) - : : +- NativeProject (51) - : : +- NativeSortMergeJoin Inner (50) - : : :- NativeSort (40) - : : : +- InputAdapter (39) - : : : +- AQEShuffleRead (38) - : : : +- ShuffleQueryStage (37), Statistics(X) - : : : +- NativeShuffleExchange (36) - : : : +- ConvertToNative (35) - : : : +- * Project (34) - : : : +- * SortMergeJoin Inner (33) - : : : :- NativeSort (23) - : : : : +- InputAdapter (22) - : : : : +- AQEShuffleRead (21) - : : : : +- ShuffleQueryStage (20), Statistics(X) - : : : : +- NativeShuffleExchange (19) - : : : : +- NativeProject (18) - : : : : +- NativeSortMergeJoin Inner (17) - : : : : :- NativeSort (8) - : : : : : +- InputAdapter (7) - : : : : : +- AQEShuffleRead (6) - : : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : : +- NativeShuffleExchange (4) - : : : : : +- NativeFilter (3) - : : : : : +- InputAdapter (2) - : : : : : +- NativeParquetScan (1) - : : : : +- NativeSort (16) - : : : : +- InputAdapter (15) - : : : : +- AQEShuffleRead (14) - : : : : +- ShuffleQueryStage (13), Statistics(X) - : : : : +- NativeShuffleExchange (12) - : : : : +- NativeFilter (11) - : : : : +- InputAdapter (10) - : : : : +- NativeParquetScan (9) - : : : +- NativeSort (32) - : : : +- InputAdapter (31) - : : : +- AQEShuffleRead (30) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- NativeShuffleExchange (28) - : : : +- NativeProject (27) - : : : +- NativeFilter (26) - : : : +- InputAdapter (25) - : : : +- NativeParquetScan (24) - : : +- NativeSort (49) - : : +- InputAdapter (48) - : : +- AQEShuffleRead (47) - : : +- ShuffleQueryStage (46), Statistics(X) - : : +- NativeShuffleExchange (45) - : : +- NativeProject (44) - : : +- NativeFilter (43) - : : +- InputAdapter (42) - : : +- NativeParquetScan (41) - : +- NativeSort (64) - : +- InputAdapter (63) - : +- AQEShuffleRead (62) - : +- ShuffleQueryStage (61), Statistics(X) - : +- NativeShuffleExchange (60) - : +- NativeFilter (59) - : +- InputAdapter (58) - : +- NativeParquetScan (57) - +- NativeSort (80) - +- InputAdapter (79) - +- AQEShuffleRead (78) - +- ShuffleQueryStage (77), Statistics(X) - +- NativeShuffleExchange (76) - +- NativeFilter (75) - +- InputAdapter (74) - +- NativeParquetScan (73) + NativeProject (89) + +- NativeHashAggregate (88) + +- ShuffleQueryStage (87), Statistics(X) + +- NativeShuffleExchange (86) + +- NativeHashAggregate (85) + +- NativeProject (84) + +- NativeProject (83) + +- NativeFilter (82) + +- NativeSortMergeJoin Inner (81) + :- NativeSort (72) + : +- InputAdapter (71) + : +- AQEShuffleRead (70) + : +- ShuffleQueryStage (69), Statistics(X) + : +- NativeShuffleExchange (68) + : +- NativeProject (67) + : +- NativeFilter (66) + : +- NativeSortMergeJoin Inner (65) + : :- NativeSort (56) + : : +- InputAdapter (55) + : : +- AQEShuffleRead (54) + : : +- ShuffleQueryStage (53), Statistics(X) + : : +- NativeShuffleExchange (52) + : : +- NativeProject (51) + : : +- NativeSortMergeJoin Inner (50) + : : :- NativeSort (40) + : : : +- InputAdapter (39) + : : : +- AQEShuffleRead (38) + : : : +- ShuffleQueryStage (37), Statistics(X) + : : : +- NativeShuffleExchange (36) + : : : +- NativeProject (35) + : : : +- NativeFilter (34) + : : : +- NativeSortMergeJoin Inner (33) + : : : :- NativeSort (23) + : : : : +- InputAdapter (22) + : : : : +- AQEShuffleRead (21) + : : : : +- ShuffleQueryStage (20), Statistics(X) + : : : : +- NativeShuffleExchange (19) + : : : : +- NativeProject (18) + : : : : +- NativeSortMergeJoin Inner (17) + : : : : :- NativeSort (8) + : : : : : +- InputAdapter (7) + : : : : : +- AQEShuffleRead (6) + : : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : : +- NativeShuffleExchange (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (16) + : : : : +- InputAdapter (15) + : : : : +- AQEShuffleRead (14) + : : : : +- ShuffleQueryStage (13), Statistics(X) + : : : : +- NativeShuffleExchange (12) + : : : : +- NativeFilter (11) + : : : : +- InputAdapter (10) + : : : : +- NativeParquetScan (9) + : : : +- NativeSort (32) + : : : +- InputAdapter (31) + : : : +- AQEShuffleRead (30) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- NativeShuffleExchange (28) + : : : +- NativeProject (27) + : : : +- NativeFilter (26) + : : : +- InputAdapter (25) + : : : +- NativeParquetScan (24) + : : +- NativeSort (49) + : : +- InputAdapter (48) + : : +- AQEShuffleRead (47) + : : +- ShuffleQueryStage (46), Statistics(X) + : : +- NativeShuffleExchange (45) + : : +- NativeProject (44) + : : +- NativeFilter (43) + : : +- InputAdapter (42) + : : +- NativeParquetScan (41) + : +- NativeSort (64) + : +- InputAdapter (63) + : +- AQEShuffleRead (62) + : +- ShuffleQueryStage (61), Statistics(X) + : +- NativeShuffleExchange (60) + : +- NativeFilter (59) + : +- InputAdapter (58) + : +- NativeParquetScan (57) + +- NativeSort (80) + +- InputAdapter (79) + +- AQEShuffleRead (78) + +- ShuffleQueryStage (77), Statistics(X) + +- NativeShuffleExchange (76) + +- NativeFilter (75) + +- InputAdapter (74) + +- NativeParquetScan (73) +- == Initial Plan == - HashAggregate (133) - +- Exchange (132) - +- HashAggregate (131) - +- Project (130) - +- SortMergeJoin Inner (129) - :- Sort (124) - : +- Exchange (123) - : +- Project (122) - : +- SortMergeJoin Inner (121) - : :- Sort (116) - : : +- Exchange (115) - : : +- Project (114) - : : +- SortMergeJoin Inner (113) - : : :- Sort (107) - : : : +- Exchange (106) - : : : +- Project (105) - : : : +- SortMergeJoin Inner (104) - : : : :- Sort (98) - : : : : +- Exchange (97) - : : : : +- Project (96) - : : : : +- SortMergeJoin Inner (95) - : : : : :- Sort (90) - : : : : : +- Exchange (89) - : : : : : +- Filter (88) - : : : : : +- Scan parquet (87) - : : : : +- Sort (94) - : : : : +- Exchange (93) - : : : : +- Filter (92) - : : : : +- Scan parquet (91) - : : : +- Sort (103) - : : : +- Exchange (102) - : : : +- Project (101) - : : : +- Filter (100) - : : : +- Scan parquet (99) - : : +- Sort (112) - : : +- Exchange (111) - : : +- Project (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- Sort (120) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Sort (128) - +- Exchange (127) - +- Filter (126) - +- Scan parquet (125) - - -(87) Scan parquet + HashAggregate (136) + +- Exchange (135) + +- HashAggregate (134) + +- Project (133) + +- SortMergeJoin Inner (132) + :- Sort (127) + : +- Exchange (126) + : +- Project (125) + : +- SortMergeJoin Inner (124) + : :- Sort (119) + : : +- Exchange (118) + : : +- Project (117) + : : +- SortMergeJoin Inner (116) + : : :- Sort (110) + : : : +- Exchange (109) + : : : +- Project (108) + : : : +- SortMergeJoin Inner (107) + : : : :- Sort (101) + : : : : +- Exchange (100) + : : : : +- Project (99) + : : : : +- SortMergeJoin Inner (98) + : : : : :- Sort (93) + : : : : : +- Exchange (92) + : : : : : +- Filter (91) + : : : : : +- Scan parquet (90) + : : : : +- Sort (97) + : : : : +- Exchange (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Sort (106) + : : : +- Exchange (105) + : : : +- Project (104) + : : : +- Filter (103) + : : : +- Scan parquet (102) + : : +- Sort (115) + : : +- Exchange (114) + : : +- Project (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Sort (123) + : +- Exchange (122) + : +- Filter (121) + : +- Scan parquet (120) + +- Sort (131) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) + + +(90) Scan parquet Output [10]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Batched: true Location: InMemoryFileIndex [file:/] @@ -171,7 +174,7 @@ Input [10]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8, #9#9, #10#10] Input [10]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8, #9#9, #10#10] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false -(91) Scan parquet +(94) Scan parquet Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [file:/] @@ -234,7 +237,7 @@ Input [9]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_qu Input [9]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Arguments: [ss_addr_sk#4 ASC NULLS FIRST], false -(99) Scan parquet +(102) Scan parquet Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Batched: true Location: InMemoryFileIndex [file:/] @@ -272,18 +275,19 @@ Input [2]: [ca_address_sk#12, ca_state#13] Input [2]: [ca_address_sk#12, ca_state#13] Arguments: [ca_address_sk#12 ASC NULLS FIRST], false -(33) SortMergeJoin [codegen id : X] +(33) NativeSortMergeJoin Left keys [1]: [ss_addr_sk#4] Right keys [1]: [ca_address_sk#12] Join type: Inner -Join condition: ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#10 >= 100.00)) AND (ss_net_profit#10 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#10 >= 150.00)) AND (ss_net_profit#10 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#10 >= 50.00)) AND (ss_net_profit#10 <= 250.00))) +Join condition: None -(34) Project [codegen id : X] -Output [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] +(34) NativeFilter Input [11]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10, ca_address_sk#12, ca_state#13] +Condition : ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#10 >= 100.00)) AND (ss_net_profit#10 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#10 >= 150.00)) AND (ss_net_profit#10 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#10 >= 50.00)) AND (ss_net_profit#10 <= 250.00))) -(35) ConvertToNative -Input [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] +(35) NativeProject +Output [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] +Input [11]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10, ca_address_sk#12, ca_state#13] (36) NativeShuffleExchange Input [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] @@ -304,7 +308,7 @@ Input [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_s Input [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false -(108) Scan parquet +(111) Scan parquet Output [2]: [d_date_sk#15, d_year#16] Batched: true Location: InMemoryFileIndex [file:/] @@ -371,7 +375,7 @@ Input [6]: [ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ex Input [6]: [ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Arguments: [ss_cdemo_sk#2 ASC NULLS FIRST], false -(117) Scan parquet +(120) Scan parquet Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] Batched: true Location: InMemoryFileIndex [file:/] @@ -405,18 +409,19 @@ Input [3]: [#17#17, #18#18, #19#19] Input [3]: [#17#17, #18#18, #19#19] Arguments: [cd_demo_sk#17 ASC NULLS FIRST], false -(65) SortMergeJoin [codegen id : X] +(65) NativeSortMergeJoin Left keys [1]: [ss_cdemo_sk#2] Right keys [1]: [cd_demo_sk#17] Join type: Inner -Join condition: ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) AND (ss_sales_price#7 >= 100.00)) AND (ss_sales_price#7 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College)) AND (ss_sales_price#7 >= 50.00)) AND (ss_sales_price#7 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)) AND (ss_sales_price#7 >= 150.00)) AND (ss_sales_price#7 <= 200.00))) +Join condition: None -(66) Project [codegen id : X] -Output [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19] +(66) NativeFilter Input [9]: [ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, #17#17, #18#18, #19#19] +Condition : ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) AND (ss_sales_price#7 >= 100.00)) AND (ss_sales_price#7 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College)) AND (ss_sales_price#7 >= 50.00)) AND (ss_sales_price#7 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)) AND (ss_sales_price#7 >= 150.00)) AND (ss_sales_price#7 <= 200.00))) -(67) ConvertToNative -Input [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19] +(67) NativeProject +Output [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19] +Input [9]: [ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, #17#17, #18#18, #19#19] (68) NativeShuffleExchange Input [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19] @@ -437,7 +442,7 @@ Input [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8 Input [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19] Arguments: [ss_hdemo_sk#3 ASC NULLS FIRST], false -(125) Scan parquet +(128) Scan parquet Output [2]: [hd_demo_sk#20, hd_dep_count#21] Batched: true Location: InMemoryFileIndex [file:/] @@ -471,261 +476,273 @@ Input [2]: [#20#20, #21#21] Input [2]: [#20#20, #21#21] Arguments: [hd_demo_sk#20 ASC NULLS FIRST], false -(81) SortMergeJoin [codegen id : X] +(81) NativeSortMergeJoin Left keys [1]: [ss_hdemo_sk#3] Right keys [1]: [hd_demo_sk#20] Join type: Inner -Join condition: (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) AND (ss_sales_price#7 >= 100.00)) AND (ss_sales_price#7 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College)) AND (ss_sales_price#7 >= 50.00)) AND (ss_sales_price#7 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)) AND (ss_sales_price#7 >= 150.00)) AND (ss_sales_price#7 <= 200.00)) AND (hd_dep_count#21 = 1))) +Join condition: None -(82) Project [codegen id : X] +(82) NativeFilter +Input [9]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19, #20#20, #21#21] +Condition : (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) AND (ss_sales_price#7 >= 100.00)) AND (ss_sales_price#7 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College)) AND (ss_sales_price#7 >= 50.00)) AND (ss_sales_price#7 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)) AND (ss_sales_price#7 >= 150.00)) AND (ss_sales_price#7 <= 200.00)) AND (hd_dep_count#21 = 1))) + +(83) NativeProject Output [3]: [ss_quantity#6, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Input [9]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19, #20#20, #21#21] -(83) HashAggregate [codegen id : X] +(84) NativeProject +Output [3]: [ss_quantity#6 AS _c0#22, UnscaledValue(ss_ext_sales_price#8) AS _c1#23, UnscaledValue(ss_ext_wholesale_cost#9) AS _c2#24] Input [3]: [ss_quantity#6, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] + +(85) NativeHashAggregate +Input [3]: [_c0#22, _c1#23, _c2#24] Keys: [] -Functions [4]: [partial_avg(ss_quantity#6), partial_avg(UnscaledValue(ss_ext_sales_price#8)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#9)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#9))] -Aggregate Attributes [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Results [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +Functions [4]: [partial_avg(_c0#22), partial_avg(_c1#23), partial_avg(_c2#24), partial_sum(_c2#24)] +Aggregate Attributes [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] +Results [7]: [#32, #33, #32, #33, #32, #33, #33] -(84) Exchange -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +(86) NativeShuffleExchange +Input [7]: [#32, #33, #32, #33, #32, #33, #33] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] -(85) ShuffleQueryStage -Output [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +(87) ShuffleQueryStage +Output [7]: [#32, #33, #32, #33, #32, #33, #33] Arguments: X -(86) HashAggregate [codegen id : X] -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +(88) NativeHashAggregate +Input [7]: [#32, #33, #32, #33, #32, #33, #33] Keys: [] Functions [4]: [avg(ss_quantity#6), avg(UnscaledValue(ss_ext_sales_price#8)), avg(UnscaledValue(ss_ext_wholesale_cost#9)), sum(UnscaledValue(ss_ext_wholesale_cost#9))] -Aggregate Attributes [4]: [avg(ss_quantity#6)#36, avg(UnscaledValue(ss_ext_sales_price#8))#37, avg(UnscaledValue(ss_ext_wholesale_cost#9))#38, sum(UnscaledValue(ss_ext_wholesale_cost#9))#39] -Results [4]: [avg(ss_quantity#6)#36 AS avg(ss_quantity)#40, cast((avg(UnscaledValue(ss_ext_sales_price#8))#37 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#41, cast((avg(UnscaledValue(ss_ext_wholesale_cost#9))#38 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#42, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#9))#39,17,2) AS sum(ss_ext_wholesale_cost)#43] +Aggregate Attributes [4]: [avg(ss_quantity#6)#34, avg(UnscaledValue(ss_ext_sales_price#8))#35, avg(UnscaledValue(ss_ext_wholesale_cost#9))#36, sum(UnscaledValue(ss_ext_wholesale_cost#9))#37] +Results [4]: [avg(ss_quantity#6)#34, avg(UnscaledValue(ss_ext_sales_price#8))#35, avg(UnscaledValue(ss_ext_wholesale_cost#9))#36, sum(UnscaledValue(ss_ext_wholesale_cost#9))#37] + +(89) NativeProject +Output [4]: [avg(ss_quantity#6)#34 AS avg(ss_quantity)#38, cast((avg(UnscaledValue(ss_ext_sales_price#8))#35 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#39, cast((avg(UnscaledValue(ss_ext_wholesale_cost#9))#36 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#40, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#9))#37,17,2) AS sum(ss_ext_wholesale_cost)#41] +Input [4]: [avg(ss_quantity#6)#34, avg(UnscaledValue(ss_ext_sales_price#8))#35, avg(UnscaledValue(ss_ext_wholesale_cost#9))#36, sum(UnscaledValue(ss_ext_wholesale_cost#9))#37] -(87) Scan parquet +(90) Scan parquet Output [10]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_sold_date_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_hdemo_sk), Or(Or(And(GreaterThanOrEqual(ss_net_profit,100.00),LessThanOrEqual(ss_net_profit,200.00)),And(GreaterThanOrEqual(ss_net_profit,150.00),LessThanOrEqual(ss_net_profit,300.00))),And(GreaterThanOrEqual(ss_net_profit,50.00),LessThanOrEqual(ss_net_profit,250.00))), Or(Or(And(GreaterThanOrEqual(ss_sales_price,100.00),LessThanOrEqual(ss_sales_price,150.00)),And(GreaterThanOrEqual(ss_sales_price,50.00),LessThanOrEqual(ss_sales_price,100.00))),And(GreaterThanOrEqual(ss_sales_price,150.00),LessThanOrEqual(ss_sales_price,200.00)))] ReadSchema: struct -(88) Filter +(91) Filter Input [10]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Condition : ((((((isnotnull(ss_store_sk#5) AND isnotnull(ss_addr_sk#4)) AND isnotnull(ss_sold_date_sk#1)) AND isnotnull(ss_cdemo_sk#2)) AND isnotnull(ss_hdemo_sk#3)) AND ((((ss_net_profit#10 >= 100.00) AND (ss_net_profit#10 <= 200.00)) OR ((ss_net_profit#10 >= 150.00) AND (ss_net_profit#10 <= 300.00))) OR ((ss_net_profit#10 >= 50.00) AND (ss_net_profit#10 <= 250.00)))) AND ((((ss_sales_price#7 >= 100.00) AND (ss_sales_price#7 <= 150.00)) OR ((ss_sales_price#7 >= 50.00) AND (ss_sales_price#7 <= 100.00))) OR ((ss_sales_price#7 >= 150.00) AND (ss_sales_price#7 <= 200.00)))) -(89) Exchange +(92) Exchange Input [10]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Arguments: hashpartitioning(ss_store_sk#5, 100), ENSURE_REQUIREMENTS, [plan_id=12] -(90) Sort +(93) Sort Input [10]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false, 0 -(91) Scan parquet +(94) Scan parquet Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(92) Filter +(95) Filter Input [1]: [s_store_sk#11] Condition : isnotnull(s_store_sk#11) -(93) Exchange +(96) Exchange Input [1]: [s_store_sk#11] Arguments: hashpartitioning(s_store_sk#11, 100), ENSURE_REQUIREMENTS, [plan_id=13] -(94) Sort +(97) Sort Input [1]: [s_store_sk#11] Arguments: [s_store_sk#11 ASC NULLS FIRST], false, 0 -(95) SortMergeJoin +(98) SortMergeJoin Left keys [1]: [ss_store_sk#5] Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None -(96) Project +(99) Project Output [9]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Input [11]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10, s_store_sk#11] -(97) Exchange +(100) Exchange Input [9]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Arguments: hashpartitioning(ss_addr_sk#4, 100), ENSURE_REQUIREMENTS, [plan_id=14] -(98) Sort +(101) Sort Input [9]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10] Arguments: [ss_addr_sk#4 ASC NULLS FIRST], false, 0 -(99) Scan parquet +(102) Scan parquet Output [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_country), EqualTo(ca_country,United States), IsNotNull(ca_address_sk), Or(Or(In(ca_state, [OH,TX]),In(ca_state, [KY,NM,OR])),In(ca_state, [MS,TX,VA]))] ReadSchema: struct -(100) Filter +(103) Filter Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] Condition : (((isnotnull(ca_country#14) AND (ca_country#14 = United States)) AND isnotnull(ca_address_sk#12)) AND ((ca_state#13 IN (TX,OH) OR ca_state#13 IN (OR,NM,KY)) OR ca_state#13 IN (VA,TX,MS))) -(101) Project +(104) Project Output [2]: [ca_address_sk#12, ca_state#13] Input [3]: [ca_address_sk#12, ca_state#13, ca_country#14] -(102) Exchange +(105) Exchange Input [2]: [ca_address_sk#12, ca_state#13] Arguments: hashpartitioning(ca_address_sk#12, 100), ENSURE_REQUIREMENTS, [plan_id=15] -(103) Sort +(106) Sort Input [2]: [ca_address_sk#12, ca_state#13] Arguments: [ca_address_sk#12 ASC NULLS FIRST], false, 0 -(104) SortMergeJoin +(107) SortMergeJoin Left keys [1]: [ss_addr_sk#4] Right keys [1]: [ca_address_sk#12] Join type: Inner Join condition: ((((ca_state#13 IN (TX,OH) AND (ss_net_profit#10 >= 100.00)) AND (ss_net_profit#10 <= 200.00)) OR ((ca_state#13 IN (OR,NM,KY) AND (ss_net_profit#10 >= 150.00)) AND (ss_net_profit#10 <= 300.00))) OR ((ca_state#13 IN (VA,TX,MS) AND (ss_net_profit#10 >= 50.00)) AND (ss_net_profit#10 <= 250.00))) -(105) Project +(108) Project Output [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Input [11]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, ss_net_profit#10, ca_address_sk#12, ca_state#13] -(106) Exchange +(109) Exchange Input [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Arguments: hashpartitioning(ss_sold_date_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=16] -(107) Sort +(110) Sort Input [7]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false, 0 -(108) Scan parquet +(111) Scan parquet Output [2]: [d_date_sk#15, d_year#16] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(109) Filter +(112) Filter Input [2]: [d_date_sk#15, d_year#16] Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) -(110) Project +(113) Project Output [1]: [d_date_sk#15] Input [2]: [d_date_sk#15, d_year#16] -(111) Exchange +(114) Exchange Input [1]: [d_date_sk#15] Arguments: hashpartitioning(d_date_sk#15, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(112) Sort +(115) Sort Input [1]: [d_date_sk#15] Arguments: [d_date_sk#15 ASC NULLS FIRST], false, 0 -(113) SortMergeJoin +(116) SortMergeJoin Left keys [1]: [ss_sold_date_sk#1] Right keys [1]: [d_date_sk#15] Join type: Inner Join condition: None -(114) Project +(117) Project Output [6]: [ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Input [8]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, d_date_sk#15] -(115) Exchange +(118) Exchange Input [6]: [ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Arguments: hashpartitioning(ss_cdemo_sk#2, 100), ENSURE_REQUIREMENTS, [plan_id=18] -(116) Sort +(119) Sort Input [6]: [ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Arguments: [ss_cdemo_sk#2 ASC NULLS FIRST], false, 0 -(117) Scan parquet +(120) Scan parquet Output [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cd_demo_sk), Or(Or(And(EqualTo(cd_marital_status,M),EqualTo(cd_education_status,Advanced Degree)),And(EqualTo(cd_marital_status,S),EqualTo(cd_education_status,College))),And(EqualTo(cd_marital_status,W),EqualTo(cd_education_status,2 yr Degree)))] ReadSchema: struct -(118) Filter +(121) Filter Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] Condition : (isnotnull(cd_demo_sk#17) AND ((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) OR ((cd_marital_status#18 = S) AND (cd_education_status#19 = College))) OR ((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)))) -(119) Exchange +(122) Exchange Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] Arguments: hashpartitioning(cd_demo_sk#17, 100), ENSURE_REQUIREMENTS, [plan_id=19] -(120) Sort +(123) Sort Input [3]: [cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] Arguments: [cd_demo_sk#17 ASC NULLS FIRST], false, 0 -(121) SortMergeJoin +(124) SortMergeJoin Left keys [1]: [ss_cdemo_sk#2] Right keys [1]: [cd_demo_sk#17] Join type: Inner Join condition: ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) AND (ss_sales_price#7 >= 100.00)) AND (ss_sales_price#7 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College)) AND (ss_sales_price#7 >= 50.00)) AND (ss_sales_price#7 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)) AND (ss_sales_price#7 >= 150.00)) AND (ss_sales_price#7 <= 200.00))) -(122) Project +(125) Project Output [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19] Input [9]: [ss_cdemo_sk#2, ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_demo_sk#17, cd_marital_status#18, cd_education_status#19] -(123) Exchange +(126) Exchange Input [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19] Arguments: hashpartitioning(ss_hdemo_sk#3, 100), ENSURE_REQUIREMENTS, [plan_id=20] -(124) Sort +(127) Sort Input [7]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19] Arguments: [ss_hdemo_sk#3 ASC NULLS FIRST], false, 0 -(125) Scan parquet +(128) Scan parquet Output [2]: [hd_demo_sk#20, hd_dep_count#21] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(hd_demo_sk), Or(EqualTo(hd_dep_count,3),EqualTo(hd_dep_count,1))] ReadSchema: struct -(126) Filter +(129) Filter Input [2]: [hd_demo_sk#20, hd_dep_count#21] Condition : (isnotnull(hd_demo_sk#20) AND ((hd_dep_count#21 = 3) OR (hd_dep_count#21 = 1))) -(127) Exchange +(130) Exchange Input [2]: [hd_demo_sk#20, hd_dep_count#21] Arguments: hashpartitioning(hd_demo_sk#20, 100), ENSURE_REQUIREMENTS, [plan_id=21] -(128) Sort +(131) Sort Input [2]: [hd_demo_sk#20, hd_dep_count#21] Arguments: [hd_demo_sk#20 ASC NULLS FIRST], false, 0 -(129) SortMergeJoin +(132) SortMergeJoin Left keys [1]: [ss_hdemo_sk#3] Right keys [1]: [hd_demo_sk#20] Join type: Inner Join condition: (((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) AND (ss_sales_price#7 >= 100.00)) AND (ss_sales_price#7 <= 150.00)) AND (hd_dep_count#21 = 3)) OR (((((cd_marital_status#18 = S) AND (cd_education_status#19 = College)) AND (ss_sales_price#7 >= 50.00)) AND (ss_sales_price#7 <= 100.00)) AND (hd_dep_count#21 = 1))) OR (((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)) AND (ss_sales_price#7 >= 150.00)) AND (ss_sales_price#7 <= 200.00)) AND (hd_dep_count#21 = 1))) -(130) Project +(133) Project Output [3]: [ss_quantity#6, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Input [9]: [ss_hdemo_sk#3, ss_quantity#6, ss_sales_price#7, ss_ext_sales_price#8, ss_ext_wholesale_cost#9, cd_marital_status#18, cd_education_status#19, hd_demo_sk#20, hd_dep_count#21] -(131) HashAggregate +(134) HashAggregate Input [3]: [ss_quantity#6, ss_ext_sales_price#8, ss_ext_wholesale_cost#9] Keys: [] Functions [4]: [partial_avg(ss_quantity#6), partial_avg(UnscaledValue(ss_ext_sales_price#8)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#9)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#9))] -Aggregate Attributes [7]: [sum#22, count#23, sum#24, count#25, sum#26, count#27, sum#28] -Results [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +Aggregate Attributes [7]: [sum#25, count#26, sum#27, count#28, sum#29, count#30, sum#31] +Results [7]: [sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48] -(132) Exchange -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +(135) Exchange +Input [7]: [sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=22] -(133) HashAggregate -Input [7]: [sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35] +(136) HashAggregate +Input [7]: [sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48] Keys: [] Functions [4]: [avg(ss_quantity#6), avg(UnscaledValue(ss_ext_sales_price#8)), avg(UnscaledValue(ss_ext_wholesale_cost#9)), sum(UnscaledValue(ss_ext_wholesale_cost#9))] -Aggregate Attributes [4]: [avg(ss_quantity#6)#36, avg(UnscaledValue(ss_ext_sales_price#8))#37, avg(UnscaledValue(ss_ext_wholesale_cost#9))#38, sum(UnscaledValue(ss_ext_wholesale_cost#9))#39] -Results [4]: [avg(ss_quantity#6)#36 AS avg(ss_quantity)#40, cast((avg(UnscaledValue(ss_ext_sales_price#8))#37 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#41, cast((avg(UnscaledValue(ss_ext_wholesale_cost#9))#38 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#42, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#9))#39,17,2) AS sum(ss_ext_wholesale_cost)#43] +Aggregate Attributes [4]: [avg(ss_quantity#6)#34, avg(UnscaledValue(ss_ext_sales_price#8))#35, avg(UnscaledValue(ss_ext_wholesale_cost#9))#36, sum(UnscaledValue(ss_ext_wholesale_cost#9))#37] +Results [4]: [avg(ss_quantity#6)#34 AS avg(ss_quantity)#38, cast((avg(UnscaledValue(ss_ext_sales_price#8))#35 / 100.0) as decimal(11,6)) AS avg(ss_ext_sales_price)#39, cast((avg(UnscaledValue(ss_ext_wholesale_cost#9))#36 / 100.0) as decimal(11,6)) AS avg(ss_ext_wholesale_cost)#40, MakeDecimal(sum(UnscaledValue(ss_ext_wholesale_cost#9))#37,17,2) AS sum(ss_ext_wholesale_cost)#41] -(134) AdaptiveSparkPlan -Output [4]: [avg(ss_quantity)#40, avg(ss_ext_sales_price)#41, avg(ss_ext_wholesale_cost)#42, sum(ss_ext_wholesale_cost)#43] +(137) AdaptiveSparkPlan +Output [4]: [avg(ss_quantity)#38, avg(ss_ext_sales_price)#39, avg(ss_ext_wholesale_cost)#40, sum(ss_ext_wholesale_cost)#41] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q15.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q15.txt index 36e1a7500..b40137dbb 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q15.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q15.txt @@ -17,9 +17,9 @@ AdaptiveSparkPlan (91) : +- AQEShuffleRead (37) : +- ShuffleQueryStage (36), Statistics(X) : +- NativeShuffleExchange (35) - : +- ConvertToNative (34) - : +- * Project (33) - : +- * SortMergeJoin Inner (32) + : +- NativeProject (34) + : +- NativeFilter (33) + : +- NativeSortMergeJoin Inner (32) : :- NativeSort (23) : : +- InputAdapter (22) : : +- AQEShuffleRead (21) @@ -225,18 +225,19 @@ Input [3]: [#6#6, #7#7, #8#8] Input [3]: [#6#6, #7#7, #8#8] Arguments: [ca_address_sk#6 ASC NULLS FIRST], false -(32) SortMergeJoin [codegen id : X] +(32) NativeSortMergeJoin Left keys [1]: [c_current_addr_sk#5] Right keys [1]: [ca_address_sk#6] Join type: Inner -Join condition: ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#3 > 500.00)) +Join condition: None -(33) Project [codegen id : X] -Output [3]: [cs_sold_date_sk#1, cs_sales_price#3, ca_zip#8] +(33) NativeFilter Input [6]: [cs_sold_date_sk#1, cs_sales_price#3, c_current_addr_sk#5, #6#6, #7#7, #8#8] +Condition : ((substr(ca_zip#8, 1, 5) IN (85669,86197,88274,83405,86475,85392,85460,80348,81792) OR ca_state#7 IN (CA,WA,GA)) OR (cs_sales_price#3 > 500.00)) -(34) ConvertToNative -Input [3]: [cs_sold_date_sk#1, cs_sales_price#3, ca_zip#8] +(34) NativeProject +Output [3]: [cs_sold_date_sk#1, cs_sales_price#3, ca_zip#8] +Input [6]: [cs_sold_date_sk#1, cs_sales_price#3, c_current_addr_sk#5, #6#6, #7#7, #8#8] (35) NativeShuffleExchange Input [3]: [cs_sold_date_sk#1, cs_sales_price#3, ca_zip#8] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q19.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q19.txt index 0e7874f20..c904bc127 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q19.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q19.txt @@ -1,144 +1,148 @@ == Physical Plan == -AdaptiveSparkPlan (135) +AdaptiveSparkPlan (139) +- == Final Plan == - TakeOrderedAndProject (86) - +- * HashAggregate (85) - +- AQEShuffleRead (84) - +- ShuffleQueryStage (83), Statistics(X) - +- Exchange (82) - +- * HashAggregate (81) - +- * Project (80) - +- * SortMergeJoin Inner (79) - :- NativeSort (70) - : +- InputAdapter (69) - : +- AQEShuffleRead (68) - : +- ShuffleQueryStage (67), Statistics(X) - : +- NativeShuffleExchange (66) - : +- NativeProject (65) - : +- NativeSortMergeJoin Inner (64) - : :- NativeSort (55) - : : +- InputAdapter (54) - : : +- AQEShuffleRead (53) - : : +- ShuffleQueryStage (52), Statistics(X) - : : +- NativeShuffleExchange (51) - : : +- NativeProject (50) - : : +- NativeSortMergeJoin Inner (49) - : : :- NativeSort (40) - : : : +- InputAdapter (39) - : : : +- AQEShuffleRead (38) - : : : +- ShuffleQueryStage (37), Statistics(X) - : : : +- NativeShuffleExchange (36) - : : : +- NativeProject (35) - : : : +- NativeSortMergeJoin Inner (34) - : : : :- NativeSort (24) - : : : : +- InputAdapter (23) - : : : : +- AQEShuffleRead (22) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- NativeShuffleExchange (20) - : : : : +- NativeProject (19) - : : : : +- NativeSortMergeJoin Inner (18) - : : : : :- NativeSort (9) - : : : : : +- InputAdapter (8) - : : : : : +- AQEShuffleRead (7) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- NativeShuffleExchange (5) - : : : : : +- NativeProject (4) - : : : : : +- NativeFilter (3) - : : : : : +- InputAdapter (2) - : : : : : +- NativeParquetScan (1) - : : : : +- NativeSort (17) - : : : : +- InputAdapter (16) - : : : : +- AQEShuffleRead (15) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- NativeShuffleExchange (13) - : : : : +- NativeFilter (12) - : : : : +- InputAdapter (11) - : : : : +- NativeParquetScan (10) - : : : +- NativeSort (33) - : : : +- InputAdapter (32) - : : : +- AQEShuffleRead (31) - : : : +- ShuffleQueryStage (30), Statistics(X) - : : : +- NativeShuffleExchange (29) - : : : +- NativeProject (28) - : : : +- NativeFilter (27) - : : : +- InputAdapter (26) - : : : +- NativeParquetScan (25) - : : +- NativeSort (48) - : : +- InputAdapter (47) - : : +- AQEShuffleRead (46) - : : +- ShuffleQueryStage (45), Statistics(X) - : : +- NativeShuffleExchange (44) - : : +- NativeFilter (43) - : : +- InputAdapter (42) - : : +- NativeParquetScan (41) - : +- NativeSort (63) - : +- InputAdapter (62) - : +- AQEShuffleRead (61) - : +- ShuffleQueryStage (60), Statistics(X) - : +- NativeShuffleExchange (59) - : +- NativeFilter (58) - : +- InputAdapter (57) - : +- NativeParquetScan (56) - +- NativeSort (78) - +- InputAdapter (77) - +- AQEShuffleRead (76) - +- ShuffleQueryStage (75), Statistics(X) - +- NativeShuffleExchange (74) - +- NativeFilter (73) - +- InputAdapter (72) - +- NativeParquetScan (71) + NativeTakeOrdered (90) + +- NativeProject (89) + +- NativeHashAggregate (88) + +- InputAdapter (87) + +- AQEShuffleRead (86) + +- ShuffleQueryStage (85), Statistics(X) + +- NativeShuffleExchange (84) + +- NativeHashAggregate (83) + +- NativeProject (82) + +- NativeProject (81) + +- NativeFilter (80) + +- NativeSortMergeJoin Inner (79) + :- NativeSort (70) + : +- InputAdapter (69) + : +- AQEShuffleRead (68) + : +- ShuffleQueryStage (67), Statistics(X) + : +- NativeShuffleExchange (66) + : +- NativeProject (65) + : +- NativeSortMergeJoin Inner (64) + : :- NativeSort (55) + : : +- InputAdapter (54) + : : +- AQEShuffleRead (53) + : : +- ShuffleQueryStage (52), Statistics(X) + : : +- NativeShuffleExchange (51) + : : +- NativeProject (50) + : : +- NativeSortMergeJoin Inner (49) + : : :- NativeSort (40) + : : : +- InputAdapter (39) + : : : +- AQEShuffleRead (38) + : : : +- ShuffleQueryStage (37), Statistics(X) + : : : +- NativeShuffleExchange (36) + : : : +- NativeProject (35) + : : : +- NativeSortMergeJoin Inner (34) + : : : :- NativeSort (24) + : : : : +- InputAdapter (23) + : : : : +- AQEShuffleRead (22) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- NativeShuffleExchange (20) + : : : : +- NativeProject (19) + : : : : +- NativeSortMergeJoin Inner (18) + : : : : :- NativeSort (9) + : : : : : +- InputAdapter (8) + : : : : : +- AQEShuffleRead (7) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- NativeShuffleExchange (5) + : : : : : +- NativeProject (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (17) + : : : : +- InputAdapter (16) + : : : : +- AQEShuffleRead (15) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- NativeShuffleExchange (13) + : : : : +- NativeFilter (12) + : : : : +- InputAdapter (11) + : : : : +- NativeParquetScan (10) + : : : +- NativeSort (33) + : : : +- InputAdapter (32) + : : : +- AQEShuffleRead (31) + : : : +- ShuffleQueryStage (30), Statistics(X) + : : : +- NativeShuffleExchange (29) + : : : +- NativeProject (28) + : : : +- NativeFilter (27) + : : : +- InputAdapter (26) + : : : +- NativeParquetScan (25) + : : +- NativeSort (48) + : : +- InputAdapter (47) + : : +- AQEShuffleRead (46) + : : +- ShuffleQueryStage (45), Statistics(X) + : : +- NativeShuffleExchange (44) + : : +- NativeFilter (43) + : : +- InputAdapter (42) + : : +- NativeParquetScan (41) + : +- NativeSort (63) + : +- InputAdapter (62) + : +- AQEShuffleRead (61) + : +- ShuffleQueryStage (60), Statistics(X) + : +- NativeShuffleExchange (59) + : +- NativeFilter (58) + : +- InputAdapter (57) + : +- NativeParquetScan (56) + +- NativeSort (78) + +- InputAdapter (77) + +- AQEShuffleRead (76) + +- ShuffleQueryStage (75), Statistics(X) + +- NativeShuffleExchange (74) + +- NativeFilter (73) + +- InputAdapter (72) + +- NativeParquetScan (71) +- == Initial Plan == - TakeOrderedAndProject (134) - +- HashAggregate (133) - +- Exchange (132) - +- HashAggregate (131) - +- Project (130) - +- SortMergeJoin Inner (129) - :- Sort (124) - : +- Exchange (123) - : +- Project (122) - : +- SortMergeJoin Inner (121) - : :- Sort (116) - : : +- Exchange (115) - : : +- Project (114) - : : +- SortMergeJoin Inner (113) - : : :- Sort (108) - : : : +- Exchange (107) - : : : +- Project (106) - : : : +- SortMergeJoin Inner (105) - : : : :- Sort (99) - : : : : +- Exchange (98) - : : : : +- Project (97) - : : : : +- SortMergeJoin Inner (96) - : : : : :- Sort (91) - : : : : : +- Exchange (90) - : : : : : +- Project (89) - : : : : : +- Filter (88) - : : : : : +- Scan parquet (87) - : : : : +- Sort (95) - : : : : +- Exchange (94) - : : : : +- Filter (93) - : : : : +- Scan parquet (92) - : : : +- Sort (104) - : : : +- Exchange (103) - : : : +- Project (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Sort (112) - : : +- Exchange (111) - : : +- Filter (110) - : : +- Scan parquet (109) - : +- Sort (120) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Sort (128) - +- Exchange (127) - +- Filter (126) - +- Scan parquet (125) - - -(87) Scan parquet + TakeOrderedAndProject (138) + +- HashAggregate (137) + +- Exchange (136) + +- HashAggregate (135) + +- Project (134) + +- SortMergeJoin Inner (133) + :- Sort (128) + : +- Exchange (127) + : +- Project (126) + : +- SortMergeJoin Inner (125) + : :- Sort (120) + : : +- Exchange (119) + : : +- Project (118) + : : +- SortMergeJoin Inner (117) + : : :- Sort (112) + : : : +- Exchange (111) + : : : +- Project (110) + : : : +- SortMergeJoin Inner (109) + : : : :- Sort (103) + : : : : +- Exchange (102) + : : : : +- Project (101) + : : : : +- SortMergeJoin Inner (100) + : : : : :- Sort (95) + : : : : : +- Exchange (94) + : : : : : +- Project (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Sort (99) + : : : : +- Exchange (98) + : : : : +- Filter (97) + : : : : +- Scan parquet (96) + : : : +- Sort (108) + : : : +- Exchange (107) + : : : +- Project (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- Sort (116) + : : +- Exchange (115) + : : +- Filter (114) + : : +- Scan parquet (113) + : +- Sort (124) + : +- Exchange (123) + : +- Filter (122) + : +- Scan parquet (121) + +- Sort (132) + +- Exchange (131) + +- Filter (130) + +- Scan parquet (129) + + +(91) Scan parquet Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location: InMemoryFileIndex [file:/] @@ -176,7 +180,7 @@ Input [1]: [d_date_sk#1] Input [1]: [d_date_sk#1] Arguments: [d_date_sk#1 ASC NULLS FIRST], false -(92) Scan parquet +(96) Scan parquet Output [5]: [ss_sold_date_sk#4, ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Batched: true Location: InMemoryFileIndex [file:/] @@ -239,7 +243,7 @@ Input [4]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Input [4]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Arguments: [ss_item_sk#5 ASC NULLS FIRST], false -(100) Scan parquet +(104) Scan parquet Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Batched: true Location: InMemoryFileIndex [file:/] @@ -306,7 +310,7 @@ Input [7]: [ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10 Input [7]: [ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Arguments: [ss_customer_sk#6 ASC NULLS FIRST], false -(109) Scan parquet +(113) Scan parquet Output [2]: [c_customer_sk#15, c_current_addr_sk#16] Batched: true Location: InMemoryFileIndex [file:/] @@ -369,7 +373,7 @@ Input [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_ma Input [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] Arguments: [c_current_addr_sk#16 ASC NULLS FIRST], false -(117) Scan parquet +(121) Scan parquet Output [2]: [ca_address_sk#17, ca_zip#18] Batched: true Location: InMemoryFileIndex [file:/] @@ -432,7 +436,7 @@ Input [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_ma Input [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] Arguments: [ss_store_sk#7 ASC NULLS FIRST], false -(125) Scan parquet +(129) Scan parquet Output [2]: [s_store_sk#19, s_zip#20] Batched: true Location: InMemoryFileIndex [file:/] @@ -466,273 +470,288 @@ Input [2]: [#19#19, #20#20] Input [2]: [#19#19, #20#20] Arguments: [s_store_sk#19 ASC NULLS FIRST], false -(79) SortMergeJoin [codegen id : X] +(79) NativeSortMergeJoin Left keys [1]: [ss_store_sk#7] Right keys [1]: [s_store_sk#19] Join type: Inner -Join condition: NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)) +Join condition: None + +(80) NativeFilter +Input [9]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, #19#19, #20#20] +Condition : NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)) -(80) Project [codegen id : X] +(81) NativeProject Output [5]: [ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Input [9]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, #19#19, #20#20] -(81) HashAggregate [codegen id : X] +(82) NativeProject +Output [5]: [i_brand#11 AS i_brand#11, i_brand_id#10 AS i_brand_id#10, i_manufact_id#12 AS i_manufact_id#12, i_manufact#13 AS i_manufact#13, UnscaledValue(ss_ext_sales_price#8) AS _c4#21] Input [5]: [ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] + +(83) NativeHashAggregate +Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, _c4#21] Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] -Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#8))] -Aggregate Attributes [1]: [sum#21] -Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +Functions [1]: [partial_sum(_c4#21)] +Aggregate Attributes [1]: [sum#22] +Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, #23] -(82) Exchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +(84) NativeShuffleExchange +Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, #23] Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 100), ENSURE_REQUIREMENTS, [plan_id=11] -(83) ShuffleQueryStage -Output [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +(85) ShuffleQueryStage +Output [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, #23] Arguments: X -(84) AQEShuffleRead -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +(86) AQEShuffleRead +Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, #23] Arguments: coalesced -(85) HashAggregate [codegen id : X] -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +(87) InputAdapter +Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, #23] + +(88) NativeHashAggregate +Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, #23] Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#8))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#8))#23] -Results [5]: [i_brand_id#10 AS brand_id#24, i_brand#11 AS brand#25, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#8))#23,17,2) AS ext_price#26] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#8))#24] +Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum(UnscaledValue(ss_ext_sales_price#8))#24] + +(89) NativeProject +Output [5]: [i_brand_id#10 AS brand_id#25, i_brand#11 AS brand#26, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#8))#24,17,2) AS ext_price#27] +Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum(UnscaledValue(ss_ext_sales_price#8))#24] -(86) TakeOrderedAndProject -Input [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] -Arguments: X, [ext_price#26 DESC NULLS LAST, brand#25 ASC NULLS FIRST, brand_id#24 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] +(90) NativeTakeOrdered +Input [5]: [brand_id#25, brand#26, i_manufact_id#12, i_manufact#13, ext_price#27] +Arguments: X, X, [ext_price#27 DESC NULLS LAST, brand#26 ASC NULLS FIRST, brand_id#25 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST] -(87) Scan parquet +(91) Scan parquet Output [3]: [d_date_sk#1, d_year#2, d_moy#3] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(88) Filter +(92) Filter Input [3]: [d_date_sk#1, d_year#2, d_moy#3] Condition : ((((isnotnull(d_moy#3) AND isnotnull(d_year#2)) AND (d_moy#3 = 11)) AND (d_year#2 = 1998)) AND isnotnull(d_date_sk#1)) -(89) Project +(93) Project Output [1]: [d_date_sk#1] Input [3]: [d_date_sk#1, d_year#2, d_moy#3] -(90) Exchange +(94) Exchange Input [1]: [d_date_sk#1] Arguments: hashpartitioning(d_date_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=12] -(91) Sort +(95) Sort Input [1]: [d_date_sk#1] Arguments: [d_date_sk#1 ASC NULLS FIRST], false, 0 -(92) Scan parquet +(96) Scan parquet Output [5]: [ss_sold_date_sk#4, ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_item_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(93) Filter +(97) Filter Input [5]: [ss_sold_date_sk#4, ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Condition : (((isnotnull(ss_sold_date_sk#4) AND isnotnull(ss_item_sk#5)) AND isnotnull(ss_customer_sk#6)) AND isnotnull(ss_store_sk#7)) -(94) Exchange +(98) Exchange Input [5]: [ss_sold_date_sk#4, ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Arguments: hashpartitioning(ss_sold_date_sk#4, 100), ENSURE_REQUIREMENTS, [plan_id=13] -(95) Sort +(99) Sort Input [5]: [ss_sold_date_sk#4, ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Arguments: [ss_sold_date_sk#4 ASC NULLS FIRST], false, 0 -(96) SortMergeJoin +(100) SortMergeJoin Left keys [1]: [d_date_sk#1] Right keys [1]: [ss_sold_date_sk#4] Join type: Inner Join condition: None -(97) Project +(101) Project Output [4]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Input [6]: [d_date_sk#1, ss_sold_date_sk#4, ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] -(98) Exchange +(102) Exchange Input [4]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Arguments: hashpartitioning(ss_item_sk#5, 100), ENSURE_REQUIREMENTS, [plan_id=14] -(99) Sort +(103) Sort Input [4]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8] Arguments: [ss_item_sk#5 ASC NULLS FIRST], false, 0 -(100) Scan parquet +(104) Scan parquet Output [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_manager_id), EqualTo(i_manager_id,8), IsNotNull(i_item_sk)] ReadSchema: struct -(101) Filter +(105) Filter Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] Condition : ((isnotnull(i_manager_id#14) AND (i_manager_id#14 = 8)) AND isnotnull(i_item_sk#9)) -(102) Project +(106) Project Output [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Input [6]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, i_manager_id#14] -(103) Exchange +(107) Exchange Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Arguments: hashpartitioning(i_item_sk#9, 100), ENSURE_REQUIREMENTS, [plan_id=15] -(104) Sort +(108) Sort Input [5]: [i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Arguments: [i_item_sk#9 ASC NULLS FIRST], false, 0 -(105) SortMergeJoin +(109) SortMergeJoin Left keys [1]: [ss_item_sk#5] Right keys [1]: [i_item_sk#9] Join type: Inner Join condition: None -(106) Project +(110) Project Output [7]: [ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Input [9]: [ss_item_sk#5, ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, i_item_sk#9, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] -(107) Exchange +(111) Exchange Input [7]: [ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Arguments: hashpartitioning(ss_customer_sk#6, 100), ENSURE_REQUIREMENTS, [plan_id=16] -(108) Sort +(112) Sort Input [7]: [ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Arguments: [ss_customer_sk#6 ASC NULLS FIRST], false, 0 -(109) Scan parquet +(113) Scan parquet Output [2]: [c_customer_sk#15, c_current_addr_sk#16] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(110) Filter +(114) Filter Input [2]: [c_customer_sk#15, c_current_addr_sk#16] Condition : (isnotnull(c_customer_sk#15) AND isnotnull(c_current_addr_sk#16)) -(111) Exchange +(115) Exchange Input [2]: [c_customer_sk#15, c_current_addr_sk#16] Arguments: hashpartitioning(c_customer_sk#15, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(112) Sort +(116) Sort Input [2]: [c_customer_sk#15, c_current_addr_sk#16] Arguments: [c_customer_sk#15 ASC NULLS FIRST], false, 0 -(113) SortMergeJoin +(117) SortMergeJoin Left keys [1]: [ss_customer_sk#6] Right keys [1]: [c_customer_sk#15] Join type: Inner Join condition: None -(114) Project +(118) Project Output [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] Input [9]: [ss_customer_sk#6, ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_customer_sk#15, c_current_addr_sk#16] -(115) Exchange +(119) Exchange Input [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] Arguments: hashpartitioning(c_current_addr_sk#16, 100), ENSURE_REQUIREMENTS, [plan_id=18] -(116) Sort +(120) Sort Input [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16] Arguments: [c_current_addr_sk#16 ASC NULLS FIRST], false, 0 -(117) Scan parquet +(121) Scan parquet Output [2]: [ca_address_sk#17, ca_zip#18] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_zip)] ReadSchema: struct -(118) Filter +(122) Filter Input [2]: [ca_address_sk#17, ca_zip#18] Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_zip#18)) -(119) Exchange +(123) Exchange Input [2]: [ca_address_sk#17, ca_zip#18] Arguments: hashpartitioning(ca_address_sk#17, 100), ENSURE_REQUIREMENTS, [plan_id=19] -(120) Sort +(124) Sort Input [2]: [ca_address_sk#17, ca_zip#18] Arguments: [ca_address_sk#17 ASC NULLS FIRST], false, 0 -(121) SortMergeJoin +(125) SortMergeJoin Left keys [1]: [c_current_addr_sk#16] Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(122) Project +(126) Project Output [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] Input [9]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, c_current_addr_sk#16, ca_address_sk#17, ca_zip#18] -(123) Exchange +(127) Exchange Input [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] Arguments: hashpartitioning(ss_store_sk#7, 100), ENSURE_REQUIREMENTS, [plan_id=20] -(124) Sort +(128) Sort Input [7]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18] Arguments: [ss_store_sk#7 ASC NULLS FIRST], false, 0 -(125) Scan parquet +(129) Scan parquet Output [2]: [s_store_sk#19, s_zip#20] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(s_zip), IsNotNull(s_store_sk)] ReadSchema: struct -(126) Filter +(130) Filter Input [2]: [s_store_sk#19, s_zip#20] Condition : (isnotnull(s_zip#20) AND isnotnull(s_store_sk#19)) -(127) Exchange +(131) Exchange Input [2]: [s_store_sk#19, s_zip#20] Arguments: hashpartitioning(s_store_sk#19, 100), ENSURE_REQUIREMENTS, [plan_id=21] -(128) Sort +(132) Sort Input [2]: [s_store_sk#19, s_zip#20] Arguments: [s_store_sk#19 ASC NULLS FIRST], false, 0 -(129) SortMergeJoin +(133) SortMergeJoin Left keys [1]: [ss_store_sk#7] Right keys [1]: [s_store_sk#19] Join type: Inner Join condition: NOT (substr(ca_zip#18, 1, 5) = substr(s_zip#20, 1, 5)) -(130) Project +(134) Project Output [5]: [ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Input [9]: [ss_store_sk#7, ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13, ca_zip#18, s_store_sk#19, s_zip#20] -(131) HashAggregate +(135) HashAggregate Input [5]: [ss_ext_sales_price#8, i_brand_id#10, i_brand#11, i_manufact_id#12, i_manufact#13] Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#8))] -Aggregate Attributes [1]: [sum#21] -Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +Aggregate Attributes [1]: [sum#22] +Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#28] -(132) Exchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +(136) Exchange +Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#28] Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(133) HashAggregate -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +(137) HashAggregate +Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#28] Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#8))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#8))#23] -Results [5]: [i_brand_id#10 AS brand_id#24, i_brand#11 AS brand#25, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#8))#23,17,2) AS ext_price#26] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#8))#24] +Results [5]: [i_brand_id#10 AS brand_id#25, i_brand#11 AS brand#26, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#8))#24,17,2) AS ext_price#27] -(134) TakeOrderedAndProject -Input [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] -Arguments: X, [ext_price#26 DESC NULLS LAST, brand#25 ASC NULLS FIRST, brand_id#24 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] +(138) TakeOrderedAndProject +Input [5]: [brand_id#25, brand#26, i_manufact_id#12, i_manufact#13, ext_price#27] +Arguments: X, [ext_price#27 DESC NULLS LAST, brand#26 ASC NULLS FIRST, brand_id#25 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#25, brand#26, i_manufact_id#12, i_manufact#13, ext_price#27] -(135) AdaptiveSparkPlan -Output [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] +(139) AdaptiveSparkPlan +Output [5]: [brand_id#25, brand#26, i_manufact_id#12, i_manufact#13, ext_price#27] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q30.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q30.txt index 53ab1e281..86e29ffa3 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q30.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q30.txt @@ -16,9 +16,9 @@ AdaptiveSparkPlan (202) : : +- AQEShuffleRead (100) : : +- ShuffleQueryStage (99), Statistics(X) : : +- NativeShuffleExchange (98) - : : +- ConvertToNative (97) - : : +- * Project (96) - : : +- * SortMergeJoin Inner (95) + : : +- NativeProject (97) + : : +- NativeFilter (96) + : : +- NativeSortMergeJoin Inner (95) : : :- NativeSort (48) : : : +- InputAdapter (47) : : : +- AQEShuffleRead (46) @@ -605,18 +605,19 @@ Condition : isnotnull((avg(ctr_total_return) * 1.2)#31) Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#25] Arguments: [ctr_state#25 ASC NULLS FIRST], false -(95) SortMergeJoin [codegen id : X] +(95) NativeSortMergeJoin Left keys [1]: [ctr_state#14] Right keys [1]: [ctr_state#25] Join type: Inner -Join condition: (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31) +Join condition: None -(96) Project [codegen id : X] -Output [2]: [ctr_customer_sk#13, ctr_total_return#15] +(96) NativeFilter Input [5]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#31, ctr_state#25] +Condition : (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31) -(97) ConvertToNative -Input [2]: [ctr_customer_sk#13, ctr_total_return#15] +(97) NativeProject +Output [2]: [ctr_customer_sk#13, ctr_total_return#15] +Input [5]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#31, ctr_state#25] (98) NativeShuffleExchange Input [2]: [ctr_customer_sk#13, ctr_total_return#15] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q31.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q31.txt index 6a00e14da..333493342 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q31.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q31.txt @@ -1,424 +1,425 @@ == Physical Plan == -AdaptiveSparkPlan (415) +AdaptiveSparkPlan (416) +- == Final Plan == - NativeSort (266) - +- InputAdapter (265) - +- AQEShuffleRead (264) - +- ShuffleQueryStage (263), Statistics(X) - +- NativeShuffleExchange (262) - +- ConvertToNative (261) - +- * Project (260) - +- * SortMergeJoin Inner (259) - :- * Project (218) - : +- * SortMergeJoin Inner (217) - : :- NativeSortMergeJoin Inner (176) - : : :- NativeProject (133) - : : : +- NativeSortMergeJoin Inner (132) - : : : :- NativeSortMergeJoin Inner (89) - : : : : :- NativeSort (46) - : : : : : +- InputAdapter (45) - : : : : : +- AQEShuffleRead (44) - : : : : : +- ShuffleQueryStage (43), Statistics(X) - : : : : : +- NativeShuffleExchange (42) - : : : : : +- NativeProject (41) - : : : : : +- NativeHashAggregate (40) - : : : : : +- InputAdapter (39) - : : : : : +- AQEShuffleRead (38) - : : : : : +- ShuffleQueryStage (37), Statistics(X) - : : : : : +- NativeShuffleExchange (36) - : : : : : +- NativeHashAggregate (35) - : : : : : +- NativeProject (34) - : : : : : +- NativeProject (33) - : : : : : +- NativeSortMergeJoin Inner (32) - : : : : : :- NativeSort (23) - : : : : : : +- InputAdapter (22) - : : : : : : +- AQEShuffleRead (21) - : : : : : : +- ShuffleQueryStage (20), Statistics(X) - : : : : : : +- NativeShuffleExchange (19) - : : : : : : +- NativeProject (18) - : : : : : : +- NativeSortMergeJoin Inner (17) - : : : : : : :- NativeSort (8) - : : : : : : : +- InputAdapter (7) - : : : : : : : +- AQEShuffleRead (6) - : : : : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : : : : +- NativeShuffleExchange (4) - : : : : : : : +- NativeFilter (3) - : : : : : : : +- InputAdapter (2) - : : : : : : : +- NativeParquetScan (1) - : : : : : : +- NativeSort (16) - : : : : : : +- InputAdapter (15) - : : : : : : +- AQEShuffleRead (14) - : : : : : : +- ShuffleQueryStage (13), Statistics(X) - : : : : : : +- NativeShuffleExchange (12) - : : : : : : +- NativeFilter (11) - : : : : : : +- InputAdapter (10) - : : : : : : +- NativeParquetScan (9) - : : : : : +- NativeSort (31) - : : : : : +- InputAdapter (30) - : : : : : +- AQEShuffleRead (29) - : : : : : +- ShuffleQueryStage (28), Statistics(X) - : : : : : +- NativeShuffleExchange (27) - : : : : : +- NativeFilter (26) - : : : : : +- InputAdapter (25) - : : : : : +- NativeParquetScan (24) - : : : : +- NativeSort (88) - : : : : +- InputAdapter (87) - : : : : +- AQEShuffleRead (86) - : : : : +- ShuffleQueryStage (85), Statistics(X) - : : : : +- NativeShuffleExchange (84) - : : : : +- NativeProject (83) - : : : : +- NativeHashAggregate (82) - : : : : +- InputAdapter (81) - : : : : +- AQEShuffleRead (80) - : : : : +- ShuffleQueryStage (79), Statistics(X) - : : : : +- NativeShuffleExchange (78) - : : : : +- NativeHashAggregate (77) - : : : : +- NativeProject (76) - : : : : +- NativeProject (75) - : : : : +- NativeSortMergeJoin Inner (74) - : : : : :- NativeSort (67) - : : : : : +- InputAdapter (66) - : : : : : +- AQEShuffleRead (65) - : : : : : +- ShuffleQueryStage (64), Statistics(X) - : : : : : +- NativeShuffleExchange (63) - : : : : : +- NativeProject (62) - : : : : : +- NativeSortMergeJoin Inner (61) - : : : : : :- NativeSort (52) - : : : : : : +- InputAdapter (51) - : : : : : : +- InputAdapter (50) - : : : : : : +- AQEShuffleRead (49) - : : : : : : +- ShuffleQueryStage (48), Statistics(X) - : : : : : : +- ReusedExchange (47) - : : : : : +- NativeSort (60) - : : : : : +- InputAdapter (59) - : : : : : +- AQEShuffleRead (58) - : : : : : +- ShuffleQueryStage (57), Statistics(X) - : : : : : +- NativeShuffleExchange (56) - : : : : : +- NativeFilter (55) - : : : : : +- InputAdapter (54) - : : : : : +- NativeParquetScan (53) - : : : : +- NativeSort (73) - : : : : +- InputAdapter (72) - : : : : +- InputAdapter (71) - : : : : +- AQEShuffleRead (70) - : : : : +- ShuffleQueryStage (69), Statistics(X) - : : : : +- ReusedExchange (68) - : : : +- NativeSort (131) - : : : +- InputAdapter (130) - : : : +- AQEShuffleRead (129) - : : : +- ShuffleQueryStage (128), Statistics(X) - : : : +- NativeShuffleExchange (127) - : : : +- NativeProject (126) - : : : +- NativeHashAggregate (125) - : : : +- InputAdapter (124) - : : : +- AQEShuffleRead (123) - : : : +- ShuffleQueryStage (122), Statistics(X) - : : : +- NativeShuffleExchange (121) - : : : +- NativeHashAggregate (120) - : : : +- NativeProject (119) - : : : +- NativeProject (118) - : : : +- NativeSortMergeJoin Inner (117) - : : : :- NativeSort (110) - : : : : +- InputAdapter (109) - : : : : +- AQEShuffleRead (108) - : : : : +- ShuffleQueryStage (107), Statistics(X) - : : : : +- NativeShuffleExchange (106) - : : : : +- NativeProject (105) - : : : : +- NativeSortMergeJoin Inner (104) - : : : : :- NativeSort (95) - : : : : : +- InputAdapter (94) - : : : : : +- InputAdapter (93) - : : : : : +- AQEShuffleRead (92) - : : : : : +- ShuffleQueryStage (91), Statistics(X) - : : : : : +- ReusedExchange (90) - : : : : +- NativeSort (103) - : : : : +- InputAdapter (102) - : : : : +- AQEShuffleRead (101) - : : : : +- ShuffleQueryStage (100), Statistics(X) - : : : : +- NativeShuffleExchange (99) - : : : : +- NativeFilter (98) - : : : : +- InputAdapter (97) - : : : : +- NativeParquetScan (96) - : : : +- NativeSort (116) - : : : +- InputAdapter (115) - : : : +- InputAdapter (114) - : : : +- AQEShuffleRead (113) - : : : +- ShuffleQueryStage (112), Statistics(X) - : : : +- ReusedExchange (111) - : : +- NativeSort (175) - : : +- InputAdapter (174) - : : +- AQEShuffleRead (173) - : : +- ShuffleQueryStage (172), Statistics(X) - : : +- NativeShuffleExchange (171) - : : +- NativeProject (170) - : : +- NativeHashAggregate (169) - : : +- InputAdapter (168) - : : +- AQEShuffleRead (167) - : : +- ShuffleQueryStage (166), Statistics(X) - : : +- NativeShuffleExchange (165) - : : +- NativeHashAggregate (164) - : : +- NativeProject (163) - : : +- NativeProject (162) - : : +- NativeSortMergeJoin Inner (161) - : : :- NativeSort (154) - : : : +- InputAdapter (153) - : : : +- AQEShuffleRead (152) - : : : +- ShuffleQueryStage (151), Statistics(X) - : : : +- NativeShuffleExchange (150) - : : : +- NativeProject (149) - : : : +- NativeSortMergeJoin Inner (148) - : : : :- NativeSort (141) - : : : : +- InputAdapter (140) - : : : : +- AQEShuffleRead (139) - : : : : +- ShuffleQueryStage (138), Statistics(X) - : : : : +- NativeShuffleExchange (137) - : : : : +- NativeFilter (136) - : : : : +- InputAdapter (135) - : : : : +- NativeParquetScan (134) - : : : +- NativeSort (147) - : : : +- InputAdapter (146) - : : : +- InputAdapter (145) - : : : +- AQEShuffleRead (144) - : : : +- ShuffleQueryStage (143), Statistics(X) - : : : +- ReusedExchange (142) - : : +- NativeSort (160) - : : +- InputAdapter (159) - : : +- InputAdapter (158) - : : +- AQEShuffleRead (157) - : : +- ShuffleQueryStage (156), Statistics(X) - : : +- ReusedExchange (155) - : +- NativeSort (216) - : +- InputAdapter (215) - : +- AQEShuffleRead (214) - : +- ShuffleQueryStage (213), Statistics(X) - : +- NativeShuffleExchange (212) - : +- NativeProject (211) - : +- NativeHashAggregate (210) - : +- InputAdapter (209) - : +- AQEShuffleRead (208) - : +- ShuffleQueryStage (207), Statistics(X) - : +- NativeShuffleExchange (206) - : +- NativeHashAggregate (205) - : +- NativeProject (204) - : +- NativeProject (203) - : +- NativeSortMergeJoin Inner (202) - : :- NativeSort (195) - : : +- InputAdapter (194) - : : +- AQEShuffleRead (193) - : : +- ShuffleQueryStage (192), Statistics(X) - : : +- NativeShuffleExchange (191) - : : +- NativeProject (190) - : : +- NativeSortMergeJoin Inner (189) - : : :- NativeSort (182) - : : : +- InputAdapter (181) - : : : +- InputAdapter (180) - : : : +- AQEShuffleRead (179) - : : : +- ShuffleQueryStage (178), Statistics(X) - : : : +- ReusedExchange (177) - : : +- NativeSort (188) - : : +- InputAdapter (187) - : : +- InputAdapter (186) - : : +- AQEShuffleRead (185) - : : +- ShuffleQueryStage (184), Statistics(X) - : : +- ReusedExchange (183) - : +- NativeSort (201) - : +- InputAdapter (200) - : +- InputAdapter (199) - : +- AQEShuffleRead (198) - : +- ShuffleQueryStage (197), Statistics(X) - : +- ReusedExchange (196) - +- NativeSort (258) - +- InputAdapter (257) - +- AQEShuffleRead (256) - +- ShuffleQueryStage (255), Statistics(X) - +- NativeShuffleExchange (254) - +- NativeProject (253) - +- NativeHashAggregate (252) - +- InputAdapter (251) - +- AQEShuffleRead (250) - +- ShuffleQueryStage (249), Statistics(X) - +- NativeShuffleExchange (248) - +- NativeHashAggregate (247) - +- NativeProject (246) - +- NativeProject (245) - +- NativeSortMergeJoin Inner (244) - :- NativeSort (237) - : +- InputAdapter (236) - : +- AQEShuffleRead (235) - : +- ShuffleQueryStage (234), Statistics(X) - : +- NativeShuffleExchange (233) - : +- NativeProject (232) - : +- NativeSortMergeJoin Inner (231) - : :- NativeSort (224) - : : +- InputAdapter (223) - : : +- InputAdapter (222) - : : +- AQEShuffleRead (221) - : : +- ShuffleQueryStage (220), Statistics(X) - : : +- ReusedExchange (219) - : +- NativeSort (230) - : +- InputAdapter (229) - : +- InputAdapter (228) - : +- AQEShuffleRead (227) - : +- ShuffleQueryStage (226), Statistics(X) - : +- ReusedExchange (225) - +- NativeSort (243) - +- InputAdapter (242) - +- InputAdapter (241) - +- AQEShuffleRead (240) - +- ShuffleQueryStage (239), Statistics(X) - +- ReusedExchange (238) + NativeSort (267) + +- InputAdapter (266) + +- AQEShuffleRead (265) + +- ShuffleQueryStage (264), Statistics(X) + +- NativeShuffleExchange (263) + +- NativeProject (262) + +- NativeFilter (261) + +- NativeSortMergeJoin Inner (260) + :- NativeProject (219) + : +- NativeFilter (218) + : +- NativeSortMergeJoin Inner (217) + : :- NativeSortMergeJoin Inner (176) + : : :- NativeProject (133) + : : : +- NativeSortMergeJoin Inner (132) + : : : :- NativeSortMergeJoin Inner (89) + : : : : :- NativeSort (46) + : : : : : +- InputAdapter (45) + : : : : : +- AQEShuffleRead (44) + : : : : : +- ShuffleQueryStage (43), Statistics(X) + : : : : : +- NativeShuffleExchange (42) + : : : : : +- NativeProject (41) + : : : : : +- NativeHashAggregate (40) + : : : : : +- InputAdapter (39) + : : : : : +- AQEShuffleRead (38) + : : : : : +- ShuffleQueryStage (37), Statistics(X) + : : : : : +- NativeShuffleExchange (36) + : : : : : +- NativeHashAggregate (35) + : : : : : +- NativeProject (34) + : : : : : +- NativeProject (33) + : : : : : +- NativeSortMergeJoin Inner (32) + : : : : : :- NativeSort (23) + : : : : : : +- InputAdapter (22) + : : : : : : +- AQEShuffleRead (21) + : : : : : : +- ShuffleQueryStage (20), Statistics(X) + : : : : : : +- NativeShuffleExchange (19) + : : : : : : +- NativeProject (18) + : : : : : : +- NativeSortMergeJoin Inner (17) + : : : : : : :- NativeSort (8) + : : : : : : : +- InputAdapter (7) + : : : : : : : +- AQEShuffleRead (6) + : : : : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : : : : +- NativeShuffleExchange (4) + : : : : : : : +- NativeFilter (3) + : : : : : : : +- InputAdapter (2) + : : : : : : : +- NativeParquetScan (1) + : : : : : : +- NativeSort (16) + : : : : : : +- InputAdapter (15) + : : : : : : +- AQEShuffleRead (14) + : : : : : : +- ShuffleQueryStage (13), Statistics(X) + : : : : : : +- NativeShuffleExchange (12) + : : : : : : +- NativeFilter (11) + : : : : : : +- InputAdapter (10) + : : : : : : +- NativeParquetScan (9) + : : : : : +- NativeSort (31) + : : : : : +- InputAdapter (30) + : : : : : +- AQEShuffleRead (29) + : : : : : +- ShuffleQueryStage (28), Statistics(X) + : : : : : +- NativeShuffleExchange (27) + : : : : : +- NativeFilter (26) + : : : : : +- InputAdapter (25) + : : : : : +- NativeParquetScan (24) + : : : : +- NativeSort (88) + : : : : +- InputAdapter (87) + : : : : +- AQEShuffleRead (86) + : : : : +- ShuffleQueryStage (85), Statistics(X) + : : : : +- NativeShuffleExchange (84) + : : : : +- NativeProject (83) + : : : : +- NativeHashAggregate (82) + : : : : +- InputAdapter (81) + : : : : +- AQEShuffleRead (80) + : : : : +- ShuffleQueryStage (79), Statistics(X) + : : : : +- NativeShuffleExchange (78) + : : : : +- NativeHashAggregate (77) + : : : : +- NativeProject (76) + : : : : +- NativeProject (75) + : : : : +- NativeSortMergeJoin Inner (74) + : : : : :- NativeSort (67) + : : : : : +- InputAdapter (66) + : : : : : +- AQEShuffleRead (65) + : : : : : +- ShuffleQueryStage (64), Statistics(X) + : : : : : +- NativeShuffleExchange (63) + : : : : : +- NativeProject (62) + : : : : : +- NativeSortMergeJoin Inner (61) + : : : : : :- NativeSort (52) + : : : : : : +- InputAdapter (51) + : : : : : : +- InputAdapter (50) + : : : : : : +- AQEShuffleRead (49) + : : : : : : +- ShuffleQueryStage (48), Statistics(X) + : : : : : : +- ReusedExchange (47) + : : : : : +- NativeSort (60) + : : : : : +- InputAdapter (59) + : : : : : +- AQEShuffleRead (58) + : : : : : +- ShuffleQueryStage (57), Statistics(X) + : : : : : +- NativeShuffleExchange (56) + : : : : : +- NativeFilter (55) + : : : : : +- InputAdapter (54) + : : : : : +- NativeParquetScan (53) + : : : : +- NativeSort (73) + : : : : +- InputAdapter (72) + : : : : +- InputAdapter (71) + : : : : +- AQEShuffleRead (70) + : : : : +- ShuffleQueryStage (69), Statistics(X) + : : : : +- ReusedExchange (68) + : : : +- NativeSort (131) + : : : +- InputAdapter (130) + : : : +- AQEShuffleRead (129) + : : : +- ShuffleQueryStage (128), Statistics(X) + : : : +- NativeShuffleExchange (127) + : : : +- NativeProject (126) + : : : +- NativeHashAggregate (125) + : : : +- InputAdapter (124) + : : : +- AQEShuffleRead (123) + : : : +- ShuffleQueryStage (122), Statistics(X) + : : : +- NativeShuffleExchange (121) + : : : +- NativeHashAggregate (120) + : : : +- NativeProject (119) + : : : +- NativeProject (118) + : : : +- NativeSortMergeJoin Inner (117) + : : : :- NativeSort (110) + : : : : +- InputAdapter (109) + : : : : +- AQEShuffleRead (108) + : : : : +- ShuffleQueryStage (107), Statistics(X) + : : : : +- NativeShuffleExchange (106) + : : : : +- NativeProject (105) + : : : : +- NativeSortMergeJoin Inner (104) + : : : : :- NativeSort (95) + : : : : : +- InputAdapter (94) + : : : : : +- InputAdapter (93) + : : : : : +- AQEShuffleRead (92) + : : : : : +- ShuffleQueryStage (91), Statistics(X) + : : : : : +- ReusedExchange (90) + : : : : +- NativeSort (103) + : : : : +- InputAdapter (102) + : : : : +- AQEShuffleRead (101) + : : : : +- ShuffleQueryStage (100), Statistics(X) + : : : : +- NativeShuffleExchange (99) + : : : : +- NativeFilter (98) + : : : : +- InputAdapter (97) + : : : : +- NativeParquetScan (96) + : : : +- NativeSort (116) + : : : +- InputAdapter (115) + : : : +- InputAdapter (114) + : : : +- AQEShuffleRead (113) + : : : +- ShuffleQueryStage (112), Statistics(X) + : : : +- ReusedExchange (111) + : : +- NativeSort (175) + : : +- InputAdapter (174) + : : +- AQEShuffleRead (173) + : : +- ShuffleQueryStage (172), Statistics(X) + : : +- NativeShuffleExchange (171) + : : +- NativeProject (170) + : : +- NativeHashAggregate (169) + : : +- InputAdapter (168) + : : +- AQEShuffleRead (167) + : : +- ShuffleQueryStage (166), Statistics(X) + : : +- NativeShuffleExchange (165) + : : +- NativeHashAggregate (164) + : : +- NativeProject (163) + : : +- NativeProject (162) + : : +- NativeSortMergeJoin Inner (161) + : : :- NativeSort (154) + : : : +- InputAdapter (153) + : : : +- AQEShuffleRead (152) + : : : +- ShuffleQueryStage (151), Statistics(X) + : : : +- NativeShuffleExchange (150) + : : : +- NativeProject (149) + : : : +- NativeSortMergeJoin Inner (148) + : : : :- NativeSort (141) + : : : : +- InputAdapter (140) + : : : : +- AQEShuffleRead (139) + : : : : +- ShuffleQueryStage (138), Statistics(X) + : : : : +- NativeShuffleExchange (137) + : : : : +- NativeFilter (136) + : : : : +- InputAdapter (135) + : : : : +- NativeParquetScan (134) + : : : +- NativeSort (147) + : : : +- InputAdapter (146) + : : : +- InputAdapter (145) + : : : +- AQEShuffleRead (144) + : : : +- ShuffleQueryStage (143), Statistics(X) + : : : +- ReusedExchange (142) + : : +- NativeSort (160) + : : +- InputAdapter (159) + : : +- InputAdapter (158) + : : +- AQEShuffleRead (157) + : : +- ShuffleQueryStage (156), Statistics(X) + : : +- ReusedExchange (155) + : +- NativeSort (216) + : +- InputAdapter (215) + : +- AQEShuffleRead (214) + : +- ShuffleQueryStage (213), Statistics(X) + : +- NativeShuffleExchange (212) + : +- NativeProject (211) + : +- NativeHashAggregate (210) + : +- InputAdapter (209) + : +- AQEShuffleRead (208) + : +- ShuffleQueryStage (207), Statistics(X) + : +- NativeShuffleExchange (206) + : +- NativeHashAggregate (205) + : +- NativeProject (204) + : +- NativeProject (203) + : +- NativeSortMergeJoin Inner (202) + : :- NativeSort (195) + : : +- InputAdapter (194) + : : +- AQEShuffleRead (193) + : : +- ShuffleQueryStage (192), Statistics(X) + : : +- NativeShuffleExchange (191) + : : +- NativeProject (190) + : : +- NativeSortMergeJoin Inner (189) + : : :- NativeSort (182) + : : : +- InputAdapter (181) + : : : +- InputAdapter (180) + : : : +- AQEShuffleRead (179) + : : : +- ShuffleQueryStage (178), Statistics(X) + : : : +- ReusedExchange (177) + : : +- NativeSort (188) + : : +- InputAdapter (187) + : : +- InputAdapter (186) + : : +- AQEShuffleRead (185) + : : +- ShuffleQueryStage (184), Statistics(X) + : : +- ReusedExchange (183) + : +- NativeSort (201) + : +- InputAdapter (200) + : +- InputAdapter (199) + : +- AQEShuffleRead (198) + : +- ShuffleQueryStage (197), Statistics(X) + : +- ReusedExchange (196) + +- NativeSort (259) + +- InputAdapter (258) + +- AQEShuffleRead (257) + +- ShuffleQueryStage (256), Statistics(X) + +- NativeShuffleExchange (255) + +- NativeProject (254) + +- NativeHashAggregate (253) + +- InputAdapter (252) + +- AQEShuffleRead (251) + +- ShuffleQueryStage (250), Statistics(X) + +- NativeShuffleExchange (249) + +- NativeHashAggregate (248) + +- NativeProject (247) + +- NativeProject (246) + +- NativeSortMergeJoin Inner (245) + :- NativeSort (238) + : +- InputAdapter (237) + : +- AQEShuffleRead (236) + : +- ShuffleQueryStage (235), Statistics(X) + : +- NativeShuffleExchange (234) + : +- NativeProject (233) + : +- NativeSortMergeJoin Inner (232) + : :- NativeSort (225) + : : +- InputAdapter (224) + : : +- InputAdapter (223) + : : +- AQEShuffleRead (222) + : : +- ShuffleQueryStage (221), Statistics(X) + : : +- ReusedExchange (220) + : +- NativeSort (231) + : +- InputAdapter (230) + : +- InputAdapter (229) + : +- AQEShuffleRead (228) + : +- ShuffleQueryStage (227), Statistics(X) + : +- ReusedExchange (226) + +- NativeSort (244) + +- InputAdapter (243) + +- InputAdapter (242) + +- AQEShuffleRead (241) + +- ShuffleQueryStage (240), Statistics(X) + +- ReusedExchange (239) +- == Initial Plan == - Sort (414) - +- Exchange (413) - +- Project (412) - +- SortMergeJoin Inner (411) - :- Project (387) - : +- SortMergeJoin Inner (386) - : :- SortMergeJoin Inner (362) - : : :- Project (338) - : : : +- SortMergeJoin Inner (337) - : : : :- SortMergeJoin Inner (313) - : : : : :- Sort (289) - : : : : : +- Exchange (288) - : : : : : +- HashAggregate (287) - : : : : : +- Exchange (286) - : : : : : +- HashAggregate (285) - : : : : : +- Project (284) - : : : : : +- SortMergeJoin Inner (283) - : : : : : :- Sort (278) - : : : : : : +- Exchange (277) - : : : : : : +- Project (276) - : : : : : : +- SortMergeJoin Inner (275) - : : : : : : :- Sort (270) - : : : : : : : +- Exchange (269) - : : : : : : : +- Filter (268) - : : : : : : : +- Scan parquet (267) - : : : : : : +- Sort (274) - : : : : : : +- Exchange (273) - : : : : : : +- Filter (272) - : : : : : : +- Scan parquet (271) - : : : : : +- Sort (282) - : : : : : +- Exchange (281) - : : : : : +- Filter (280) - : : : : : +- Scan parquet (279) - : : : : +- Sort (312) - : : : : +- Exchange (311) - : : : : +- HashAggregate (310) - : : : : +- Exchange (309) - : : : : +- HashAggregate (308) - : : : : +- Project (307) - : : : : +- SortMergeJoin Inner (306) - : : : : :- Sort (301) - : : : : : +- Exchange (300) - : : : : : +- Project (299) - : : : : : +- SortMergeJoin Inner (298) - : : : : : :- Sort (293) - : : : : : : +- Exchange (292) - : : : : : : +- Filter (291) - : : : : : : +- Scan parquet (290) - : : : : : +- Sort (297) - : : : : : +- Exchange (296) - : : : : : +- Filter (295) - : : : : : +- Scan parquet (294) - : : : : +- Sort (305) - : : : : +- Exchange (304) - : : : : +- Filter (303) - : : : : +- Scan parquet (302) - : : : +- Sort (336) - : : : +- Exchange (335) - : : : +- HashAggregate (334) - : : : +- Exchange (333) - : : : +- HashAggregate (332) - : : : +- Project (331) - : : : +- SortMergeJoin Inner (330) - : : : :- Sort (325) - : : : : +- Exchange (324) - : : : : +- Project (323) - : : : : +- SortMergeJoin Inner (322) - : : : : :- Sort (317) - : : : : : +- Exchange (316) - : : : : : +- Filter (315) - : : : : : +- Scan parquet (314) - : : : : +- Sort (321) - : : : : +- Exchange (320) - : : : : +- Filter (319) - : : : : +- Scan parquet (318) - : : : +- Sort (329) - : : : +- Exchange (328) - : : : +- Filter (327) - : : : +- Scan parquet (326) - : : +- Sort (361) - : : +- Exchange (360) - : : +- HashAggregate (359) - : : +- Exchange (358) - : : +- HashAggregate (357) - : : +- Project (356) - : : +- SortMergeJoin Inner (355) - : : :- Sort (350) - : : : +- Exchange (349) - : : : +- Project (348) - : : : +- SortMergeJoin Inner (347) - : : : :- Sort (342) - : : : : +- Exchange (341) - : : : : +- Filter (340) - : : : : +- Scan parquet (339) - : : : +- Sort (346) - : : : +- Exchange (345) - : : : +- Filter (344) - : : : +- Scan parquet (343) - : : +- Sort (354) - : : +- Exchange (353) - : : +- Filter (352) - : : +- Scan parquet (351) - : +- Sort (385) - : +- Exchange (384) - : +- HashAggregate (383) - : +- Exchange (382) - : +- HashAggregate (381) - : +- Project (380) - : +- SortMergeJoin Inner (379) - : :- Sort (374) - : : +- Exchange (373) - : : +- Project (372) - : : +- SortMergeJoin Inner (371) - : : :- Sort (366) - : : : +- Exchange (365) - : : : +- Filter (364) - : : : +- Scan parquet (363) - : : +- Sort (370) - : : +- Exchange (369) - : : +- Filter (368) - : : +- Scan parquet (367) - : +- Sort (378) - : +- Exchange (377) - : +- Filter (376) - : +- Scan parquet (375) - +- Sort (410) - +- Exchange (409) - +- HashAggregate (408) - +- Exchange (407) - +- HashAggregate (406) - +- Project (405) - +- SortMergeJoin Inner (404) - :- Sort (399) - : +- Exchange (398) - : +- Project (397) - : +- SortMergeJoin Inner (396) - : :- Sort (391) - : : +- Exchange (390) - : : +- Filter (389) - : : +- Scan parquet (388) - : +- Sort (395) - : +- Exchange (394) - : +- Filter (393) - : +- Scan parquet (392) - +- Sort (403) - +- Exchange (402) - +- Filter (401) - +- Scan parquet (400) - - -(267) Scan parquet + Sort (415) + +- Exchange (414) + +- Project (413) + +- SortMergeJoin Inner (412) + :- Project (388) + : +- SortMergeJoin Inner (387) + : :- SortMergeJoin Inner (363) + : : :- Project (339) + : : : +- SortMergeJoin Inner (338) + : : : :- SortMergeJoin Inner (314) + : : : : :- Sort (290) + : : : : : +- Exchange (289) + : : : : : +- HashAggregate (288) + : : : : : +- Exchange (287) + : : : : : +- HashAggregate (286) + : : : : : +- Project (285) + : : : : : +- SortMergeJoin Inner (284) + : : : : : :- Sort (279) + : : : : : : +- Exchange (278) + : : : : : : +- Project (277) + : : : : : : +- SortMergeJoin Inner (276) + : : : : : : :- Sort (271) + : : : : : : : +- Exchange (270) + : : : : : : : +- Filter (269) + : : : : : : : +- Scan parquet (268) + : : : : : : +- Sort (275) + : : : : : : +- Exchange (274) + : : : : : : +- Filter (273) + : : : : : : +- Scan parquet (272) + : : : : : +- Sort (283) + : : : : : +- Exchange (282) + : : : : : +- Filter (281) + : : : : : +- Scan parquet (280) + : : : : +- Sort (313) + : : : : +- Exchange (312) + : : : : +- HashAggregate (311) + : : : : +- Exchange (310) + : : : : +- HashAggregate (309) + : : : : +- Project (308) + : : : : +- SortMergeJoin Inner (307) + : : : : :- Sort (302) + : : : : : +- Exchange (301) + : : : : : +- Project (300) + : : : : : +- SortMergeJoin Inner (299) + : : : : : :- Sort (294) + : : : : : : +- Exchange (293) + : : : : : : +- Filter (292) + : : : : : : +- Scan parquet (291) + : : : : : +- Sort (298) + : : : : : +- Exchange (297) + : : : : : +- Filter (296) + : : : : : +- Scan parquet (295) + : : : : +- Sort (306) + : : : : +- Exchange (305) + : : : : +- Filter (304) + : : : : +- Scan parquet (303) + : : : +- Sort (337) + : : : +- Exchange (336) + : : : +- HashAggregate (335) + : : : +- Exchange (334) + : : : +- HashAggregate (333) + : : : +- Project (332) + : : : +- SortMergeJoin Inner (331) + : : : :- Sort (326) + : : : : +- Exchange (325) + : : : : +- Project (324) + : : : : +- SortMergeJoin Inner (323) + : : : : :- Sort (318) + : : : : : +- Exchange (317) + : : : : : +- Filter (316) + : : : : : +- Scan parquet (315) + : : : : +- Sort (322) + : : : : +- Exchange (321) + : : : : +- Filter (320) + : : : : +- Scan parquet (319) + : : : +- Sort (330) + : : : +- Exchange (329) + : : : +- Filter (328) + : : : +- Scan parquet (327) + : : +- Sort (362) + : : +- Exchange (361) + : : +- HashAggregate (360) + : : +- Exchange (359) + : : +- HashAggregate (358) + : : +- Project (357) + : : +- SortMergeJoin Inner (356) + : : :- Sort (351) + : : : +- Exchange (350) + : : : +- Project (349) + : : : +- SortMergeJoin Inner (348) + : : : :- Sort (343) + : : : : +- Exchange (342) + : : : : +- Filter (341) + : : : : +- Scan parquet (340) + : : : +- Sort (347) + : : : +- Exchange (346) + : : : +- Filter (345) + : : : +- Scan parquet (344) + : : +- Sort (355) + : : +- Exchange (354) + : : +- Filter (353) + : : +- Scan parquet (352) + : +- Sort (386) + : +- Exchange (385) + : +- HashAggregate (384) + : +- Exchange (383) + : +- HashAggregate (382) + : +- Project (381) + : +- SortMergeJoin Inner (380) + : :- Sort (375) + : : +- Exchange (374) + : : +- Project (373) + : : +- SortMergeJoin Inner (372) + : : :- Sort (367) + : : : +- Exchange (366) + : : : +- Filter (365) + : : : +- Scan parquet (364) + : : +- Sort (371) + : : +- Exchange (370) + : : +- Filter (369) + : : +- Scan parquet (368) + : +- Sort (379) + : +- Exchange (378) + : +- Filter (377) + : +- Scan parquet (376) + +- Sort (411) + +- Exchange (410) + +- HashAggregate (409) + +- Exchange (408) + +- HashAggregate (407) + +- Project (406) + +- SortMergeJoin Inner (405) + :- Sort (400) + : +- Exchange (399) + : +- Project (398) + : +- SortMergeJoin Inner (397) + : :- Sort (392) + : : +- Exchange (391) + : : +- Filter (390) + : : +- Scan parquet (389) + : +- Sort (396) + : +- Exchange (395) + : +- Filter (394) + : +- Scan parquet (393) + +- Sort (404) + +- Exchange (403) + +- Filter (402) + +- Scan parquet (401) + + +(268) Scan parquet Output [3]: [ss_sold_date_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] Batched: true Location: InMemoryFileIndex [file:/] @@ -452,7 +453,7 @@ Input [3]: [#1#1, #2#2, #3#3] Input [3]: [#1#1, #2#2, #3#3] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false -(271) Scan parquet +(272) Scan parquet Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location: InMemoryFileIndex [file:/] @@ -515,7 +516,7 @@ Input [4]: [ss_addr_sk#2, ss_ext_sales_price#3, d_year#5, d_qoy#6] Input [4]: [ss_addr_sk#2, ss_ext_sales_price#3, d_year#5, d_qoy#6] Arguments: [ss_addr_sk#2 ASC NULLS FIRST], false -(279) Scan parquet +(280) Scan parquet Output [2]: [ca_address_sk#7, ca_county#8] Batched: true Location: InMemoryFileIndex [file:/] @@ -637,7 +638,7 @@ Input [3]: [#14#14, #15#15, #16#16] Input [3]: [#14#14, #15#15, #16#16] Arguments: [ss_sold_date_sk#14 ASC NULLS FIRST], false -(294) Scan parquet +(295) Scan parquet Output [3]: [d_date_sk#17, d_year#18, d_qoy#19] Batched: true Location: InMemoryFileIndex [file:/] @@ -816,7 +817,7 @@ Input [3]: [#25#25, #26#26, #27#27] Input [3]: [#25#25, #26#26, #27#27] Arguments: [ss_sold_date_sk#25 ASC NULLS FIRST], false -(318) Scan parquet +(319) Scan parquet Output [3]: [d_date_sk#28, d_year#29, d_qoy#30] Batched: true Location: InMemoryFileIndex [file:/] @@ -977,7 +978,7 @@ Join condition: None Output [5]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35] Input [7]: [ca_county#8, d_year#5, store_sales#13, ca_county#21, store_sales#24, ca_county#32, store_sales#35] -(339) Scan parquet +(340) Scan parquet Output [3]: [ws_sold_date_sk#36, ws_bill_addr_sk#37, ws_ext_sales_price#38] Batched: true Location: InMemoryFileIndex [file:/] @@ -1317,926 +1318,931 @@ Input [2]: [ca_county#55, web_sales#58] Input [2]: [ca_county#55, web_sales#58] Arguments: [ca_county#55 ASC NULLS FIRST], false -(217) SortMergeJoin [codegen id : X] +(217) NativeSortMergeJoin Left keys [1]: [ca_county#43] Right keys [1]: [ca_county#55] Join type: Inner -Join condition: (CASE WHEN (web_sales#47 > 0.00) THEN (web_sales#58 / web_sales#47) END > CASE WHEN (store_sales#13 > 0.00) THEN (store_sales#24 / store_sales#13) END) +Join condition: None -(218) Project [codegen id : X] +(218) NativeFilter +Input [9]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35, ca_county#43, web_sales#47, ca_county#55, web_sales#58] +Condition : (CASE WHEN (web_sales#47 > 0.00) THEN (web_sales#58 / web_sales#47) END > CASE WHEN (store_sales#13 > 0.00) THEN (store_sales#24 / store_sales#13) END) + +(219) NativeProject Output [8]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35, ca_county#43, web_sales#47, web_sales#58] Input [9]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35, ca_county#43, web_sales#47, ca_county#55, web_sales#58] -(219) ReusedExchange [Reuses operator id: 137] +(220) ReusedExchange [Reuses operator id: 137] Output [3]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61] -(220) ShuffleQueryStage +(221) ShuffleQueryStage Output [3]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61] Arguments: X -(221) AQEShuffleRead +(222) AQEShuffleRead Input [3]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61] Arguments: coalesced -(222) InputAdapter +(223) InputAdapter Input [3]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61] Arguments: [#59, #60, #61] -(223) InputAdapter +(224) InputAdapter Input [3]: [#59#59, #60#60, #61#61] -(224) NativeSort +(225) NativeSort Input [3]: [#59#59, #60#60, #61#61] Arguments: [ws_sold_date_sk#59 ASC NULLS FIRST], false -(225) ReusedExchange [Reuses operator id: 99] +(226) ReusedExchange [Reuses operator id: 99] Output [3]: [d_date_sk#62, d_year#63, d_qoy#64] -(226) ShuffleQueryStage +(227) ShuffleQueryStage Output [3]: [d_date_sk#62, d_year#63, d_qoy#64] Arguments: X -(227) AQEShuffleRead +(228) AQEShuffleRead Input [3]: [d_date_sk#62, d_year#63, d_qoy#64] Arguments: coalesced -(228) InputAdapter +(229) InputAdapter Input [3]: [d_date_sk#62, d_year#63, d_qoy#64] Arguments: [#62, #63, #64] -(229) InputAdapter +(230) InputAdapter Input [3]: [#62#62, #63#63, #64#64] -(230) NativeSort +(231) NativeSort Input [3]: [#62#62, #63#63, #64#64] Arguments: [d_date_sk#62 ASC NULLS FIRST], false -(231) NativeSortMergeJoin +(232) NativeSortMergeJoin Left keys [1]: [ws_sold_date_sk#59] Right keys [1]: [d_date_sk#62] Join type: Inner Join condition: None -(232) NativeProject +(233) NativeProject Output [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] Input [6]: [#59#59, #60#60, #61#61, #62#62, #63#63, #64#64] -(233) NativeShuffleExchange +(234) NativeShuffleExchange Input [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] Arguments: hashpartitioning(ws_bill_addr_sk#60, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(234) ShuffleQueryStage +(235) ShuffleQueryStage Output [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] Arguments: X -(235) AQEShuffleRead +(236) AQEShuffleRead Input [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] Arguments: coalesced -(236) InputAdapter +(237) InputAdapter Input [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] -(237) NativeSort +(238) NativeSort Input [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] Arguments: [ws_bill_addr_sk#60 ASC NULLS FIRST], false -(238) ReusedExchange [Reuses operator id: 27] +(239) ReusedExchange [Reuses operator id: 27] Output [2]: [ca_address_sk#65, ca_county#66] -(239) ShuffleQueryStage +(240) ShuffleQueryStage Output [2]: [ca_address_sk#65, ca_county#66] Arguments: X -(240) AQEShuffleRead +(241) AQEShuffleRead Input [2]: [ca_address_sk#65, ca_county#66] Arguments: coalesced -(241) InputAdapter +(242) InputAdapter Input [2]: [ca_address_sk#65, ca_county#66] Arguments: [#65, #66] -(242) InputAdapter +(243) InputAdapter Input [2]: [#65#65, #66#66] -(243) NativeSort +(244) NativeSort Input [2]: [#65#65, #66#66] Arguments: [ca_address_sk#65 ASC NULLS FIRST], false -(244) NativeSortMergeJoin +(245) NativeSortMergeJoin Left keys [1]: [ws_bill_addr_sk#60] Right keys [1]: [ca_address_sk#65] Join type: Inner Join condition: None -(245) NativeProject +(246) NativeProject Output [4]: [ws_ext_sales_price#61, d_year#63, d_qoy#64, ca_county#66] Input [6]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64, #65#65, #66#66] -(246) NativeProject +(247) NativeProject Output [4]: [ca_county#66 AS ca_county#66, d_qoy#64 AS d_qoy#64, d_year#63 AS d_year#63, UnscaledValue(ws_ext_sales_price#61) AS _c3#67] Input [4]: [ws_ext_sales_price#61, d_year#63, d_qoy#64, ca_county#66] -(247) NativeHashAggregate +(248) NativeHashAggregate Input [4]: [ca_county#66, d_qoy#64, d_year#63, _c3#67] Keys [3]: [ca_county#66, d_qoy#64, d_year#63] Functions [1]: [partial_sum(_c3#67)] Aggregate Attributes [1]: [sum#68] Results [4]: [ca_county#66, d_qoy#64, d_year#63, #11] -(248) NativeShuffleExchange +(249) NativeShuffleExchange Input [4]: [ca_county#66, d_qoy#64, d_year#63, #11] Arguments: hashpartitioning(ca_county#66, d_qoy#64, d_year#63, 100), ENSURE_REQUIREMENTS, [plan_id=23] -(249) ShuffleQueryStage +(250) ShuffleQueryStage Output [4]: [ca_county#66, d_qoy#64, d_year#63, #11] Arguments: X -(250) AQEShuffleRead +(251) AQEShuffleRead Input [4]: [ca_county#66, d_qoy#64, d_year#63, #11] Arguments: coalesced -(251) InputAdapter +(252) InputAdapter Input [4]: [ca_county#66, d_qoy#64, d_year#63, #11] -(252) NativeHashAggregate +(253) NativeHashAggregate Input [4]: [ca_county#66, d_qoy#64, d_year#63, #11] Keys [3]: [ca_county#66, d_qoy#64, d_year#63] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#61))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#61))#46] Results [4]: [ca_county#66, d_qoy#64, d_year#63, sum(UnscaledValue(ws_ext_sales_price#61))#46] -(253) NativeProject +(254) NativeProject Output [2]: [ca_county#66, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#61))#46,17,2) AS web_sales#69] Input [4]: [ca_county#66, d_qoy#64, d_year#63, sum(UnscaledValue(ws_ext_sales_price#61))#46] -(254) NativeShuffleExchange +(255) NativeShuffleExchange Input [2]: [ca_county#66, web_sales#69] Arguments: hashpartitioning(ca_county#66, 100), ENSURE_REQUIREMENTS, [plan_id=24] -(255) ShuffleQueryStage +(256) ShuffleQueryStage Output [2]: [ca_county#66, web_sales#69] Arguments: X -(256) AQEShuffleRead +(257) AQEShuffleRead Input [2]: [ca_county#66, web_sales#69] Arguments: coalesced -(257) InputAdapter +(258) InputAdapter Input [2]: [ca_county#66, web_sales#69] -(258) NativeSort +(259) NativeSort Input [2]: [ca_county#66, web_sales#69] Arguments: [ca_county#66 ASC NULLS FIRST], false -(259) SortMergeJoin [codegen id : X] +(260) NativeSortMergeJoin Left keys [1]: [ca_county#43] Right keys [1]: [ca_county#66] Join type: Inner -Join condition: (CASE WHEN (web_sales#58 > 0.00) THEN (web_sales#69 / web_sales#58) END > CASE WHEN (store_sales#24 > 0.00) THEN (store_sales#35 / store_sales#24) END) +Join condition: None -(260) Project [codegen id : X] -Output [6]: [ca_county#8, d_year#5, (web_sales#58 / web_sales#47) AS web_q1_q2_increase#70, (store_sales#24 / store_sales#13) AS store_q1_q2_increase#71, (web_sales#69 / web_sales#58) AS web_q2_q3_increase#72, (store_sales#35 / store_sales#24) AS store_q2_q3_increase#73] +(261) NativeFilter Input [10]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35, ca_county#43, web_sales#47, web_sales#58, ca_county#66, web_sales#69] +Condition : (CASE WHEN (web_sales#58 > 0.00) THEN (web_sales#69 / web_sales#58) END > CASE WHEN (store_sales#24 > 0.00) THEN (store_sales#35 / store_sales#24) END) -(261) ConvertToNative -Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] +(262) NativeProject +Output [6]: [ca_county#8, d_year#5, (web_sales#58 / web_sales#47) AS web_q1_q2_increase#70, (store_sales#24 / store_sales#13) AS store_q1_q2_increase#71, (web_sales#69 / web_sales#58) AS web_q2_q3_increase#72, (store_sales#35 / store_sales#24) AS store_q2_q3_increase#73] +Input [10]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35, ca_county#43, web_sales#47, web_sales#58, ca_county#66, web_sales#69] -(262) NativeShuffleExchange +(263) NativeShuffleExchange Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 100), ENSURE_REQUIREMENTS, [plan_id=25] -(263) ShuffleQueryStage +(264) ShuffleQueryStage Output [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] Arguments: X -(264) AQEShuffleRead +(265) AQEShuffleRead Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] Arguments: coalesced -(265) InputAdapter +(266) InputAdapter Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] -(266) NativeSort +(267) NativeSort Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] Arguments: [ca_county#8 ASC NULLS FIRST], true -(267) Scan parquet +(268) Scan parquet Output [3]: [ss_sold_date_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(268) Filter +(269) Filter Input [3]: [ss_sold_date_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] Condition : (isnotnull(ss_sold_date_sk#1) AND isnotnull(ss_addr_sk#2)) -(269) Exchange +(270) Exchange Input [3]: [ss_sold_date_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] Arguments: hashpartitioning(ss_sold_date_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=26] -(270) Sort +(271) Sort Input [3]: [ss_sold_date_sk#1, ss_addr_sk#2, ss_ext_sales_price#3] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false, 0 -(271) Scan parquet +(272) Scan parquet Output [3]: [d_date_sk#4, d_year#5, d_qoy#6] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(272) Filter +(273) Filter Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] Condition : ((((isnotnull(d_qoy#6) AND isnotnull(d_year#5)) AND (d_qoy#6 = 1)) AND (d_year#5 = 2000)) AND isnotnull(d_date_sk#4)) -(273) Exchange +(274) Exchange Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] Arguments: hashpartitioning(d_date_sk#4, 100), ENSURE_REQUIREMENTS, [plan_id=27] -(274) Sort +(275) Sort Input [3]: [d_date_sk#4, d_year#5, d_qoy#6] Arguments: [d_date_sk#4 ASC NULLS FIRST], false, 0 -(275) SortMergeJoin +(276) SortMergeJoin Left keys [1]: [ss_sold_date_sk#1] Right keys [1]: [d_date_sk#4] Join type: Inner Join condition: None -(276) Project +(277) Project Output [4]: [ss_addr_sk#2, ss_ext_sales_price#3, d_year#5, d_qoy#6] Input [6]: [ss_sold_date_sk#1, ss_addr_sk#2, ss_ext_sales_price#3, d_date_sk#4, d_year#5, d_qoy#6] -(277) Exchange +(278) Exchange Input [4]: [ss_addr_sk#2, ss_ext_sales_price#3, d_year#5, d_qoy#6] Arguments: hashpartitioning(ss_addr_sk#2, 100), ENSURE_REQUIREMENTS, [plan_id=28] -(278) Sort +(279) Sort Input [4]: [ss_addr_sk#2, ss_ext_sales_price#3, d_year#5, d_qoy#6] Arguments: [ss_addr_sk#2 ASC NULLS FIRST], false, 0 -(279) Scan parquet +(280) Scan parquet Output [2]: [ca_address_sk#7, ca_county#8] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(280) Filter +(281) Filter Input [2]: [ca_address_sk#7, ca_county#8] Condition : (isnotnull(ca_address_sk#7) AND isnotnull(ca_county#8)) -(281) Exchange +(282) Exchange Input [2]: [ca_address_sk#7, ca_county#8] Arguments: hashpartitioning(ca_address_sk#7, 100), ENSURE_REQUIREMENTS, [plan_id=29] -(282) Sort +(283) Sort Input [2]: [ca_address_sk#7, ca_county#8] Arguments: [ca_address_sk#7 ASC NULLS FIRST], false, 0 -(283) SortMergeJoin +(284) SortMergeJoin Left keys [1]: [ss_addr_sk#2] Right keys [1]: [ca_address_sk#7] Join type: Inner Join condition: None -(284) Project +(285) Project Output [4]: [ss_ext_sales_price#3, d_year#5, d_qoy#6, ca_county#8] Input [6]: [ss_addr_sk#2, ss_ext_sales_price#3, d_year#5, d_qoy#6, ca_address_sk#7, ca_county#8] -(285) HashAggregate +(286) HashAggregate Input [4]: [ss_ext_sales_price#3, d_year#5, d_qoy#6, ca_county#8] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum#10] Results [4]: [ca_county#8, d_qoy#6, d_year#5, sum#74] -(286) Exchange +(287) Exchange Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#74] Arguments: hashpartitioning(ca_county#8, d_qoy#6, d_year#5, 100), ENSURE_REQUIREMENTS, [plan_id=30] -(287) HashAggregate +(288) HashAggregate Input [4]: [ca_county#8, d_qoy#6, d_year#5, sum#74] Keys [3]: [ca_county#8, d_qoy#6, d_year#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#12] Results [3]: [ca_county#8, d_year#5, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#12,17,2) AS store_sales#13] -(288) Exchange +(289) Exchange Input [3]: [ca_county#8, d_year#5, store_sales#13] Arguments: hashpartitioning(ca_county#8, 100), ENSURE_REQUIREMENTS, [plan_id=31] -(289) Sort +(290) Sort Input [3]: [ca_county#8, d_year#5, store_sales#13] Arguments: [ca_county#8 ASC NULLS FIRST], false, 0 -(290) Scan parquet +(291) Scan parquet Output [3]: [ss_sold_date_sk#14, ss_addr_sk#15, ss_ext_sales_price#16] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(291) Filter +(292) Filter Input [3]: [ss_sold_date_sk#14, ss_addr_sk#15, ss_ext_sales_price#16] Condition : (isnotnull(ss_sold_date_sk#14) AND isnotnull(ss_addr_sk#15)) -(292) Exchange +(293) Exchange Input [3]: [ss_sold_date_sk#14, ss_addr_sk#15, ss_ext_sales_price#16] Arguments: hashpartitioning(ss_sold_date_sk#14, 100), ENSURE_REQUIREMENTS, [plan_id=32] -(293) Sort +(294) Sort Input [3]: [ss_sold_date_sk#14, ss_addr_sk#15, ss_ext_sales_price#16] Arguments: [ss_sold_date_sk#14 ASC NULLS FIRST], false, 0 -(294) Scan parquet +(295) Scan parquet Output [3]: [d_date_sk#17, d_year#18, d_qoy#19] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(295) Filter +(296) Filter Input [3]: [d_date_sk#17, d_year#18, d_qoy#19] Condition : ((((isnotnull(d_qoy#19) AND isnotnull(d_year#18)) AND (d_qoy#19 = 2)) AND (d_year#18 = 2000)) AND isnotnull(d_date_sk#17)) -(296) Exchange +(297) Exchange Input [3]: [d_date_sk#17, d_year#18, d_qoy#19] Arguments: hashpartitioning(d_date_sk#17, 100), ENSURE_REQUIREMENTS, [plan_id=33] -(297) Sort +(298) Sort Input [3]: [d_date_sk#17, d_year#18, d_qoy#19] Arguments: [d_date_sk#17 ASC NULLS FIRST], false, 0 -(298) SortMergeJoin +(299) SortMergeJoin Left keys [1]: [ss_sold_date_sk#14] Right keys [1]: [d_date_sk#17] Join type: Inner Join condition: None -(299) Project +(300) Project Output [4]: [ss_addr_sk#15, ss_ext_sales_price#16, d_year#18, d_qoy#19] Input [6]: [ss_sold_date_sk#14, ss_addr_sk#15, ss_ext_sales_price#16, d_date_sk#17, d_year#18, d_qoy#19] -(300) Exchange +(301) Exchange Input [4]: [ss_addr_sk#15, ss_ext_sales_price#16, d_year#18, d_qoy#19] Arguments: hashpartitioning(ss_addr_sk#15, 100), ENSURE_REQUIREMENTS, [plan_id=34] -(301) Sort +(302) Sort Input [4]: [ss_addr_sk#15, ss_ext_sales_price#16, d_year#18, d_qoy#19] Arguments: [ss_addr_sk#15 ASC NULLS FIRST], false, 0 -(302) Scan parquet +(303) Scan parquet Output [2]: [ca_address_sk#20, ca_county#21] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(303) Filter +(304) Filter Input [2]: [ca_address_sk#20, ca_county#21] Condition : (isnotnull(ca_address_sk#20) AND isnotnull(ca_county#21)) -(304) Exchange +(305) Exchange Input [2]: [ca_address_sk#20, ca_county#21] Arguments: hashpartitioning(ca_address_sk#20, 100), ENSURE_REQUIREMENTS, [plan_id=35] -(305) Sort +(306) Sort Input [2]: [ca_address_sk#20, ca_county#21] Arguments: [ca_address_sk#20 ASC NULLS FIRST], false, 0 -(306) SortMergeJoin +(307) SortMergeJoin Left keys [1]: [ss_addr_sk#15] Right keys [1]: [ca_address_sk#20] Join type: Inner Join condition: None -(307) Project +(308) Project Output [4]: [ss_ext_sales_price#16, d_year#18, d_qoy#19, ca_county#21] Input [6]: [ss_addr_sk#15, ss_ext_sales_price#16, d_year#18, d_qoy#19, ca_address_sk#20, ca_county#21] -(308) HashAggregate +(309) HashAggregate Input [4]: [ss_ext_sales_price#16, d_year#18, d_qoy#19, ca_county#21] Keys [3]: [ca_county#21, d_qoy#19, d_year#18] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#16))] Aggregate Attributes [1]: [sum#23] Results [4]: [ca_county#21, d_qoy#19, d_year#18, sum#75] -(309) Exchange +(310) Exchange Input [4]: [ca_county#21, d_qoy#19, d_year#18, sum#75] Arguments: hashpartitioning(ca_county#21, d_qoy#19, d_year#18, 100), ENSURE_REQUIREMENTS, [plan_id=36] -(310) HashAggregate +(311) HashAggregate Input [4]: [ca_county#21, d_qoy#19, d_year#18, sum#75] Keys [3]: [ca_county#21, d_qoy#19, d_year#18] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#16))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#16))#12] Results [2]: [ca_county#21, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#16))#12,17,2) AS store_sales#24] -(311) Exchange +(312) Exchange Input [2]: [ca_county#21, store_sales#24] Arguments: hashpartitioning(ca_county#21, 100), ENSURE_REQUIREMENTS, [plan_id=37] -(312) Sort +(313) Sort Input [2]: [ca_county#21, store_sales#24] Arguments: [ca_county#21 ASC NULLS FIRST], false, 0 -(313) SortMergeJoin +(314) SortMergeJoin Left keys [1]: [ca_county#8] Right keys [1]: [ca_county#21] Join type: Inner Join condition: None -(314) Scan parquet +(315) Scan parquet Output [3]: [ss_sold_date_sk#25, ss_addr_sk#26, ss_ext_sales_price#27] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(315) Filter +(316) Filter Input [3]: [ss_sold_date_sk#25, ss_addr_sk#26, ss_ext_sales_price#27] Condition : (isnotnull(ss_sold_date_sk#25) AND isnotnull(ss_addr_sk#26)) -(316) Exchange +(317) Exchange Input [3]: [ss_sold_date_sk#25, ss_addr_sk#26, ss_ext_sales_price#27] Arguments: hashpartitioning(ss_sold_date_sk#25, 100), ENSURE_REQUIREMENTS, [plan_id=38] -(317) Sort +(318) Sort Input [3]: [ss_sold_date_sk#25, ss_addr_sk#26, ss_ext_sales_price#27] Arguments: [ss_sold_date_sk#25 ASC NULLS FIRST], false, 0 -(318) Scan parquet +(319) Scan parquet Output [3]: [d_date_sk#28, d_year#29, d_qoy#30] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(319) Filter +(320) Filter Input [3]: [d_date_sk#28, d_year#29, d_qoy#30] Condition : ((((isnotnull(d_qoy#30) AND isnotnull(d_year#29)) AND (d_qoy#30 = 3)) AND (d_year#29 = 2000)) AND isnotnull(d_date_sk#28)) -(320) Exchange +(321) Exchange Input [3]: [d_date_sk#28, d_year#29, d_qoy#30] Arguments: hashpartitioning(d_date_sk#28, 100), ENSURE_REQUIREMENTS, [plan_id=39] -(321) Sort +(322) Sort Input [3]: [d_date_sk#28, d_year#29, d_qoy#30] Arguments: [d_date_sk#28 ASC NULLS FIRST], false, 0 -(322) SortMergeJoin +(323) SortMergeJoin Left keys [1]: [ss_sold_date_sk#25] Right keys [1]: [d_date_sk#28] Join type: Inner Join condition: None -(323) Project +(324) Project Output [4]: [ss_addr_sk#26, ss_ext_sales_price#27, d_year#29, d_qoy#30] Input [6]: [ss_sold_date_sk#25, ss_addr_sk#26, ss_ext_sales_price#27, d_date_sk#28, d_year#29, d_qoy#30] -(324) Exchange +(325) Exchange Input [4]: [ss_addr_sk#26, ss_ext_sales_price#27, d_year#29, d_qoy#30] Arguments: hashpartitioning(ss_addr_sk#26, 100), ENSURE_REQUIREMENTS, [plan_id=40] -(325) Sort +(326) Sort Input [4]: [ss_addr_sk#26, ss_ext_sales_price#27, d_year#29, d_qoy#30] Arguments: [ss_addr_sk#26 ASC NULLS FIRST], false, 0 -(326) Scan parquet +(327) Scan parquet Output [2]: [ca_address_sk#31, ca_county#32] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(327) Filter +(328) Filter Input [2]: [ca_address_sk#31, ca_county#32] Condition : (isnotnull(ca_address_sk#31) AND isnotnull(ca_county#32)) -(328) Exchange +(329) Exchange Input [2]: [ca_address_sk#31, ca_county#32] Arguments: hashpartitioning(ca_address_sk#31, 100), ENSURE_REQUIREMENTS, [plan_id=41] -(329) Sort +(330) Sort Input [2]: [ca_address_sk#31, ca_county#32] Arguments: [ca_address_sk#31 ASC NULLS FIRST], false, 0 -(330) SortMergeJoin +(331) SortMergeJoin Left keys [1]: [ss_addr_sk#26] Right keys [1]: [ca_address_sk#31] Join type: Inner Join condition: None -(331) Project +(332) Project Output [4]: [ss_ext_sales_price#27, d_year#29, d_qoy#30, ca_county#32] Input [6]: [ss_addr_sk#26, ss_ext_sales_price#27, d_year#29, d_qoy#30, ca_address_sk#31, ca_county#32] -(332) HashAggregate +(333) HashAggregate Input [4]: [ss_ext_sales_price#27, d_year#29, d_qoy#30, ca_county#32] Keys [3]: [ca_county#32, d_qoy#30, d_year#29] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#27))] Aggregate Attributes [1]: [sum#34] Results [4]: [ca_county#32, d_qoy#30, d_year#29, sum#76] -(333) Exchange +(334) Exchange Input [4]: [ca_county#32, d_qoy#30, d_year#29, sum#76] Arguments: hashpartitioning(ca_county#32, d_qoy#30, d_year#29, 100), ENSURE_REQUIREMENTS, [plan_id=42] -(334) HashAggregate +(335) HashAggregate Input [4]: [ca_county#32, d_qoy#30, d_year#29, sum#76] Keys [3]: [ca_county#32, d_qoy#30, d_year#29] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#27))#12] Results [2]: [ca_county#32, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#27))#12,17,2) AS store_sales#35] -(335) Exchange +(336) Exchange Input [2]: [ca_county#32, store_sales#35] Arguments: hashpartitioning(ca_county#32, 100), ENSURE_REQUIREMENTS, [plan_id=43] -(336) Sort +(337) Sort Input [2]: [ca_county#32, store_sales#35] Arguments: [ca_county#32 ASC NULLS FIRST], false, 0 -(337) SortMergeJoin +(338) SortMergeJoin Left keys [1]: [ca_county#21] Right keys [1]: [ca_county#32] Join type: Inner Join condition: None -(338) Project +(339) Project Output [5]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35] Input [7]: [ca_county#8, d_year#5, store_sales#13, ca_county#21, store_sales#24, ca_county#32, store_sales#35] -(339) Scan parquet +(340) Scan parquet Output [3]: [ws_sold_date_sk#36, ws_bill_addr_sk#37, ws_ext_sales_price#38] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_sold_date_sk), IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(340) Filter +(341) Filter Input [3]: [ws_sold_date_sk#36, ws_bill_addr_sk#37, ws_ext_sales_price#38] Condition : (isnotnull(ws_sold_date_sk#36) AND isnotnull(ws_bill_addr_sk#37)) -(341) Exchange +(342) Exchange Input [3]: [ws_sold_date_sk#36, ws_bill_addr_sk#37, ws_ext_sales_price#38] Arguments: hashpartitioning(ws_sold_date_sk#36, 100), ENSURE_REQUIREMENTS, [plan_id=44] -(342) Sort +(343) Sort Input [3]: [ws_sold_date_sk#36, ws_bill_addr_sk#37, ws_ext_sales_price#38] Arguments: [ws_sold_date_sk#36 ASC NULLS FIRST], false, 0 -(343) Scan parquet +(344) Scan parquet Output [3]: [d_date_sk#39, d_year#40, d_qoy#41] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(344) Filter +(345) Filter Input [3]: [d_date_sk#39, d_year#40, d_qoy#41] Condition : ((((isnotnull(d_qoy#41) AND isnotnull(d_year#40)) AND (d_qoy#41 = 1)) AND (d_year#40 = 2000)) AND isnotnull(d_date_sk#39)) -(345) Exchange +(346) Exchange Input [3]: [d_date_sk#39, d_year#40, d_qoy#41] Arguments: hashpartitioning(d_date_sk#39, 100), ENSURE_REQUIREMENTS, [plan_id=45] -(346) Sort +(347) Sort Input [3]: [d_date_sk#39, d_year#40, d_qoy#41] Arguments: [d_date_sk#39 ASC NULLS FIRST], false, 0 -(347) SortMergeJoin +(348) SortMergeJoin Left keys [1]: [ws_sold_date_sk#36] Right keys [1]: [d_date_sk#39] Join type: Inner Join condition: None -(348) Project +(349) Project Output [4]: [ws_bill_addr_sk#37, ws_ext_sales_price#38, d_year#40, d_qoy#41] Input [6]: [ws_sold_date_sk#36, ws_bill_addr_sk#37, ws_ext_sales_price#38, d_date_sk#39, d_year#40, d_qoy#41] -(349) Exchange +(350) Exchange Input [4]: [ws_bill_addr_sk#37, ws_ext_sales_price#38, d_year#40, d_qoy#41] Arguments: hashpartitioning(ws_bill_addr_sk#37, 100), ENSURE_REQUIREMENTS, [plan_id=46] -(350) Sort +(351) Sort Input [4]: [ws_bill_addr_sk#37, ws_ext_sales_price#38, d_year#40, d_qoy#41] Arguments: [ws_bill_addr_sk#37 ASC NULLS FIRST], false, 0 -(351) Scan parquet +(352) Scan parquet Output [2]: [ca_address_sk#42, ca_county#43] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(352) Filter +(353) Filter Input [2]: [ca_address_sk#42, ca_county#43] Condition : (isnotnull(ca_address_sk#42) AND isnotnull(ca_county#43)) -(353) Exchange +(354) Exchange Input [2]: [ca_address_sk#42, ca_county#43] Arguments: hashpartitioning(ca_address_sk#42, 100), ENSURE_REQUIREMENTS, [plan_id=47] -(354) Sort +(355) Sort Input [2]: [ca_address_sk#42, ca_county#43] Arguments: [ca_address_sk#42 ASC NULLS FIRST], false, 0 -(355) SortMergeJoin +(356) SortMergeJoin Left keys [1]: [ws_bill_addr_sk#37] Right keys [1]: [ca_address_sk#42] Join type: Inner Join condition: None -(356) Project +(357) Project Output [4]: [ws_ext_sales_price#38, d_year#40, d_qoy#41, ca_county#43] Input [6]: [ws_bill_addr_sk#37, ws_ext_sales_price#38, d_year#40, d_qoy#41, ca_address_sk#42, ca_county#43] -(357) HashAggregate +(358) HashAggregate Input [4]: [ws_ext_sales_price#38, d_year#40, d_qoy#41, ca_county#43] Keys [3]: [ca_county#43, d_qoy#41, d_year#40] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#38))] Aggregate Attributes [1]: [sum#45] Results [4]: [ca_county#43, d_qoy#41, d_year#40, sum#77] -(358) Exchange +(359) Exchange Input [4]: [ca_county#43, d_qoy#41, d_year#40, sum#77] Arguments: hashpartitioning(ca_county#43, d_qoy#41, d_year#40, 100), ENSURE_REQUIREMENTS, [plan_id=48] -(359) HashAggregate +(360) HashAggregate Input [4]: [ca_county#43, d_qoy#41, d_year#40, sum#77] Keys [3]: [ca_county#43, d_qoy#41, d_year#40] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#38))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#38))#46] Results [2]: [ca_county#43, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#38))#46,17,2) AS web_sales#47] -(360) Exchange +(361) Exchange Input [2]: [ca_county#43, web_sales#47] Arguments: hashpartitioning(ca_county#43, 100), ENSURE_REQUIREMENTS, [plan_id=49] -(361) Sort +(362) Sort Input [2]: [ca_county#43, web_sales#47] Arguments: [ca_county#43 ASC NULLS FIRST], false, 0 -(362) SortMergeJoin +(363) SortMergeJoin Left keys [1]: [ca_county#8] Right keys [1]: [ca_county#43] Join type: Inner Join condition: None -(363) Scan parquet +(364) Scan parquet Output [3]: [ws_sold_date_sk#48, ws_bill_addr_sk#49, ws_ext_sales_price#50] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_sold_date_sk), IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(364) Filter +(365) Filter Input [3]: [ws_sold_date_sk#48, ws_bill_addr_sk#49, ws_ext_sales_price#50] Condition : (isnotnull(ws_sold_date_sk#48) AND isnotnull(ws_bill_addr_sk#49)) -(365) Exchange +(366) Exchange Input [3]: [ws_sold_date_sk#48, ws_bill_addr_sk#49, ws_ext_sales_price#50] Arguments: hashpartitioning(ws_sold_date_sk#48, 100), ENSURE_REQUIREMENTS, [plan_id=50] -(366) Sort +(367) Sort Input [3]: [ws_sold_date_sk#48, ws_bill_addr_sk#49, ws_ext_sales_price#50] Arguments: [ws_sold_date_sk#48 ASC NULLS FIRST], false, 0 -(367) Scan parquet +(368) Scan parquet Output [3]: [d_date_sk#51, d_year#52, d_qoy#53] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(368) Filter +(369) Filter Input [3]: [d_date_sk#51, d_year#52, d_qoy#53] Condition : ((((isnotnull(d_qoy#53) AND isnotnull(d_year#52)) AND (d_qoy#53 = 2)) AND (d_year#52 = 2000)) AND isnotnull(d_date_sk#51)) -(369) Exchange +(370) Exchange Input [3]: [d_date_sk#51, d_year#52, d_qoy#53] Arguments: hashpartitioning(d_date_sk#51, 100), ENSURE_REQUIREMENTS, [plan_id=51] -(370) Sort +(371) Sort Input [3]: [d_date_sk#51, d_year#52, d_qoy#53] Arguments: [d_date_sk#51 ASC NULLS FIRST], false, 0 -(371) SortMergeJoin +(372) SortMergeJoin Left keys [1]: [ws_sold_date_sk#48] Right keys [1]: [d_date_sk#51] Join type: Inner Join condition: None -(372) Project +(373) Project Output [4]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, d_year#52, d_qoy#53] Input [6]: [ws_sold_date_sk#48, ws_bill_addr_sk#49, ws_ext_sales_price#50, d_date_sk#51, d_year#52, d_qoy#53] -(373) Exchange +(374) Exchange Input [4]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, d_year#52, d_qoy#53] Arguments: hashpartitioning(ws_bill_addr_sk#49, 100), ENSURE_REQUIREMENTS, [plan_id=52] -(374) Sort +(375) Sort Input [4]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, d_year#52, d_qoy#53] Arguments: [ws_bill_addr_sk#49 ASC NULLS FIRST], false, 0 -(375) Scan parquet +(376) Scan parquet Output [2]: [ca_address_sk#54, ca_county#55] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(376) Filter +(377) Filter Input [2]: [ca_address_sk#54, ca_county#55] Condition : (isnotnull(ca_address_sk#54) AND isnotnull(ca_county#55)) -(377) Exchange +(378) Exchange Input [2]: [ca_address_sk#54, ca_county#55] Arguments: hashpartitioning(ca_address_sk#54, 100), ENSURE_REQUIREMENTS, [plan_id=53] -(378) Sort +(379) Sort Input [2]: [ca_address_sk#54, ca_county#55] Arguments: [ca_address_sk#54 ASC NULLS FIRST], false, 0 -(379) SortMergeJoin +(380) SortMergeJoin Left keys [1]: [ws_bill_addr_sk#49] Right keys [1]: [ca_address_sk#54] Join type: Inner Join condition: None -(380) Project +(381) Project Output [4]: [ws_ext_sales_price#50, d_year#52, d_qoy#53, ca_county#55] Input [6]: [ws_bill_addr_sk#49, ws_ext_sales_price#50, d_year#52, d_qoy#53, ca_address_sk#54, ca_county#55] -(381) HashAggregate +(382) HashAggregate Input [4]: [ws_ext_sales_price#50, d_year#52, d_qoy#53, ca_county#55] Keys [3]: [ca_county#55, d_qoy#53, d_year#52] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#50))] Aggregate Attributes [1]: [sum#57] Results [4]: [ca_county#55, d_qoy#53, d_year#52, sum#78] -(382) Exchange +(383) Exchange Input [4]: [ca_county#55, d_qoy#53, d_year#52, sum#78] Arguments: hashpartitioning(ca_county#55, d_qoy#53, d_year#52, 100), ENSURE_REQUIREMENTS, [plan_id=54] -(383) HashAggregate +(384) HashAggregate Input [4]: [ca_county#55, d_qoy#53, d_year#52, sum#78] Keys [3]: [ca_county#55, d_qoy#53, d_year#52] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#50))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#50))#46] Results [2]: [ca_county#55, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#50))#46,17,2) AS web_sales#58] -(384) Exchange +(385) Exchange Input [2]: [ca_county#55, web_sales#58] Arguments: hashpartitioning(ca_county#55, 100), ENSURE_REQUIREMENTS, [plan_id=55] -(385) Sort +(386) Sort Input [2]: [ca_county#55, web_sales#58] Arguments: [ca_county#55 ASC NULLS FIRST], false, 0 -(386) SortMergeJoin +(387) SortMergeJoin Left keys [1]: [ca_county#43] Right keys [1]: [ca_county#55] Join type: Inner Join condition: (CASE WHEN (web_sales#47 > 0.00) THEN (web_sales#58 / web_sales#47) END > CASE WHEN (store_sales#13 > 0.00) THEN (store_sales#24 / store_sales#13) END) -(387) Project +(388) Project Output [8]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35, ca_county#43, web_sales#47, web_sales#58] Input [9]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35, ca_county#43, web_sales#47, ca_county#55, web_sales#58] -(388) Scan parquet +(389) Scan parquet Output [3]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_sold_date_sk), IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(389) Filter +(390) Filter Input [3]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61] Condition : (isnotnull(ws_sold_date_sk#59) AND isnotnull(ws_bill_addr_sk#60)) -(390) Exchange +(391) Exchange Input [3]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61] Arguments: hashpartitioning(ws_sold_date_sk#59, 100), ENSURE_REQUIREMENTS, [plan_id=56] -(391) Sort +(392) Sort Input [3]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61] Arguments: [ws_sold_date_sk#59 ASC NULLS FIRST], false, 0 -(392) Scan parquet +(393) Scan parquet Output [3]: [d_date_sk#62, d_year#63, d_qoy#64] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(393) Filter +(394) Filter Input [3]: [d_date_sk#62, d_year#63, d_qoy#64] Condition : ((((isnotnull(d_qoy#64) AND isnotnull(d_year#63)) AND (d_qoy#64 = 3)) AND (d_year#63 = 2000)) AND isnotnull(d_date_sk#62)) -(394) Exchange +(395) Exchange Input [3]: [d_date_sk#62, d_year#63, d_qoy#64] Arguments: hashpartitioning(d_date_sk#62, 100), ENSURE_REQUIREMENTS, [plan_id=57] -(395) Sort +(396) Sort Input [3]: [d_date_sk#62, d_year#63, d_qoy#64] Arguments: [d_date_sk#62 ASC NULLS FIRST], false, 0 -(396) SortMergeJoin +(397) SortMergeJoin Left keys [1]: [ws_sold_date_sk#59] Right keys [1]: [d_date_sk#62] Join type: Inner Join condition: None -(397) Project +(398) Project Output [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] Input [6]: [ws_sold_date_sk#59, ws_bill_addr_sk#60, ws_ext_sales_price#61, d_date_sk#62, d_year#63, d_qoy#64] -(398) Exchange +(399) Exchange Input [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] Arguments: hashpartitioning(ws_bill_addr_sk#60, 100), ENSURE_REQUIREMENTS, [plan_id=58] -(399) Sort +(400) Sort Input [4]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64] Arguments: [ws_bill_addr_sk#60 ASC NULLS FIRST], false, 0 -(400) Scan parquet +(401) Scan parquet Output [2]: [ca_address_sk#65, ca_county#66] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_county)] ReadSchema: struct -(401) Filter +(402) Filter Input [2]: [ca_address_sk#65, ca_county#66] Condition : (isnotnull(ca_address_sk#65) AND isnotnull(ca_county#66)) -(402) Exchange +(403) Exchange Input [2]: [ca_address_sk#65, ca_county#66] Arguments: hashpartitioning(ca_address_sk#65, 100), ENSURE_REQUIREMENTS, [plan_id=59] -(403) Sort +(404) Sort Input [2]: [ca_address_sk#65, ca_county#66] Arguments: [ca_address_sk#65 ASC NULLS FIRST], false, 0 -(404) SortMergeJoin +(405) SortMergeJoin Left keys [1]: [ws_bill_addr_sk#60] Right keys [1]: [ca_address_sk#65] Join type: Inner Join condition: None -(405) Project +(406) Project Output [4]: [ws_ext_sales_price#61, d_year#63, d_qoy#64, ca_county#66] Input [6]: [ws_bill_addr_sk#60, ws_ext_sales_price#61, d_year#63, d_qoy#64, ca_address_sk#65, ca_county#66] -(406) HashAggregate +(407) HashAggregate Input [4]: [ws_ext_sales_price#61, d_year#63, d_qoy#64, ca_county#66] Keys [3]: [ca_county#66, d_qoy#64, d_year#63] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#61))] Aggregate Attributes [1]: [sum#68] Results [4]: [ca_county#66, d_qoy#64, d_year#63, sum#79] -(407) Exchange +(408) Exchange Input [4]: [ca_county#66, d_qoy#64, d_year#63, sum#79] Arguments: hashpartitioning(ca_county#66, d_qoy#64, d_year#63, 100), ENSURE_REQUIREMENTS, [plan_id=60] -(408) HashAggregate +(409) HashAggregate Input [4]: [ca_county#66, d_qoy#64, d_year#63, sum#79] Keys [3]: [ca_county#66, d_qoy#64, d_year#63] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#61))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#61))#46] Results [2]: [ca_county#66, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#61))#46,17,2) AS web_sales#69] -(409) Exchange +(410) Exchange Input [2]: [ca_county#66, web_sales#69] Arguments: hashpartitioning(ca_county#66, 100), ENSURE_REQUIREMENTS, [plan_id=61] -(410) Sort +(411) Sort Input [2]: [ca_county#66, web_sales#69] Arguments: [ca_county#66 ASC NULLS FIRST], false, 0 -(411) SortMergeJoin +(412) SortMergeJoin Left keys [1]: [ca_county#43] Right keys [1]: [ca_county#66] Join type: Inner Join condition: (CASE WHEN (web_sales#58 > 0.00) THEN (web_sales#69 / web_sales#58) END > CASE WHEN (store_sales#24 > 0.00) THEN (store_sales#35 / store_sales#24) END) -(412) Project +(413) Project Output [6]: [ca_county#8, d_year#5, (web_sales#58 / web_sales#47) AS web_q1_q2_increase#70, (store_sales#24 / store_sales#13) AS store_q1_q2_increase#71, (web_sales#69 / web_sales#58) AS web_q2_q3_increase#72, (store_sales#35 / store_sales#24) AS store_q2_q3_increase#73] Input [10]: [ca_county#8, d_year#5, store_sales#13, store_sales#24, store_sales#35, ca_county#43, web_sales#47, web_sales#58, ca_county#66, web_sales#69] -(413) Exchange +(414) Exchange Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] Arguments: rangepartitioning(ca_county#8 ASC NULLS FIRST, 100), ENSURE_REQUIREMENTS, [plan_id=62] -(414) Sort +(415) Sort Input [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] Arguments: [ca_county#8 ASC NULLS FIRST], true, 0 -(415) AdaptiveSparkPlan +(416) AdaptiveSparkPlan Output [6]: [ca_county#8, d_year#5, web_q1_q2_increase#70, store_q1_q2_increase#71, web_q2_q3_increase#72, store_q2_q3_increase#73] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q32.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q32.txt index dd041fa6d..c4a339073 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q32.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q32.txt @@ -14,9 +14,9 @@ AdaptiveSparkPlan (112) : +- AQEShuffleRead (54) : +- ShuffleQueryStage (53), Statistics(X) : +- NativeShuffleExchange (52) - : +- ConvertToNative (51) - : +- * Project (50) - : +- * SortMergeJoin Inner (49) + : +- NativeProject (51) + : +- NativeFilter (50) + : +- NativeSortMergeJoin Inner (49) : :- NativeProject (19) : : +- NativeSortMergeJoin Inner (18) : : :- NativeSort (8) @@ -324,18 +324,19 @@ Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#17) Input [2]: [(1.3 * avg(cs_ext_discount_amt))#17, cs_item_sk#7] Arguments: [cs_item_sk#7 ASC NULLS FIRST], false -(49) SortMergeJoin [codegen id : X] +(49) NativeSortMergeJoin Left keys [1]: [i_item_sk#4] Right keys [1]: [cs_item_sk#7] Join type: Inner -Join condition: (cast(cs_ext_discount_amt#3 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#17) +Join condition: None -(50) Project [codegen id : X] -Output [2]: [cs_sold_date_sk#1, cs_ext_discount_amt#3] +(50) NativeFilter Input [5]: [cs_sold_date_sk#1, cs_ext_discount_amt#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#17, cs_item_sk#7] +Condition : (cast(cs_ext_discount_amt#3 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#17) -(51) ConvertToNative -Input [2]: [cs_sold_date_sk#1, cs_ext_discount_amt#3] +(51) NativeProject +Output [2]: [cs_sold_date_sk#1, cs_ext_discount_amt#3] +Input [5]: [cs_sold_date_sk#1, cs_ext_discount_amt#3, i_item_sk#4, (1.3 * avg(cs_ext_discount_amt))#17, cs_item_sk#7] (52) NativeShuffleExchange Input [2]: [cs_sold_date_sk#1, cs_ext_discount_amt#3] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q4.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q4.txt index 2295ee84a..adb4c1b83 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q4.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q4.txt @@ -1,388 +1,389 @@ == Physical Plan == -AdaptiveSparkPlan (379) +AdaptiveSparkPlan (380) +- == Final Plan == - TakeOrderedAndProject (227) - +- * Project (226) - +- * SortMergeJoin Inner (225) - :- NativeProject (197) - : +- NativeSortMergeJoin Inner (196) - : :- ConvertToNative (152) - : : +- * Project (151) - : : +- * SortMergeJoin Inner (150) - : : :- NativeProject (122) - : : : +- NativeSortMergeJoin Inner (121) - : : : :- NativeSortMergeJoin Inner (77) - : : : : :- NativeSort (47) - : : : : : +- InputAdapter (46) - : : : : : +- AQEShuffleRead (45) - : : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : : +- NativeShuffleExchange (43) - : : : : : +- NativeFilter (42) - : : : : : +- NativeProject (41) - : : : : : +- NativeHashAggregate (40) - : : : : : +- InputAdapter (39) - : : : : : +- AQEShuffleRead (38) - : : : : : +- ShuffleQueryStage (37), Statistics(X) - : : : : : +- NativeShuffleExchange (36) - : : : : : +- NativeHashAggregate (35) - : : : : : +- NativeProject (34) - : : : : : +- NativeProject (33) - : : : : : +- NativeSortMergeJoin Inner (32) - : : : : : :- NativeSort (23) - : : : : : : +- InputAdapter (22) - : : : : : : +- AQEShuffleRead (21) - : : : : : : +- ShuffleQueryStage (20), Statistics(X) - : : : : : : +- NativeShuffleExchange (19) - : : : : : : +- NativeProject (18) - : : : : : : +- NativeSortMergeJoin Inner (17) - : : : : : : :- NativeSort (8) - : : : : : : : +- InputAdapter (7) - : : : : : : : +- AQEShuffleRead (6) - : : : : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : : : : +- NativeShuffleExchange (4) - : : : : : : : +- NativeFilter (3) - : : : : : : : +- InputAdapter (2) - : : : : : : : +- NativeParquetScan (1) - : : : : : : +- NativeSort (16) - : : : : : : +- InputAdapter (15) - : : : : : : +- AQEShuffleRead (14) - : : : : : : +- ShuffleQueryStage (13), Statistics(X) - : : : : : : +- NativeShuffleExchange (12) - : : : : : : +- NativeFilter (11) - : : : : : : +- InputAdapter (10) - : : : : : : +- NativeParquetScan (9) - : : : : : +- NativeSort (31) - : : : : : +- InputAdapter (30) - : : : : : +- AQEShuffleRead (29) - : : : : : +- ShuffleQueryStage (28), Statistics(X) - : : : : : +- NativeShuffleExchange (27) - : : : : : +- NativeFilter (26) - : : : : : +- InputAdapter (25) - : : : : : +- NativeParquetScan (24) - : : : : +- NativeSort (76) - : : : : +- InputAdapter (75) - : : : : +- AQEShuffleRead (74) - : : : : +- ShuffleQueryStage (73), Statistics(X) - : : : : +- NativeShuffleExchange (72) - : : : : +- NativeProject (71) - : : : : +- NativeHashAggregate (70) - : : : : +- InputAdapter (69) - : : : : +- AQEShuffleRead (68) - : : : : +- ShuffleQueryStage (67), Statistics(X) - : : : : +- NativeShuffleExchange (66) - : : : : +- NativeHashAggregate (65) - : : : : +- NativeProject (64) - : : : : +- NativeProject (63) - : : : : +- NativeSortMergeJoin Inner (62) - : : : : :- NativeSort (53) - : : : : : +- InputAdapter (52) - : : : : : +- InputAdapter (51) - : : : : : +- AQEShuffleRead (50) - : : : : : +- ShuffleQueryStage (49), Statistics(X) - : : : : : +- ReusedExchange (48) - : : : : +- NativeSort (61) - : : : : +- InputAdapter (60) - : : : : +- AQEShuffleRead (59) - : : : : +- ShuffleQueryStage (58), Statistics(X) - : : : : +- NativeShuffleExchange (57) - : : : : +- NativeFilter (56) - : : : : +- InputAdapter (55) - : : : : +- NativeParquetScan (54) - : : : +- NativeSort (120) - : : : +- InputAdapter (119) - : : : +- AQEShuffleRead (118) - : : : +- ShuffleQueryStage (117), Statistics(X) - : : : +- NativeShuffleExchange (116) - : : : +- NativeFilter (115) - : : : +- NativeProject (114) - : : : +- NativeHashAggregate (113) - : : : +- InputAdapter (112) - : : : +- AQEShuffleRead (111) - : : : +- ShuffleQueryStage (110), Statistics(X) - : : : +- NativeShuffleExchange (109) - : : : +- NativeHashAggregate (108) - : : : +- NativeProject (107) - : : : +- NativeProject (106) - : : : +- NativeSortMergeJoin Inner (105) - : : : :- NativeSort (98) - : : : : +- InputAdapter (97) - : : : : +- AQEShuffleRead (96) - : : : : +- ShuffleQueryStage (95), Statistics(X) - : : : : +- NativeShuffleExchange (94) - : : : : +- NativeProject (93) - : : : : +- NativeSortMergeJoin Inner (92) - : : : : :- NativeSort (83) - : : : : : +- InputAdapter (82) - : : : : : +- InputAdapter (81) - : : : : : +- AQEShuffleRead (80) - : : : : : +- ShuffleQueryStage (79), Statistics(X) - : : : : : +- ReusedExchange (78) - : : : : +- NativeSort (91) - : : : : +- InputAdapter (90) - : : : : +- AQEShuffleRead (89) - : : : : +- ShuffleQueryStage (88), Statistics(X) - : : : : +- NativeShuffleExchange (87) - : : : : +- NativeFilter (86) - : : : : +- InputAdapter (85) - : : : : +- NativeParquetScan (84) - : : : +- NativeSort (104) - : : : +- InputAdapter (103) - : : : +- InputAdapter (102) - : : : +- AQEShuffleRead (101) - : : : +- ShuffleQueryStage (100), Statistics(X) - : : : +- ReusedExchange (99) - : : +- NativeSort (149) - : : +- InputAdapter (148) - : : +- AQEShuffleRead (147) - : : +- ShuffleQueryStage (146), Statistics(X) - : : +- NativeShuffleExchange (145) - : : +- NativeProject (144) - : : +- NativeHashAggregate (143) - : : +- InputAdapter (142) - : : +- AQEShuffleRead (141) - : : +- ShuffleQueryStage (140), Statistics(X) - : : +- NativeShuffleExchange (139) - : : +- NativeHashAggregate (138) - : : +- NativeProject (137) - : : +- NativeProject (136) - : : +- NativeSortMergeJoin Inner (135) - : : :- NativeSort (128) - : : : +- InputAdapter (127) - : : : +- InputAdapter (126) - : : : +- AQEShuffleRead (125) - : : : +- ShuffleQueryStage (124), Statistics(X) - : : : +- ReusedExchange (123) - : : +- NativeSort (134) - : : +- InputAdapter (133) - : : +- InputAdapter (132) - : : +- AQEShuffleRead (131) - : : +- ShuffleQueryStage (130), Statistics(X) - : : +- ReusedExchange (129) - : +- NativeSort (195) - : +- InputAdapter (194) - : +- AQEShuffleRead (193) - : +- ShuffleQueryStage (192), Statistics(X) - : +- NativeShuffleExchange (191) - : +- NativeFilter (190) - : +- NativeProject (189) - : +- NativeHashAggregate (188) - : +- InputAdapter (187) - : +- AQEShuffleRead (186) - : +- ShuffleQueryStage (185), Statistics(X) - : +- NativeShuffleExchange (184) - : +- NativeHashAggregate (183) - : +- NativeProject (182) - : +- NativeProject (181) - : +- NativeSortMergeJoin Inner (180) - : :- NativeSort (173) - : : +- InputAdapter (172) - : : +- AQEShuffleRead (171) - : : +- ShuffleQueryStage (170), Statistics(X) - : : +- NativeShuffleExchange (169) - : : +- NativeProject (168) - : : +- NativeSortMergeJoin Inner (167) - : : :- NativeSort (158) - : : : +- InputAdapter (157) - : : : +- InputAdapter (156) - : : : +- AQEShuffleRead (155) - : : : +- ShuffleQueryStage (154), Statistics(X) - : : : +- ReusedExchange (153) - : : +- NativeSort (166) - : : +- InputAdapter (165) - : : +- AQEShuffleRead (164) - : : +- ShuffleQueryStage (163), Statistics(X) - : : +- NativeShuffleExchange (162) - : : +- NativeFilter (161) - : : +- InputAdapter (160) - : : +- NativeParquetScan (159) - : +- NativeSort (179) - : +- InputAdapter (178) - : +- InputAdapter (177) - : +- AQEShuffleRead (176) - : +- ShuffleQueryStage (175), Statistics(X) - : +- ReusedExchange (174) - +- NativeSort (224) - +- InputAdapter (223) - +- AQEShuffleRead (222) - +- ShuffleQueryStage (221), Statistics(X) - +- NativeShuffleExchange (220) - +- NativeProject (219) - +- NativeHashAggregate (218) - +- InputAdapter (217) - +- AQEShuffleRead (216) - +- ShuffleQueryStage (215), Statistics(X) - +- NativeShuffleExchange (214) - +- NativeHashAggregate (213) - +- NativeProject (212) - +- NativeProject (211) - +- NativeSortMergeJoin Inner (210) - :- NativeSort (203) - : +- InputAdapter (202) - : +- InputAdapter (201) - : +- AQEShuffleRead (200) - : +- ShuffleQueryStage (199), Statistics(X) - : +- ReusedExchange (198) - +- NativeSort (209) - +- InputAdapter (208) - +- InputAdapter (207) - +- AQEShuffleRead (206) - +- ShuffleQueryStage (205), Statistics(X) - +- ReusedExchange (204) + NativeTakeOrdered (228) + +- NativeProject (227) + +- NativeFilter (226) + +- NativeSortMergeJoin Inner (225) + :- NativeProject (197) + : +- NativeSortMergeJoin Inner (196) + : :- NativeProject (152) + : : +- NativeFilter (151) + : : +- NativeSortMergeJoin Inner (150) + : : :- NativeProject (122) + : : : +- NativeSortMergeJoin Inner (121) + : : : :- NativeSortMergeJoin Inner (77) + : : : : :- NativeSort (47) + : : : : : +- InputAdapter (46) + : : : : : +- AQEShuffleRead (45) + : : : : : +- ShuffleQueryStage (44), Statistics(X) + : : : : : +- NativeShuffleExchange (43) + : : : : : +- NativeFilter (42) + : : : : : +- NativeProject (41) + : : : : : +- NativeHashAggregate (40) + : : : : : +- InputAdapter (39) + : : : : : +- AQEShuffleRead (38) + : : : : : +- ShuffleQueryStage (37), Statistics(X) + : : : : : +- NativeShuffleExchange (36) + : : : : : +- NativeHashAggregate (35) + : : : : : +- NativeProject (34) + : : : : : +- NativeProject (33) + : : : : : +- NativeSortMergeJoin Inner (32) + : : : : : :- NativeSort (23) + : : : : : : +- InputAdapter (22) + : : : : : : +- AQEShuffleRead (21) + : : : : : : +- ShuffleQueryStage (20), Statistics(X) + : : : : : : +- NativeShuffleExchange (19) + : : : : : : +- NativeProject (18) + : : : : : : +- NativeSortMergeJoin Inner (17) + : : : : : : :- NativeSort (8) + : : : : : : : +- InputAdapter (7) + : : : : : : : +- AQEShuffleRead (6) + : : : : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : : : : +- NativeShuffleExchange (4) + : : : : : : : +- NativeFilter (3) + : : : : : : : +- InputAdapter (2) + : : : : : : : +- NativeParquetScan (1) + : : : : : : +- NativeSort (16) + : : : : : : +- InputAdapter (15) + : : : : : : +- AQEShuffleRead (14) + : : : : : : +- ShuffleQueryStage (13), Statistics(X) + : : : : : : +- NativeShuffleExchange (12) + : : : : : : +- NativeFilter (11) + : : : : : : +- InputAdapter (10) + : : : : : : +- NativeParquetScan (9) + : : : : : +- NativeSort (31) + : : : : : +- InputAdapter (30) + : : : : : +- AQEShuffleRead (29) + : : : : : +- ShuffleQueryStage (28), Statistics(X) + : : : : : +- NativeShuffleExchange (27) + : : : : : +- NativeFilter (26) + : : : : : +- InputAdapter (25) + : : : : : +- NativeParquetScan (24) + : : : : +- NativeSort (76) + : : : : +- InputAdapter (75) + : : : : +- AQEShuffleRead (74) + : : : : +- ShuffleQueryStage (73), Statistics(X) + : : : : +- NativeShuffleExchange (72) + : : : : +- NativeProject (71) + : : : : +- NativeHashAggregate (70) + : : : : +- InputAdapter (69) + : : : : +- AQEShuffleRead (68) + : : : : +- ShuffleQueryStage (67), Statistics(X) + : : : : +- NativeShuffleExchange (66) + : : : : +- NativeHashAggregate (65) + : : : : +- NativeProject (64) + : : : : +- NativeProject (63) + : : : : +- NativeSortMergeJoin Inner (62) + : : : : :- NativeSort (53) + : : : : : +- InputAdapter (52) + : : : : : +- InputAdapter (51) + : : : : : +- AQEShuffleRead (50) + : : : : : +- ShuffleQueryStage (49), Statistics(X) + : : : : : +- ReusedExchange (48) + : : : : +- NativeSort (61) + : : : : +- InputAdapter (60) + : : : : +- AQEShuffleRead (59) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- NativeShuffleExchange (57) + : : : : +- NativeFilter (56) + : : : : +- InputAdapter (55) + : : : : +- NativeParquetScan (54) + : : : +- NativeSort (120) + : : : +- InputAdapter (119) + : : : +- AQEShuffleRead (118) + : : : +- ShuffleQueryStage (117), Statistics(X) + : : : +- NativeShuffleExchange (116) + : : : +- NativeFilter (115) + : : : +- NativeProject (114) + : : : +- NativeHashAggregate (113) + : : : +- InputAdapter (112) + : : : +- AQEShuffleRead (111) + : : : +- ShuffleQueryStage (110), Statistics(X) + : : : +- NativeShuffleExchange (109) + : : : +- NativeHashAggregate (108) + : : : +- NativeProject (107) + : : : +- NativeProject (106) + : : : +- NativeSortMergeJoin Inner (105) + : : : :- NativeSort (98) + : : : : +- InputAdapter (97) + : : : : +- AQEShuffleRead (96) + : : : : +- ShuffleQueryStage (95), Statistics(X) + : : : : +- NativeShuffleExchange (94) + : : : : +- NativeProject (93) + : : : : +- NativeSortMergeJoin Inner (92) + : : : : :- NativeSort (83) + : : : : : +- InputAdapter (82) + : : : : : +- InputAdapter (81) + : : : : : +- AQEShuffleRead (80) + : : : : : +- ShuffleQueryStage (79), Statistics(X) + : : : : : +- ReusedExchange (78) + : : : : +- NativeSort (91) + : : : : +- InputAdapter (90) + : : : : +- AQEShuffleRead (89) + : : : : +- ShuffleQueryStage (88), Statistics(X) + : : : : +- NativeShuffleExchange (87) + : : : : +- NativeFilter (86) + : : : : +- InputAdapter (85) + : : : : +- NativeParquetScan (84) + : : : +- NativeSort (104) + : : : +- InputAdapter (103) + : : : +- InputAdapter (102) + : : : +- AQEShuffleRead (101) + : : : +- ShuffleQueryStage (100), Statistics(X) + : : : +- ReusedExchange (99) + : : +- NativeSort (149) + : : +- InputAdapter (148) + : : +- AQEShuffleRead (147) + : : +- ShuffleQueryStage (146), Statistics(X) + : : +- NativeShuffleExchange (145) + : : +- NativeProject (144) + : : +- NativeHashAggregate (143) + : : +- InputAdapter (142) + : : +- AQEShuffleRead (141) + : : +- ShuffleQueryStage (140), Statistics(X) + : : +- NativeShuffleExchange (139) + : : +- NativeHashAggregate (138) + : : +- NativeProject (137) + : : +- NativeProject (136) + : : +- NativeSortMergeJoin Inner (135) + : : :- NativeSort (128) + : : : +- InputAdapter (127) + : : : +- InputAdapter (126) + : : : +- AQEShuffleRead (125) + : : : +- ShuffleQueryStage (124), Statistics(X) + : : : +- ReusedExchange (123) + : : +- NativeSort (134) + : : +- InputAdapter (133) + : : +- InputAdapter (132) + : : +- AQEShuffleRead (131) + : : +- ShuffleQueryStage (130), Statistics(X) + : : +- ReusedExchange (129) + : +- NativeSort (195) + : +- InputAdapter (194) + : +- AQEShuffleRead (193) + : +- ShuffleQueryStage (192), Statistics(X) + : +- NativeShuffleExchange (191) + : +- NativeFilter (190) + : +- NativeProject (189) + : +- NativeHashAggregate (188) + : +- InputAdapter (187) + : +- AQEShuffleRead (186) + : +- ShuffleQueryStage (185), Statistics(X) + : +- NativeShuffleExchange (184) + : +- NativeHashAggregate (183) + : +- NativeProject (182) + : +- NativeProject (181) + : +- NativeSortMergeJoin Inner (180) + : :- NativeSort (173) + : : +- InputAdapter (172) + : : +- AQEShuffleRead (171) + : : +- ShuffleQueryStage (170), Statistics(X) + : : +- NativeShuffleExchange (169) + : : +- NativeProject (168) + : : +- NativeSortMergeJoin Inner (167) + : : :- NativeSort (158) + : : : +- InputAdapter (157) + : : : +- InputAdapter (156) + : : : +- AQEShuffleRead (155) + : : : +- ShuffleQueryStage (154), Statistics(X) + : : : +- ReusedExchange (153) + : : +- NativeSort (166) + : : +- InputAdapter (165) + : : +- AQEShuffleRead (164) + : : +- ShuffleQueryStage (163), Statistics(X) + : : +- NativeShuffleExchange (162) + : : +- NativeFilter (161) + : : +- InputAdapter (160) + : : +- NativeParquetScan (159) + : +- NativeSort (179) + : +- InputAdapter (178) + : +- InputAdapter (177) + : +- AQEShuffleRead (176) + : +- ShuffleQueryStage (175), Statistics(X) + : +- ReusedExchange (174) + +- NativeSort (224) + +- InputAdapter (223) + +- AQEShuffleRead (222) + +- ShuffleQueryStage (221), Statistics(X) + +- NativeShuffleExchange (220) + +- NativeProject (219) + +- NativeHashAggregate (218) + +- InputAdapter (217) + +- AQEShuffleRead (216) + +- ShuffleQueryStage (215), Statistics(X) + +- NativeShuffleExchange (214) + +- NativeHashAggregate (213) + +- NativeProject (212) + +- NativeProject (211) + +- NativeSortMergeJoin Inner (210) + :- NativeSort (203) + : +- InputAdapter (202) + : +- InputAdapter (201) + : +- AQEShuffleRead (200) + : +- ShuffleQueryStage (199), Statistics(X) + : +- ReusedExchange (198) + +- NativeSort (209) + +- InputAdapter (208) + +- InputAdapter (207) + +- AQEShuffleRead (206) + +- ShuffleQueryStage (205), Statistics(X) + +- ReusedExchange (204) +- == Initial Plan == - TakeOrderedAndProject (378) - +- Project (377) - +- SortMergeJoin Inner (376) - :- Project (352) - : +- SortMergeJoin Inner (351) - : :- Project (326) - : : +- SortMergeJoin Inner (325) - : : :- Project (301) - : : : +- SortMergeJoin Inner (300) - : : : :- SortMergeJoin Inner (275) - : : : : :- Sort (251) - : : : : : +- Exchange (250) - : : : : : +- Filter (249) - : : : : : +- HashAggregate (248) - : : : : : +- Exchange (247) - : : : : : +- HashAggregate (246) - : : : : : +- Project (245) - : : : : : +- SortMergeJoin Inner (244) - : : : : : :- Sort (239) - : : : : : : +- Exchange (238) - : : : : : : +- Project (237) - : : : : : : +- SortMergeJoin Inner (236) - : : : : : : :- Sort (231) - : : : : : : : +- Exchange (230) - : : : : : : : +- Filter (229) - : : : : : : : +- Scan parquet (228) - : : : : : : +- Sort (235) - : : : : : : +- Exchange (234) - : : : : : : +- Filter (233) - : : : : : : +- Scan parquet (232) - : : : : : +- Sort (243) - : : : : : +- Exchange (242) - : : : : : +- Filter (241) - : : : : : +- Scan parquet (240) - : : : : +- Sort (274) - : : : : +- Exchange (273) - : : : : +- HashAggregate (272) - : : : : +- Exchange (271) - : : : : +- HashAggregate (270) - : : : : +- Project (269) - : : : : +- SortMergeJoin Inner (268) - : : : : :- Sort (263) - : : : : : +- Exchange (262) - : : : : : +- Project (261) - : : : : : +- SortMergeJoin Inner (260) - : : : : : :- Sort (255) - : : : : : : +- Exchange (254) - : : : : : : +- Filter (253) - : : : : : : +- Scan parquet (252) - : : : : : +- Sort (259) - : : : : : +- Exchange (258) - : : : : : +- Filter (257) - : : : : : +- Scan parquet (256) - : : : : +- Sort (267) - : : : : +- Exchange (266) - : : : : +- Filter (265) - : : : : +- Scan parquet (264) - : : : +- Sort (299) - : : : +- Exchange (298) - : : : +- Filter (297) - : : : +- HashAggregate (296) - : : : +- Exchange (295) - : : : +- HashAggregate (294) - : : : +- Project (293) - : : : +- SortMergeJoin Inner (292) - : : : :- Sort (287) - : : : : +- Exchange (286) - : : : : +- Project (285) - : : : : +- SortMergeJoin Inner (284) - : : : : :- Sort (279) - : : : : : +- Exchange (278) - : : : : : +- Filter (277) - : : : : : +- Scan parquet (276) - : : : : +- Sort (283) - : : : : +- Exchange (282) - : : : : +- Filter (281) - : : : : +- Scan parquet (280) - : : : +- Sort (291) - : : : +- Exchange (290) - : : : +- Filter (289) - : : : +- Scan parquet (288) - : : +- Sort (324) - : : +- Exchange (323) - : : +- HashAggregate (322) - : : +- Exchange (321) - : : +- HashAggregate (320) - : : +- Project (319) - : : +- SortMergeJoin Inner (318) - : : :- Sort (313) - : : : +- Exchange (312) - : : : +- Project (311) - : : : +- SortMergeJoin Inner (310) - : : : :- Sort (305) - : : : : +- Exchange (304) - : : : : +- Filter (303) - : : : : +- Scan parquet (302) - : : : +- Sort (309) - : : : +- Exchange (308) - : : : +- Filter (307) - : : : +- Scan parquet (306) - : : +- Sort (317) - : : +- Exchange (316) - : : +- Filter (315) - : : +- Scan parquet (314) - : +- Sort (350) - : +- Exchange (349) - : +- Filter (348) - : +- HashAggregate (347) - : +- Exchange (346) - : +- HashAggregate (345) - : +- Project (344) - : +- SortMergeJoin Inner (343) - : :- Sort (338) - : : +- Exchange (337) - : : +- Project (336) - : : +- SortMergeJoin Inner (335) - : : :- Sort (330) - : : : +- Exchange (329) - : : : +- Filter (328) - : : : +- Scan parquet (327) - : : +- Sort (334) - : : +- Exchange (333) - : : +- Filter (332) - : : +- Scan parquet (331) - : +- Sort (342) - : +- Exchange (341) - : +- Filter (340) - : +- Scan parquet (339) - +- Sort (375) - +- Exchange (374) - +- HashAggregate (373) - +- Exchange (372) - +- HashAggregate (371) - +- Project (370) - +- SortMergeJoin Inner (369) - :- Sort (364) - : +- Exchange (363) - : +- Project (362) - : +- SortMergeJoin Inner (361) - : :- Sort (356) - : : +- Exchange (355) - : : +- Filter (354) - : : +- Scan parquet (353) - : +- Sort (360) - : +- Exchange (359) - : +- Filter (358) - : +- Scan parquet (357) - +- Sort (368) - +- Exchange (367) - +- Filter (366) - +- Scan parquet (365) - - -(228) Scan parquet + TakeOrderedAndProject (379) + +- Project (378) + +- SortMergeJoin Inner (377) + :- Project (353) + : +- SortMergeJoin Inner (352) + : :- Project (327) + : : +- SortMergeJoin Inner (326) + : : :- Project (302) + : : : +- SortMergeJoin Inner (301) + : : : :- SortMergeJoin Inner (276) + : : : : :- Sort (252) + : : : : : +- Exchange (251) + : : : : : +- Filter (250) + : : : : : +- HashAggregate (249) + : : : : : +- Exchange (248) + : : : : : +- HashAggregate (247) + : : : : : +- Project (246) + : : : : : +- SortMergeJoin Inner (245) + : : : : : :- Sort (240) + : : : : : : +- Exchange (239) + : : : : : : +- Project (238) + : : : : : : +- SortMergeJoin Inner (237) + : : : : : : :- Sort (232) + : : : : : : : +- Exchange (231) + : : : : : : : +- Filter (230) + : : : : : : : +- Scan parquet (229) + : : : : : : +- Sort (236) + : : : : : : +- Exchange (235) + : : : : : : +- Filter (234) + : : : : : : +- Scan parquet (233) + : : : : : +- Sort (244) + : : : : : +- Exchange (243) + : : : : : +- Filter (242) + : : : : : +- Scan parquet (241) + : : : : +- Sort (275) + : : : : +- Exchange (274) + : : : : +- HashAggregate (273) + : : : : +- Exchange (272) + : : : : +- HashAggregate (271) + : : : : +- Project (270) + : : : : +- SortMergeJoin Inner (269) + : : : : :- Sort (264) + : : : : : +- Exchange (263) + : : : : : +- Project (262) + : : : : : +- SortMergeJoin Inner (261) + : : : : : :- Sort (256) + : : : : : : +- Exchange (255) + : : : : : : +- Filter (254) + : : : : : : +- Scan parquet (253) + : : : : : +- Sort (260) + : : : : : +- Exchange (259) + : : : : : +- Filter (258) + : : : : : +- Scan parquet (257) + : : : : +- Sort (268) + : : : : +- Exchange (267) + : : : : +- Filter (266) + : : : : +- Scan parquet (265) + : : : +- Sort (300) + : : : +- Exchange (299) + : : : +- Filter (298) + : : : +- HashAggregate (297) + : : : +- Exchange (296) + : : : +- HashAggregate (295) + : : : +- Project (294) + : : : +- SortMergeJoin Inner (293) + : : : :- Sort (288) + : : : : +- Exchange (287) + : : : : +- Project (286) + : : : : +- SortMergeJoin Inner (285) + : : : : :- Sort (280) + : : : : : +- Exchange (279) + : : : : : +- Filter (278) + : : : : : +- Scan parquet (277) + : : : : +- Sort (284) + : : : : +- Exchange (283) + : : : : +- Filter (282) + : : : : +- Scan parquet (281) + : : : +- Sort (292) + : : : +- Exchange (291) + : : : +- Filter (290) + : : : +- Scan parquet (289) + : : +- Sort (325) + : : +- Exchange (324) + : : +- HashAggregate (323) + : : +- Exchange (322) + : : +- HashAggregate (321) + : : +- Project (320) + : : +- SortMergeJoin Inner (319) + : : :- Sort (314) + : : : +- Exchange (313) + : : : +- Project (312) + : : : +- SortMergeJoin Inner (311) + : : : :- Sort (306) + : : : : +- Exchange (305) + : : : : +- Filter (304) + : : : : +- Scan parquet (303) + : : : +- Sort (310) + : : : +- Exchange (309) + : : : +- Filter (308) + : : : +- Scan parquet (307) + : : +- Sort (318) + : : +- Exchange (317) + : : +- Filter (316) + : : +- Scan parquet (315) + : +- Sort (351) + : +- Exchange (350) + : +- Filter (349) + : +- HashAggregate (348) + : +- Exchange (347) + : +- HashAggregate (346) + : +- Project (345) + : +- SortMergeJoin Inner (344) + : :- Sort (339) + : : +- Exchange (338) + : : +- Project (337) + : : +- SortMergeJoin Inner (336) + : : :- Sort (331) + : : : +- Exchange (330) + : : : +- Filter (329) + : : : +- Scan parquet (328) + : : +- Sort (335) + : : +- Exchange (334) + : : +- Filter (333) + : : +- Scan parquet (332) + : +- Sort (343) + : +- Exchange (342) + : +- Filter (341) + : +- Scan parquet (340) + +- Sort (376) + +- Exchange (375) + +- HashAggregate (374) + +- Exchange (373) + +- HashAggregate (372) + +- Project (371) + +- SortMergeJoin Inner (370) + :- Sort (365) + : +- Exchange (364) + : +- Project (363) + : +- SortMergeJoin Inner (362) + : :- Sort (357) + : : +- Exchange (356) + : : +- Filter (355) + : : +- Scan parquet (354) + : +- Sort (361) + : +- Exchange (360) + : +- Filter (359) + : +- Scan parquet (358) + +- Sort (369) + +- Exchange (368) + +- Filter (367) + +- Scan parquet (366) + + +(229) Scan parquet Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location: InMemoryFileIndex [file:/] @@ -416,7 +417,7 @@ Input [8]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8] Input [8]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8] Arguments: [c_customer_sk#1 ASC NULLS FIRST], false -(232) Scan parquet +(233) Scan parquet Output [6]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Batched: true Location: InMemoryFileIndex [file:/] @@ -479,7 +480,7 @@ Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_fl Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Arguments: [ss_sold_date_sk#9 ASC NULLS FIRST], false -(240) Scan parquet +(241) Scan parquet Output [2]: [d_date_sk#15, d_year#16] Batched: true Location: InMemoryFileIndex [file:/] @@ -605,7 +606,7 @@ Input [12]: [#24#24, #25#25, #26#26, #27#27, #28#28, #29#29, #30#30, #31#31, #32 Input [12]: [#24#24, #25#25, #26#26, #27#27, #28#28, #29#29, #30#30, #31#31, #32#32, #33#33, #34#34, #35#35] Arguments: [ss_sold_date_sk#31 ASC NULLS FIRST], false -(264) Scan parquet +(265) Scan parquet Output [2]: [d_date_sk#36, d_year#37] Batched: true Location: InMemoryFileIndex [file:/] @@ -733,7 +734,7 @@ Input [8]: [#49#49, #50#50, #51#51, #52#52, #53#53, #54#54, #55#55, #56#56] Input [8]: [#49#49, #50#50, #51#51, #52#52, #53#53, #54#54, #55#55, #56#56] Arguments: [c_customer_sk#49 ASC NULLS FIRST], false -(280) Scan parquet +(281) Scan parquet Output [6]: [cs_sold_date_sk#57, cs_bill_customer_sk#58, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] Batched: true Location: InMemoryFileIndex [file:/] @@ -1008,18 +1009,19 @@ Input [2]: [customer_id#88, year_total#89] Input [2]: [customer_id#88, year_total#89] Arguments: [customer_id#88 ASC NULLS FIRST], false -(150) SortMergeJoin [codegen id : X] +(150) NativeSortMergeJoin Left keys [1]: [customer_id#22] Right keys [1]: [customer_id#88] Join type: Inner -Join condition: (CASE WHEN (year_total#70 > 0.000000) THEN (year_total#89 / year_total#70) END > CASE WHEN (year_total#23 > 0.000000) THEN (year_total#48 / year_total#23) END) +Join condition: None -(151) Project [codegen id : X] -Output [10]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89] +(151) NativeFilter Input [13]: [customer_id#22, year_total#23, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#48, year_total#70, customer_id#88, year_total#89] +Condition : (CASE WHEN (year_total#70 > 0.000000) THEN (year_total#89 / year_total#70) END > CASE WHEN (year_total#23 > 0.000000) THEN (year_total#48 / year_total#23) END) -(152) ConvertToNative -Input [10]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89] +(152) NativeProject +Output [10]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89] +Input [13]: [customer_id#22, year_total#23, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#48, year_total#70, customer_id#88, year_total#89] (153) ReusedExchange [Reuses operator id: 4] Output [8]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97] @@ -1043,7 +1045,7 @@ Input [8]: [#90#90, #91#91, #92#92, #93#93, #94#94, #95#95, #96#96, #97#97] Input [8]: [#90#90, #91#91, #92#92, #93#93, #94#94, #95#95, #96#96, #97#97] Arguments: [c_customer_sk#90 ASC NULLS FIRST], false -(331) Scan parquet +(332) Scan parquet Output [6]: [ws_sold_date_sk#98, ws_bill_customer_sk#99, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] Batched: true Location: InMemoryFileIndex [file:/] @@ -1318,749 +1320,753 @@ Input [2]: [customer_id#129, year_total#130] Input [2]: [customer_id#129, year_total#130] Arguments: [customer_id#129 ASC NULLS FIRST], false -(225) SortMergeJoin [codegen id : X] +(225) NativeSortMergeJoin Left keys [1]: [customer_id#22] Right keys [1]: [customer_id#129] Join type: Inner -Join condition: (CASE WHEN (year_total#70 > 0.000000) THEN (year_total#89 / year_total#70) END > CASE WHEN (year_total#111 > 0.000000) THEN (year_total#130 / year_total#111) END) +Join condition: None -(226) Project [codegen id : X] +(226) NativeFilter +Input [13]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89, year_total#111, customer_id#129, year_total#130] +Condition : (CASE WHEN (year_total#70 > 0.000000) THEN (year_total#89 / year_total#70) END > CASE WHEN (year_total#111 > 0.000000) THEN (year_total#130 / year_total#111) END) + +(227) NativeProject Output [7]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47] Input [13]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89, year_total#111, customer_id#129, year_total#130] -(227) TakeOrderedAndProject +(228) NativeTakeOrdered Input [7]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47] -Arguments: X, [customer_id#41 ASC NULLS FIRST, customer_first_name#42 ASC NULLS FIRST, customer_last_name#43 ASC NULLS FIRST, customer_preferred_cust_flag#44 ASC NULLS FIRST, customer_birth_country#45 ASC NULLS FIRST, customer_login#46 ASC NULLS FIRST, customer_email_address#47 ASC NULLS FIRST], [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47] +Arguments: X, X, [customer_id#41 ASC NULLS FIRST, customer_first_name#42 ASC NULLS FIRST, customer_last_name#43 ASC NULLS FIRST, customer_preferred_cust_flag#44 ASC NULLS FIRST, customer_birth_country#45 ASC NULLS FIRST, customer_login#46 ASC NULLS FIRST, customer_email_address#47 ASC NULLS FIRST] -(228) Scan parquet +(229) Scan parquet Output [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(229) Filter +(230) Filter Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) -(230) Exchange +(231) Exchange Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: hashpartitioning(c_customer_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(231) Sort +(232) Sort Input [8]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0 -(232) Scan parquet +(233) Scan parquet Output [6]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(233) Filter +(234) Filter Input [6]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Condition : (isnotnull(ss_customer_sk#10) AND isnotnull(ss_sold_date_sk#9)) -(234) Exchange +(235) Exchange Input [6]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Arguments: hashpartitioning(ss_customer_sk#10, 100), ENSURE_REQUIREMENTS, [plan_id=23] -(235) Sort +(236) Sort Input [6]: [ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Arguments: [ss_customer_sk#10 ASC NULLS FIRST], false, 0 -(236) SortMergeJoin +(237) SortMergeJoin Left keys [1]: [c_customer_sk#1] Right keys [1]: [ss_customer_sk#10] Join type: Inner Join condition: None -(237) Project +(238) Project Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Input [14]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_customer_sk#10, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] -(238) Exchange +(239) Exchange Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Arguments: hashpartitioning(ss_sold_date_sk#9, 100), ENSURE_REQUIREMENTS, [plan_id=24] -(239) Sort +(240) Sort Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14] Arguments: [ss_sold_date_sk#9 ASC NULLS FIRST], false, 0 -(240) Scan parquet +(241) Scan parquet Output [2]: [d_date_sk#15, d_year#16] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(241) Filter +(242) Filter Input [2]: [d_date_sk#15, d_year#16] Condition : ((isnotnull(d_year#16) AND (d_year#16 = 2001)) AND isnotnull(d_date_sk#15)) -(242) Exchange +(243) Exchange Input [2]: [d_date_sk#15, d_year#16] Arguments: hashpartitioning(d_date_sk#15, 100), ENSURE_REQUIREMENTS, [plan_id=25] -(243) Sort +(244) Sort Input [2]: [d_date_sk#15, d_year#16] Arguments: [d_date_sk#15 ASC NULLS FIRST], false, 0 -(244) SortMergeJoin +(245) SortMergeJoin Left keys [1]: [ss_sold_date_sk#9] Right keys [1]: [d_date_sk#15] Join type: Inner Join condition: None -(245) Project +(246) Project Output [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14, d_year#16] Input [14]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_sold_date_sk#9, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14, d_date_sk#15, d_year#16] -(246) HashAggregate +(247) HashAggregate Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, ss_ext_discount_amt#11, ss_ext_sales_price#12, ss_ext_wholesale_cost#13, ss_ext_list_price#14, d_year#16] Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] Functions [1]: [partial_sum(((((ss_ext_list_price#14 - ss_ext_wholesale_cost#13) - ss_ext_discount_amt#11) + ss_ext_sales_price#12) / 2))] Aggregate Attributes [2]: [sum#18, isEmpty#19] Results [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#131, isEmpty#132] -(247) Exchange +(248) Exchange Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#131, isEmpty#132] Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, 100), ENSURE_REQUIREMENTS, [plan_id=26] -(248) HashAggregate +(249) HashAggregate Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16, sum#131, isEmpty#132] Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#16] Functions [1]: [sum(((((ss_ext_list_price#14 - ss_ext_wholesale_cost#13) - ss_ext_discount_amt#11) + ss_ext_sales_price#12) / 2))] Aggregate Attributes [1]: [sum(((((ss_ext_list_price#14 - ss_ext_wholesale_cost#13) - ss_ext_discount_amt#11) + ss_ext_sales_price#12) / 2))#21] Results [2]: [c_customer_id#2 AS customer_id#22, sum(((((ss_ext_list_price#14 - ss_ext_wholesale_cost#13) - ss_ext_discount_amt#11) + ss_ext_sales_price#12) / 2))#21 AS year_total#23] -(249) Filter +(250) Filter Input [2]: [customer_id#22, year_total#23] Condition : (isnotnull(year_total#23) AND (year_total#23 > 0.000000)) -(250) Exchange +(251) Exchange Input [2]: [customer_id#22, year_total#23] Arguments: hashpartitioning(customer_id#22, 100), ENSURE_REQUIREMENTS, [plan_id=27] -(251) Sort +(252) Sort Input [2]: [customer_id#22, year_total#23] Arguments: [customer_id#22 ASC NULLS FIRST], false, 0 -(252) Scan parquet +(253) Scan parquet Output [8]: [c_customer_sk#133, c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(253) Filter +(254) Filter Input [8]: [c_customer_sk#133, c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30] Condition : (isnotnull(c_customer_sk#133) AND isnotnull(c_customer_id#24)) -(254) Exchange +(255) Exchange Input [8]: [c_customer_sk#133, c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30] Arguments: hashpartitioning(c_customer_sk#133, 100), ENSURE_REQUIREMENTS, [plan_id=28] -(255) Sort +(256) Sort Input [8]: [c_customer_sk#133, c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30] Arguments: [c_customer_sk#133 ASC NULLS FIRST], false, 0 -(256) Scan parquet +(257) Scan parquet Output [6]: [ss_sold_date_sk#31, ss_customer_sk#134, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(257) Filter +(258) Filter Input [6]: [ss_sold_date_sk#31, ss_customer_sk#134, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35] Condition : (isnotnull(ss_customer_sk#134) AND isnotnull(ss_sold_date_sk#31)) -(258) Exchange +(259) Exchange Input [6]: [ss_sold_date_sk#31, ss_customer_sk#134, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35] Arguments: hashpartitioning(ss_customer_sk#134, 100), ENSURE_REQUIREMENTS, [plan_id=29] -(259) Sort +(260) Sort Input [6]: [ss_sold_date_sk#31, ss_customer_sk#134, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35] Arguments: [ss_customer_sk#134 ASC NULLS FIRST], false, 0 -(260) SortMergeJoin +(261) SortMergeJoin Left keys [1]: [c_customer_sk#133] Right keys [1]: [ss_customer_sk#134] Join type: Inner Join condition: None -(261) Project +(262) Project Output [12]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, ss_sold_date_sk#31, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35] Input [14]: [c_customer_sk#133, c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, ss_sold_date_sk#31, ss_customer_sk#134, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35] -(262) Exchange +(263) Exchange Input [12]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, ss_sold_date_sk#31, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35] Arguments: hashpartitioning(ss_sold_date_sk#31, 100), ENSURE_REQUIREMENTS, [plan_id=30] -(263) Sort +(264) Sort Input [12]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, ss_sold_date_sk#31, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35] Arguments: [ss_sold_date_sk#31 ASC NULLS FIRST], false, 0 -(264) Scan parquet +(265) Scan parquet Output [2]: [d_date_sk#36, d_year#37] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(265) Filter +(266) Filter Input [2]: [d_date_sk#36, d_year#37] Condition : ((isnotnull(d_year#37) AND (d_year#37 = 2002)) AND isnotnull(d_date_sk#36)) -(266) Exchange +(267) Exchange Input [2]: [d_date_sk#36, d_year#37] Arguments: hashpartitioning(d_date_sk#36, 100), ENSURE_REQUIREMENTS, [plan_id=31] -(267) Sort +(268) Sort Input [2]: [d_date_sk#36, d_year#37] Arguments: [d_date_sk#36 ASC NULLS FIRST], false, 0 -(268) SortMergeJoin +(269) SortMergeJoin Left keys [1]: [ss_sold_date_sk#31] Right keys [1]: [d_date_sk#36] Join type: Inner Join condition: None -(269) Project +(270) Project Output [12]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35, d_year#37] Input [14]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, ss_sold_date_sk#31, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35, d_date_sk#36, d_year#37] -(270) HashAggregate +(271) HashAggregate Input [12]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, ss_ext_discount_amt#32, ss_ext_sales_price#33, ss_ext_wholesale_cost#34, ss_ext_list_price#35, d_year#37] Keys [8]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, d_year#37] Functions [1]: [partial_sum(((((ss_ext_list_price#35 - ss_ext_wholesale_cost#34) - ss_ext_discount_amt#32) + ss_ext_sales_price#33) / 2))] Aggregate Attributes [2]: [sum#39, isEmpty#40] Results [10]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, d_year#37, sum#135, isEmpty#136] -(271) Exchange +(272) Exchange Input [10]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, d_year#37, sum#135, isEmpty#136] Arguments: hashpartitioning(c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, d_year#37, 100), ENSURE_REQUIREMENTS, [plan_id=32] -(272) HashAggregate +(273) HashAggregate Input [10]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, d_year#37, sum#135, isEmpty#136] Keys [8]: [c_customer_id#24, c_first_name#25, c_last_name#26, c_preferred_cust_flag#27, c_birth_country#28, c_login#29, c_email_address#30, d_year#37] Functions [1]: [sum(((((ss_ext_list_price#35 - ss_ext_wholesale_cost#34) - ss_ext_discount_amt#32) + ss_ext_sales_price#33) / 2))] Aggregate Attributes [1]: [sum(((((ss_ext_list_price#35 - ss_ext_wholesale_cost#34) - ss_ext_discount_amt#32) + ss_ext_sales_price#33) / 2))#21] Results [8]: [c_customer_id#24 AS customer_id#41, c_first_name#25 AS customer_first_name#42, c_last_name#26 AS customer_last_name#43, c_preferred_cust_flag#27 AS customer_preferred_cust_flag#44, c_birth_country#28 AS customer_birth_country#45, c_login#29 AS customer_login#46, c_email_address#30 AS customer_email_address#47, sum(((((ss_ext_list_price#35 - ss_ext_wholesale_cost#34) - ss_ext_discount_amt#32) + ss_ext_sales_price#33) / 2))#21 AS year_total#48] -(273) Exchange +(274) Exchange Input [8]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#48] Arguments: hashpartitioning(customer_id#41, 100), ENSURE_REQUIREMENTS, [plan_id=33] -(274) Sort +(275) Sort Input [8]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#48] Arguments: [customer_id#41 ASC NULLS FIRST], false, 0 -(275) SortMergeJoin +(276) SortMergeJoin Left keys [1]: [customer_id#22] Right keys [1]: [customer_id#41] Join type: Inner Join condition: None -(276) Scan parquet +(277) Scan parquet Output [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(277) Filter +(278) Filter Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] Condition : (isnotnull(c_customer_sk#49) AND isnotnull(c_customer_id#50)) -(278) Exchange +(279) Exchange Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] Arguments: hashpartitioning(c_customer_sk#49, 100), ENSURE_REQUIREMENTS, [plan_id=34] -(279) Sort +(280) Sort Input [8]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56] Arguments: [c_customer_sk#49 ASC NULLS FIRST], false, 0 -(280) Scan parquet +(281) Scan parquet Output [6]: [cs_sold_date_sk#57, cs_bill_customer_sk#58, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_sold_date_sk)] ReadSchema: struct -(281) Filter +(282) Filter Input [6]: [cs_sold_date_sk#57, cs_bill_customer_sk#58, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] Condition : (isnotnull(cs_bill_customer_sk#58) AND isnotnull(cs_sold_date_sk#57)) -(282) Exchange +(283) Exchange Input [6]: [cs_sold_date_sk#57, cs_bill_customer_sk#58, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] Arguments: hashpartitioning(cs_bill_customer_sk#58, 100), ENSURE_REQUIREMENTS, [plan_id=35] -(283) Sort +(284) Sort Input [6]: [cs_sold_date_sk#57, cs_bill_customer_sk#58, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] Arguments: [cs_bill_customer_sk#58 ASC NULLS FIRST], false, 0 -(284) SortMergeJoin +(285) SortMergeJoin Left keys [1]: [c_customer_sk#49] Right keys [1]: [cs_bill_customer_sk#58] Join type: Inner Join condition: None -(285) Project +(286) Project Output [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, cs_sold_date_sk#57, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] Input [14]: [c_customer_sk#49, c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, cs_sold_date_sk#57, cs_bill_customer_sk#58, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] -(286) Exchange +(287) Exchange Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, cs_sold_date_sk#57, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] Arguments: hashpartitioning(cs_sold_date_sk#57, 100), ENSURE_REQUIREMENTS, [plan_id=36] -(287) Sort +(288) Sort Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, cs_sold_date_sk#57, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62] Arguments: [cs_sold_date_sk#57 ASC NULLS FIRST], false, 0 -(288) Scan parquet +(289) Scan parquet Output [2]: [d_date_sk#63, d_year#64] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(289) Filter +(290) Filter Input [2]: [d_date_sk#63, d_year#64] Condition : ((isnotnull(d_year#64) AND (d_year#64 = 2001)) AND isnotnull(d_date_sk#63)) -(290) Exchange +(291) Exchange Input [2]: [d_date_sk#63, d_year#64] Arguments: hashpartitioning(d_date_sk#63, 100), ENSURE_REQUIREMENTS, [plan_id=37] -(291) Sort +(292) Sort Input [2]: [d_date_sk#63, d_year#64] Arguments: [d_date_sk#63 ASC NULLS FIRST], false, 0 -(292) SortMergeJoin +(293) SortMergeJoin Left keys [1]: [cs_sold_date_sk#57] Right keys [1]: [d_date_sk#63] Join type: Inner Join condition: None -(293) Project +(294) Project Output [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62, d_year#64] Input [14]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, cs_sold_date_sk#57, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62, d_date_sk#63, d_year#64] -(294) HashAggregate +(295) HashAggregate Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, cs_ext_discount_amt#59, cs_ext_sales_price#60, cs_ext_wholesale_cost#61, cs_ext_list_price#62, d_year#64] Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#64] Functions [1]: [partial_sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#61) - cs_ext_discount_amt#59) + cs_ext_sales_price#60) / 2))] Aggregate Attributes [2]: [sum#66, isEmpty#67] Results [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#64, sum#137, isEmpty#138] -(295) Exchange +(296) Exchange Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#64, sum#137, isEmpty#138] Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#64, 100), ENSURE_REQUIREMENTS, [plan_id=38] -(296) HashAggregate +(297) HashAggregate Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#64, sum#137, isEmpty#138] Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#64] Functions [1]: [sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#61) - cs_ext_discount_amt#59) + cs_ext_sales_price#60) / 2))] Aggregate Attributes [1]: [sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#61) - cs_ext_discount_amt#59) + cs_ext_sales_price#60) / 2))#68] Results [2]: [c_customer_id#50 AS customer_id#69, sum(((((cs_ext_list_price#62 - cs_ext_wholesale_cost#61) - cs_ext_discount_amt#59) + cs_ext_sales_price#60) / 2))#68 AS year_total#70] -(297) Filter +(298) Filter Input [2]: [customer_id#69, year_total#70] Condition : (isnotnull(year_total#70) AND (year_total#70 > 0.000000)) -(298) Exchange +(299) Exchange Input [2]: [customer_id#69, year_total#70] Arguments: hashpartitioning(customer_id#69, 100), ENSURE_REQUIREMENTS, [plan_id=39] -(299) Sort +(300) Sort Input [2]: [customer_id#69, year_total#70] Arguments: [customer_id#69 ASC NULLS FIRST], false, 0 -(300) SortMergeJoin +(301) SortMergeJoin Left keys [1]: [customer_id#22] Right keys [1]: [customer_id#69] Join type: Inner Join condition: None -(301) Project +(302) Project Output [11]: [customer_id#22, year_total#23, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#48, year_total#70] Input [12]: [customer_id#22, year_total#23, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#48, customer_id#69, year_total#70] -(302) Scan parquet +(303) Scan parquet Output [8]: [c_customer_sk#139, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(303) Filter +(304) Filter Input [8]: [c_customer_sk#139, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Condition : (isnotnull(c_customer_sk#139) AND isnotnull(c_customer_id#71)) -(304) Exchange +(305) Exchange Input [8]: [c_customer_sk#139, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Arguments: hashpartitioning(c_customer_sk#139, 100), ENSURE_REQUIREMENTS, [plan_id=40] -(305) Sort +(306) Sort Input [8]: [c_customer_sk#139, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77] Arguments: [c_customer_sk#139 ASC NULLS FIRST], false, 0 -(306) Scan parquet +(307) Scan parquet Output [6]: [cs_sold_date_sk#78, cs_bill_customer_sk#140, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cs_bill_customer_sk), IsNotNull(cs_sold_date_sk)] ReadSchema: struct -(307) Filter +(308) Filter Input [6]: [cs_sold_date_sk#78, cs_bill_customer_sk#140, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82] Condition : (isnotnull(cs_bill_customer_sk#140) AND isnotnull(cs_sold_date_sk#78)) -(308) Exchange +(309) Exchange Input [6]: [cs_sold_date_sk#78, cs_bill_customer_sk#140, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82] Arguments: hashpartitioning(cs_bill_customer_sk#140, 100), ENSURE_REQUIREMENTS, [plan_id=41] -(309) Sort +(310) Sort Input [6]: [cs_sold_date_sk#78, cs_bill_customer_sk#140, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82] Arguments: [cs_bill_customer_sk#140 ASC NULLS FIRST], false, 0 -(310) SortMergeJoin +(311) SortMergeJoin Left keys [1]: [c_customer_sk#139] Right keys [1]: [cs_bill_customer_sk#140] Join type: Inner Join condition: None -(311) Project +(312) Project Output [12]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, cs_sold_date_sk#78, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82] Input [14]: [c_customer_sk#139, c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, cs_sold_date_sk#78, cs_bill_customer_sk#140, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82] -(312) Exchange +(313) Exchange Input [12]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, cs_sold_date_sk#78, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82] Arguments: hashpartitioning(cs_sold_date_sk#78, 100), ENSURE_REQUIREMENTS, [plan_id=42] -(313) Sort +(314) Sort Input [12]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, cs_sold_date_sk#78, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82] Arguments: [cs_sold_date_sk#78 ASC NULLS FIRST], false, 0 -(314) Scan parquet +(315) Scan parquet Output [2]: [d_date_sk#83, d_year#84] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(315) Filter +(316) Filter Input [2]: [d_date_sk#83, d_year#84] Condition : ((isnotnull(d_year#84) AND (d_year#84 = 2002)) AND isnotnull(d_date_sk#83)) -(316) Exchange +(317) Exchange Input [2]: [d_date_sk#83, d_year#84] Arguments: hashpartitioning(d_date_sk#83, 100), ENSURE_REQUIREMENTS, [plan_id=43] -(317) Sort +(318) Sort Input [2]: [d_date_sk#83, d_year#84] Arguments: [d_date_sk#83 ASC NULLS FIRST], false, 0 -(318) SortMergeJoin +(319) SortMergeJoin Left keys [1]: [cs_sold_date_sk#78] Right keys [1]: [d_date_sk#83] Join type: Inner Join condition: None -(319) Project +(320) Project Output [12]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82, d_year#84] Input [14]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, cs_sold_date_sk#78, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82, d_date_sk#83, d_year#84] -(320) HashAggregate +(321) HashAggregate Input [12]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, cs_ext_discount_amt#79, cs_ext_sales_price#80, cs_ext_wholesale_cost#81, cs_ext_list_price#82, d_year#84] Keys [8]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#84] Functions [1]: [partial_sum(((((cs_ext_list_price#82 - cs_ext_wholesale_cost#81) - cs_ext_discount_amt#79) + cs_ext_sales_price#80) / 2))] Aggregate Attributes [2]: [sum#86, isEmpty#87] Results [10]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#84, sum#141, isEmpty#142] -(321) Exchange +(322) Exchange Input [10]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#84, sum#141, isEmpty#142] Arguments: hashpartitioning(c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#84, 100), ENSURE_REQUIREMENTS, [plan_id=44] -(322) HashAggregate +(323) HashAggregate Input [10]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#84, sum#141, isEmpty#142] Keys [8]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#84] Functions [1]: [sum(((((cs_ext_list_price#82 - cs_ext_wholesale_cost#81) - cs_ext_discount_amt#79) + cs_ext_sales_price#80) / 2))] Aggregate Attributes [1]: [sum(((((cs_ext_list_price#82 - cs_ext_wholesale_cost#81) - cs_ext_discount_amt#79) + cs_ext_sales_price#80) / 2))#68] Results [2]: [c_customer_id#71 AS customer_id#88, sum(((((cs_ext_list_price#82 - cs_ext_wholesale_cost#81) - cs_ext_discount_amt#79) + cs_ext_sales_price#80) / 2))#68 AS year_total#89] -(323) Exchange +(324) Exchange Input [2]: [customer_id#88, year_total#89] Arguments: hashpartitioning(customer_id#88, 100), ENSURE_REQUIREMENTS, [plan_id=45] -(324) Sort +(325) Sort Input [2]: [customer_id#88, year_total#89] Arguments: [customer_id#88 ASC NULLS FIRST], false, 0 -(325) SortMergeJoin +(326) SortMergeJoin Left keys [1]: [customer_id#22] Right keys [1]: [customer_id#88] Join type: Inner Join condition: (CASE WHEN (year_total#70 > 0.000000) THEN (year_total#89 / year_total#70) END > CASE WHEN (year_total#23 > 0.000000) THEN (year_total#48 / year_total#23) END) -(326) Project +(327) Project Output [10]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89] Input [13]: [customer_id#22, year_total#23, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#48, year_total#70, customer_id#88, year_total#89] -(327) Scan parquet +(328) Scan parquet Output [8]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(328) Filter +(329) Filter Input [8]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97] Condition : (isnotnull(c_customer_sk#90) AND isnotnull(c_customer_id#91)) -(329) Exchange +(330) Exchange Input [8]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97] Arguments: hashpartitioning(c_customer_sk#90, 100), ENSURE_REQUIREMENTS, [plan_id=46] -(330) Sort +(331) Sort Input [8]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97] Arguments: [c_customer_sk#90 ASC NULLS FIRST], false, 0 -(331) Scan parquet +(332) Scan parquet Output [6]: [ws_sold_date_sk#98, ws_bill_customer_sk#99, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(332) Filter +(333) Filter Input [6]: [ws_sold_date_sk#98, ws_bill_customer_sk#99, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] Condition : (isnotnull(ws_bill_customer_sk#99) AND isnotnull(ws_sold_date_sk#98)) -(333) Exchange +(334) Exchange Input [6]: [ws_sold_date_sk#98, ws_bill_customer_sk#99, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] Arguments: hashpartitioning(ws_bill_customer_sk#99, 100), ENSURE_REQUIREMENTS, [plan_id=47] -(334) Sort +(335) Sort Input [6]: [ws_sold_date_sk#98, ws_bill_customer_sk#99, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] Arguments: [ws_bill_customer_sk#99 ASC NULLS FIRST], false, 0 -(335) SortMergeJoin +(336) SortMergeJoin Left keys [1]: [c_customer_sk#90] Right keys [1]: [ws_bill_customer_sk#99] Join type: Inner Join condition: None -(336) Project +(337) Project Output [12]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, ws_sold_date_sk#98, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] Input [14]: [c_customer_sk#90, c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, ws_sold_date_sk#98, ws_bill_customer_sk#99, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] -(337) Exchange +(338) Exchange Input [12]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, ws_sold_date_sk#98, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] Arguments: hashpartitioning(ws_sold_date_sk#98, 100), ENSURE_REQUIREMENTS, [plan_id=48] -(338) Sort +(339) Sort Input [12]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, ws_sold_date_sk#98, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103] Arguments: [ws_sold_date_sk#98 ASC NULLS FIRST], false, 0 -(339) Scan parquet +(340) Scan parquet Output [2]: [d_date_sk#104, d_year#105] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(340) Filter +(341) Filter Input [2]: [d_date_sk#104, d_year#105] Condition : ((isnotnull(d_year#105) AND (d_year#105 = 2001)) AND isnotnull(d_date_sk#104)) -(341) Exchange +(342) Exchange Input [2]: [d_date_sk#104, d_year#105] Arguments: hashpartitioning(d_date_sk#104, 100), ENSURE_REQUIREMENTS, [plan_id=49] -(342) Sort +(343) Sort Input [2]: [d_date_sk#104, d_year#105] Arguments: [d_date_sk#104 ASC NULLS FIRST], false, 0 -(343) SortMergeJoin +(344) SortMergeJoin Left keys [1]: [ws_sold_date_sk#98] Right keys [1]: [d_date_sk#104] Join type: Inner Join condition: None -(344) Project +(345) Project Output [12]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103, d_year#105] Input [14]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, ws_sold_date_sk#98, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103, d_date_sk#104, d_year#105] -(345) HashAggregate +(346) HashAggregate Input [12]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, ws_ext_discount_amt#100, ws_ext_sales_price#101, ws_ext_wholesale_cost#102, ws_ext_list_price#103, d_year#105] Keys [8]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#105] Functions [1]: [partial_sum(((((ws_ext_list_price#103 - ws_ext_wholesale_cost#102) - ws_ext_discount_amt#100) + ws_ext_sales_price#101) / 2))] Aggregate Attributes [2]: [sum#107, isEmpty#108] Results [10]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#105, sum#143, isEmpty#144] -(346) Exchange +(347) Exchange Input [10]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#105, sum#143, isEmpty#144] Arguments: hashpartitioning(c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#105, 100), ENSURE_REQUIREMENTS, [plan_id=50] -(347) HashAggregate +(348) HashAggregate Input [10]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#105, sum#143, isEmpty#144] Keys [8]: [c_customer_id#91, c_first_name#92, c_last_name#93, c_preferred_cust_flag#94, c_birth_country#95, c_login#96, c_email_address#97, d_year#105] Functions [1]: [sum(((((ws_ext_list_price#103 - ws_ext_wholesale_cost#102) - ws_ext_discount_amt#100) + ws_ext_sales_price#101) / 2))] Aggregate Attributes [1]: [sum(((((ws_ext_list_price#103 - ws_ext_wholesale_cost#102) - ws_ext_discount_amt#100) + ws_ext_sales_price#101) / 2))#109] Results [2]: [c_customer_id#91 AS customer_id#110, sum(((((ws_ext_list_price#103 - ws_ext_wholesale_cost#102) - ws_ext_discount_amt#100) + ws_ext_sales_price#101) / 2))#109 AS year_total#111] -(348) Filter +(349) Filter Input [2]: [customer_id#110, year_total#111] Condition : (isnotnull(year_total#111) AND (year_total#111 > 0.000000)) -(349) Exchange +(350) Exchange Input [2]: [customer_id#110, year_total#111] Arguments: hashpartitioning(customer_id#110, 100), ENSURE_REQUIREMENTS, [plan_id=51] -(350) Sort +(351) Sort Input [2]: [customer_id#110, year_total#111] Arguments: [customer_id#110 ASC NULLS FIRST], false, 0 -(351) SortMergeJoin +(352) SortMergeJoin Left keys [1]: [customer_id#22] Right keys [1]: [customer_id#110] Join type: Inner Join condition: None -(352) Project +(353) Project Output [11]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89, year_total#111] Input [12]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89, customer_id#110, year_total#111] -(353) Scan parquet +(354) Scan parquet Output [8]: [c_customer_sk#145, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(354) Filter +(355) Filter Input [8]: [c_customer_sk#145, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] Condition : (isnotnull(c_customer_sk#145) AND isnotnull(c_customer_id#112)) -(355) Exchange +(356) Exchange Input [8]: [c_customer_sk#145, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] Arguments: hashpartitioning(c_customer_sk#145, 100), ENSURE_REQUIREMENTS, [plan_id=52] -(356) Sort +(357) Sort Input [8]: [c_customer_sk#145, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118] Arguments: [c_customer_sk#145 ASC NULLS FIRST], false, 0 -(357) Scan parquet +(358) Scan parquet Output [6]: [ws_sold_date_sk#119, ws_bill_customer_sk#146, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(358) Filter +(359) Filter Input [6]: [ws_sold_date_sk#119, ws_bill_customer_sk#146, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123] Condition : (isnotnull(ws_bill_customer_sk#146) AND isnotnull(ws_sold_date_sk#119)) -(359) Exchange +(360) Exchange Input [6]: [ws_sold_date_sk#119, ws_bill_customer_sk#146, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123] Arguments: hashpartitioning(ws_bill_customer_sk#146, 100), ENSURE_REQUIREMENTS, [plan_id=53] -(360) Sort +(361) Sort Input [6]: [ws_sold_date_sk#119, ws_bill_customer_sk#146, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123] Arguments: [ws_bill_customer_sk#146 ASC NULLS FIRST], false, 0 -(361) SortMergeJoin +(362) SortMergeJoin Left keys [1]: [c_customer_sk#145] Right keys [1]: [ws_bill_customer_sk#146] Join type: Inner Join condition: None -(362) Project +(363) Project Output [12]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, ws_sold_date_sk#119, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123] Input [14]: [c_customer_sk#145, c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, ws_sold_date_sk#119, ws_bill_customer_sk#146, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123] -(363) Exchange +(364) Exchange Input [12]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, ws_sold_date_sk#119, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123] Arguments: hashpartitioning(ws_sold_date_sk#119, 100), ENSURE_REQUIREMENTS, [plan_id=54] -(364) Sort +(365) Sort Input [12]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, ws_sold_date_sk#119, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123] Arguments: [ws_sold_date_sk#119 ASC NULLS FIRST], false, 0 -(365) Scan parquet +(366) Scan parquet Output [2]: [d_date_sk#124, d_year#125] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(366) Filter +(367) Filter Input [2]: [d_date_sk#124, d_year#125] Condition : ((isnotnull(d_year#125) AND (d_year#125 = 2002)) AND isnotnull(d_date_sk#124)) -(367) Exchange +(368) Exchange Input [2]: [d_date_sk#124, d_year#125] Arguments: hashpartitioning(d_date_sk#124, 100), ENSURE_REQUIREMENTS, [plan_id=55] -(368) Sort +(369) Sort Input [2]: [d_date_sk#124, d_year#125] Arguments: [d_date_sk#124 ASC NULLS FIRST], false, 0 -(369) SortMergeJoin +(370) SortMergeJoin Left keys [1]: [ws_sold_date_sk#119] Right keys [1]: [d_date_sk#124] Join type: Inner Join condition: None -(370) Project +(371) Project Output [12]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123, d_year#125] Input [14]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, ws_sold_date_sk#119, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123, d_date_sk#124, d_year#125] -(371) HashAggregate +(372) HashAggregate Input [12]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, ws_ext_discount_amt#120, ws_ext_sales_price#121, ws_ext_wholesale_cost#122, ws_ext_list_price#123, d_year#125] Keys [8]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, d_year#125] Functions [1]: [partial_sum(((((ws_ext_list_price#123 - ws_ext_wholesale_cost#122) - ws_ext_discount_amt#120) + ws_ext_sales_price#121) / 2))] Aggregate Attributes [2]: [sum#127, isEmpty#128] Results [10]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, d_year#125, sum#147, isEmpty#148] -(372) Exchange +(373) Exchange Input [10]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, d_year#125, sum#147, isEmpty#148] Arguments: hashpartitioning(c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, d_year#125, 100), ENSURE_REQUIREMENTS, [plan_id=56] -(373) HashAggregate +(374) HashAggregate Input [10]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, d_year#125, sum#147, isEmpty#148] Keys [8]: [c_customer_id#112, c_first_name#113, c_last_name#114, c_preferred_cust_flag#115, c_birth_country#116, c_login#117, c_email_address#118, d_year#125] Functions [1]: [sum(((((ws_ext_list_price#123 - ws_ext_wholesale_cost#122) - ws_ext_discount_amt#120) + ws_ext_sales_price#121) / 2))] Aggregate Attributes [1]: [sum(((((ws_ext_list_price#123 - ws_ext_wholesale_cost#122) - ws_ext_discount_amt#120) + ws_ext_sales_price#121) / 2))#109] Results [2]: [c_customer_id#112 AS customer_id#129, sum(((((ws_ext_list_price#123 - ws_ext_wholesale_cost#122) - ws_ext_discount_amt#120) + ws_ext_sales_price#121) / 2))#109 AS year_total#130] -(374) Exchange +(375) Exchange Input [2]: [customer_id#129, year_total#130] Arguments: hashpartitioning(customer_id#129, 100), ENSURE_REQUIREMENTS, [plan_id=57] -(375) Sort +(376) Sort Input [2]: [customer_id#129, year_total#130] Arguments: [customer_id#129 ASC NULLS FIRST], false, 0 -(376) SortMergeJoin +(377) SortMergeJoin Left keys [1]: [customer_id#22] Right keys [1]: [customer_id#129] Join type: Inner Join condition: (CASE WHEN (year_total#70 > 0.000000) THEN (year_total#89 / year_total#70) END > CASE WHEN (year_total#111 > 0.000000) THEN (year_total#130 / year_total#111) END) -(377) Project +(378) Project Output [7]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47] Input [13]: [customer_id#22, customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47, year_total#70, year_total#89, year_total#111, customer_id#129, year_total#130] -(378) TakeOrderedAndProject +(379) TakeOrderedAndProject Input [7]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47] Arguments: X, [customer_id#41 ASC NULLS FIRST, customer_first_name#42 ASC NULLS FIRST, customer_last_name#43 ASC NULLS FIRST, customer_preferred_cust_flag#44 ASC NULLS FIRST, customer_birth_country#45 ASC NULLS FIRST, customer_login#46 ASC NULLS FIRST, customer_email_address#47 ASC NULLS FIRST], [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47] -(379) AdaptiveSparkPlan +(380) AdaptiveSparkPlan Output [7]: [customer_id#41, customer_first_name#42, customer_last_name#43, customer_preferred_cust_flag#44, customer_birth_country#45, customer_login#46, customer_email_address#47] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q46.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q46.txt index 1966e185e..15800c342 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q46.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q46.txt @@ -1,165 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (156) +AdaptiveSparkPlan (157) +- == Final Plan == - TakeOrderedAndProject (99) - +- * Project (98) - +- * SortMergeJoin Inner (97) - :- NativeSort (90) - : +- InputAdapter (89) - : +- AQEShuffleRead (88) - : +- ShuffleQueryStage (87), Statistics(X) - : +- NativeShuffleExchange (86) - : +- NativeProject (85) - : +- NativeSortMergeJoin Inner (84) - : :- NativeSort (75) - : : +- InputAdapter (74) - : : +- AQEShuffleRead (73) - : : +- ShuffleQueryStage (72), Statistics(X) - : : +- NativeShuffleExchange (71) - : : +- NativeProject (70) - : : +- NativeHashAggregate (69) - : : +- NativeHashAggregate (68) - : : +- NativeProject (67) - : : +- NativeProject (66) - : : +- NativeSortMergeJoin Inner (65) - : : :- NativeSort (56) - : : : +- InputAdapter (55) - : : : +- AQEShuffleRead (54) - : : : +- ShuffleQueryStage (53), Statistics(X) - : : : +- NativeShuffleExchange (52) - : : : +- NativeProject (51) - : : : +- NativeSortMergeJoin Inner (50) - : : : :- NativeSort (40) - : : : : +- InputAdapter (39) - : : : : +- AQEShuffleRead (38) - : : : : +- ShuffleQueryStage (37), Statistics(X) - : : : : +- NativeShuffleExchange (36) - : : : : +- NativeProject (35) - : : : : +- NativeSortMergeJoin Inner (34) - : : : : :- NativeSort (24) - : : : : : +- InputAdapter (23) - : : : : : +- AQEShuffleRead (22) - : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : +- NativeShuffleExchange (20) - : : : : : +- NativeProject (19) - : : : : : +- NativeSortMergeJoin Inner (18) - : : : : : :- NativeSort (8) - : : : : : : +- InputAdapter (7) - : : : : : : +- AQEShuffleRead (6) - : : : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : : : +- NativeShuffleExchange (4) - : : : : : : +- NativeFilter (3) - : : : : : : +- InputAdapter (2) - : : : : : : +- NativeParquetScan (1) - : : : : : +- NativeSort (17) - : : : : : +- InputAdapter (16) - : : : : : +- AQEShuffleRead (15) - : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : +- NativeShuffleExchange (13) - : : : : : +- NativeProject (12) - : : : : : +- NativeFilter (11) - : : : : : +- InputAdapter (10) - : : : : : +- NativeParquetScan (9) - : : : : +- NativeSort (33) - : : : : +- InputAdapter (32) - : : : : +- AQEShuffleRead (31) - : : : : +- ShuffleQueryStage (30), Statistics(X) - : : : : +- NativeShuffleExchange (29) - : : : : +- NativeProject (28) - : : : : +- NativeFilter (27) - : : : : +- InputAdapter (26) - : : : : +- NativeParquetScan (25) - : : : +- NativeSort (49) - : : : +- InputAdapter (48) - : : : +- AQEShuffleRead (47) - : : : +- ShuffleQueryStage (46), Statistics(X) - : : : +- NativeShuffleExchange (45) - : : : +- NativeProject (44) - : : : +- NativeFilter (43) - : : : +- InputAdapter (42) - : : : +- NativeParquetScan (41) - : : +- NativeSort (64) - : : +- InputAdapter (63) - : : +- AQEShuffleRead (62) - : : +- ShuffleQueryStage (61), Statistics(X) - : : +- NativeShuffleExchange (60) - : : +- NativeFilter (59) - : : +- InputAdapter (58) - : : +- NativeParquetScan (57) - : +- NativeSort (83) - : +- InputAdapter (82) - : +- AQEShuffleRead (81) - : +- ShuffleQueryStage (80), Statistics(X) - : +- NativeShuffleExchange (79) - : +- NativeFilter (78) - : +- InputAdapter (77) - : +- NativeParquetScan (76) - +- NativeSort (96) - +- InputAdapter (95) - +- InputAdapter (94) - +- AQEShuffleRead (93) - +- ShuffleQueryStage (92), Statistics(X) - +- ReusedExchange (91) + NativeTakeOrdered (100) + +- NativeProject (99) + +- NativeFilter (98) + +- NativeSortMergeJoin Inner (97) + :- NativeSort (90) + : +- InputAdapter (89) + : +- AQEShuffleRead (88) + : +- ShuffleQueryStage (87), Statistics(X) + : +- NativeShuffleExchange (86) + : +- NativeProject (85) + : +- NativeSortMergeJoin Inner (84) + : :- NativeSort (75) + : : +- InputAdapter (74) + : : +- AQEShuffleRead (73) + : : +- ShuffleQueryStage (72), Statistics(X) + : : +- NativeShuffleExchange (71) + : : +- NativeProject (70) + : : +- NativeHashAggregate (69) + : : +- NativeHashAggregate (68) + : : +- NativeProject (67) + : : +- NativeProject (66) + : : +- NativeSortMergeJoin Inner (65) + : : :- NativeSort (56) + : : : +- InputAdapter (55) + : : : +- AQEShuffleRead (54) + : : : +- ShuffleQueryStage (53), Statistics(X) + : : : +- NativeShuffleExchange (52) + : : : +- NativeProject (51) + : : : +- NativeSortMergeJoin Inner (50) + : : : :- NativeSort (40) + : : : : +- InputAdapter (39) + : : : : +- AQEShuffleRead (38) + : : : : +- ShuffleQueryStage (37), Statistics(X) + : : : : +- NativeShuffleExchange (36) + : : : : +- NativeProject (35) + : : : : +- NativeSortMergeJoin Inner (34) + : : : : :- NativeSort (24) + : : : : : +- InputAdapter (23) + : : : : : +- AQEShuffleRead (22) + : : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : : +- NativeShuffleExchange (20) + : : : : : +- NativeProject (19) + : : : : : +- NativeSortMergeJoin Inner (18) + : : : : : :- NativeSort (8) + : : : : : : +- InputAdapter (7) + : : : : : : +- AQEShuffleRead (6) + : : : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : : : +- NativeShuffleExchange (4) + : : : : : : +- NativeFilter (3) + : : : : : : +- InputAdapter (2) + : : : : : : +- NativeParquetScan (1) + : : : : : +- NativeSort (17) + : : : : : +- InputAdapter (16) + : : : : : +- AQEShuffleRead (15) + : : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : : +- NativeShuffleExchange (13) + : : : : : +- NativeProject (12) + : : : : : +- NativeFilter (11) + : : : : : +- InputAdapter (10) + : : : : : +- NativeParquetScan (9) + : : : : +- NativeSort (33) + : : : : +- InputAdapter (32) + : : : : +- AQEShuffleRead (31) + : : : : +- ShuffleQueryStage (30), Statistics(X) + : : : : +- NativeShuffleExchange (29) + : : : : +- NativeProject (28) + : : : : +- NativeFilter (27) + : : : : +- InputAdapter (26) + : : : : +- NativeParquetScan (25) + : : : +- NativeSort (49) + : : : +- InputAdapter (48) + : : : +- AQEShuffleRead (47) + : : : +- ShuffleQueryStage (46), Statistics(X) + : : : +- NativeShuffleExchange (45) + : : : +- NativeProject (44) + : : : +- NativeFilter (43) + : : : +- InputAdapter (42) + : : : +- NativeParquetScan (41) + : : +- NativeSort (64) + : : +- InputAdapter (63) + : : +- AQEShuffleRead (62) + : : +- ShuffleQueryStage (61), Statistics(X) + : : +- NativeShuffleExchange (60) + : : +- NativeFilter (59) + : : +- InputAdapter (58) + : : +- NativeParquetScan (57) + : +- NativeSort (83) + : +- InputAdapter (82) + : +- AQEShuffleRead (81) + : +- ShuffleQueryStage (80), Statistics(X) + : +- NativeShuffleExchange (79) + : +- NativeFilter (78) + : +- InputAdapter (77) + : +- NativeParquetScan (76) + +- NativeSort (96) + +- InputAdapter (95) + +- InputAdapter (94) + +- AQEShuffleRead (93) + +- ShuffleQueryStage (92), Statistics(X) + +- ReusedExchange (91) +- == Initial Plan == - TakeOrderedAndProject (155) - +- Project (154) - +- SortMergeJoin Inner (153) - :- Sort (148) - : +- Exchange (147) - : +- Project (146) - : +- SortMergeJoin Inner (145) - : :- Sort (140) - : : +- Exchange (139) - : : +- HashAggregate (138) - : : +- HashAggregate (137) - : : +- Project (136) - : : +- SortMergeJoin Inner (135) - : : :- Sort (130) - : : : +- Exchange (129) - : : : +- Project (128) - : : : +- SortMergeJoin Inner (127) - : : : :- Sort (121) - : : : : +- Exchange (120) - : : : : +- Project (119) - : : : : +- SortMergeJoin Inner (118) - : : : : :- Sort (112) - : : : : : +- Exchange (111) - : : : : : +- Project (110) - : : : : : +- SortMergeJoin Inner (109) - : : : : : :- Sort (103) - : : : : : : +- Exchange (102) - : : : : : : +- Filter (101) - : : : : : : +- Scan parquet (100) - : : : : : +- Sort (108) - : : : : : +- Exchange (107) - : : : : : +- Project (106) - : : : : : +- Filter (105) - : : : : : +- Scan parquet (104) - : : : : +- Sort (117) - : : : : +- Exchange (116) - : : : : +- Project (115) - : : : : +- Filter (114) - : : : : +- Scan parquet (113) - : : : +- Sort (126) - : : : +- Exchange (125) - : : : +- Project (124) - : : : +- Filter (123) - : : : +- Scan parquet (122) - : : +- Sort (134) - : : +- Exchange (133) - : : +- Filter (132) - : : +- Scan parquet (131) - : +- Sort (144) - : +- Exchange (143) - : +- Filter (142) - : +- Scan parquet (141) - +- Sort (152) - +- Exchange (151) - +- Filter (150) - +- Scan parquet (149) - - -(100) Scan parquet + TakeOrderedAndProject (156) + +- Project (155) + +- SortMergeJoin Inner (154) + :- Sort (149) + : +- Exchange (148) + : +- Project (147) + : +- SortMergeJoin Inner (146) + : :- Sort (141) + : : +- Exchange (140) + : : +- HashAggregate (139) + : : +- HashAggregate (138) + : : +- Project (137) + : : +- SortMergeJoin Inner (136) + : : :- Sort (131) + : : : +- Exchange (130) + : : : +- Project (129) + : : : +- SortMergeJoin Inner (128) + : : : :- Sort (122) + : : : : +- Exchange (121) + : : : : +- Project (120) + : : : : +- SortMergeJoin Inner (119) + : : : : :- Sort (113) + : : : : : +- Exchange (112) + : : : : : +- Project (111) + : : : : : +- SortMergeJoin Inner (110) + : : : : : :- Sort (104) + : : : : : : +- Exchange (103) + : : : : : : +- Filter (102) + : : : : : : +- Scan parquet (101) + : : : : : +- Sort (109) + : : : : : +- Exchange (108) + : : : : : +- Project (107) + : : : : : +- Filter (106) + : : : : : +- Scan parquet (105) + : : : : +- Sort (118) + : : : : +- Exchange (117) + : : : : +- Project (116) + : : : : +- Filter (115) + : : : : +- Scan parquet (114) + : : : +- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- Filter (124) + : : : +- Scan parquet (123) + : : +- Sort (135) + : : +- Exchange (134) + : : +- Filter (133) + : : +- Scan parquet (132) + : +- Sort (145) + : +- Exchange (144) + : +- Filter (143) + : +- Scan parquet (142) + +- Sort (153) + +- Exchange (152) + +- Filter (151) + +- Scan parquet (150) + + +(101) Scan parquet Output [8]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Batched: true Location: InMemoryFileIndex [file:/] @@ -193,7 +194,7 @@ Input [8]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8] Input [8]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false -(104) Scan parquet +(105) Scan parquet Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location: InMemoryFileIndex [file:/] @@ -260,7 +261,7 @@ Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_tic Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false -(113) Scan parquet +(114) Scan parquet Output [2]: [s_store_sk#12, s_city#13] Batched: true Location: InMemoryFileIndex [file:/] @@ -327,7 +328,7 @@ Input [6]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, s Input [6]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: [ss_hdemo_sk#3 ASC NULLS FIRST], false -(122) Scan parquet +(123) Scan parquet Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Batched: true Location: InMemoryFileIndex [file:/] @@ -394,7 +395,7 @@ Input [5]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, Input [5]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: [ss_addr_sk#4 ASC NULLS FIRST], false -(131) Scan parquet +(132) Scan parquet Output [2]: [ca_address_sk#17, ca_city#18] Batched: true Location: InMemoryFileIndex [file:/] @@ -479,7 +480,7 @@ Input [5]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#26, amt#27, profit Input [5]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#26, amt#27, profit#28] Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false -(141) Scan parquet +(142) Scan parquet Output [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Batched: true Location: InMemoryFileIndex [file:/] @@ -564,284 +565,288 @@ Input [2]: [#33#33, #34#34] Input [2]: [#33#33, #34#34] Arguments: [ca_address_sk#33 ASC NULLS FIRST], false -(97) SortMergeJoin [codegen id : X] +(97) NativeSortMergeJoin Left keys [1]: [c_current_addr_sk#30] Right keys [1]: [ca_address_sk#33] Join type: Inner -Join condition: NOT (ca_city#34 = bought_city#26) +Join condition: None + +(98) NativeFilter +Input [9]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#30, c_first_name#31, c_last_name#32, #33#33, #34#34] +Condition : NOT (ca_city#34 = bought_city#26) -(98) Project [codegen id : X] +(99) NativeProject Output [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#6, amt#27, profit#28] Input [9]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#30, c_first_name#31, c_last_name#32, #33#33, #34#34] -(99) TakeOrderedAndProject +(100) NativeTakeOrdered Input [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#6, amt#27, profit#28] -Arguments: X, [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, ca_city#34 ASC NULLS FIRST, bought_city#26 ASC NULLS FIRST, ss_ticket_number#6 ASC NULLS FIRST], [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#6, amt#27, profit#28] +Arguments: X, X, [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, ca_city#34 ASC NULLS FIRST, bought_city#26 ASC NULLS FIRST, ss_ticket_number#6 ASC NULLS FIRST] -(100) Scan parquet +(101) Scan parquet Output [8]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(101) Filter +(102) Filter Input [8]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Condition : ((((isnotnull(ss_sold_date_sk#1) AND isnotnull(ss_store_sk#5)) AND isnotnull(ss_hdemo_sk#3)) AND isnotnull(ss_addr_sk#4)) AND isnotnull(ss_customer_sk#2)) -(102) Exchange +(103) Exchange Input [8]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: hashpartitioning(ss_sold_date_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=12] -(103) Sort +(104) Sort Input [8]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false, 0 -(104) Scan parquet +(105) Scan parquet Output [3]: [d_date_sk#9, d_year#10, d_dow#11] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(105) Filter +(106) Filter Input [3]: [d_date_sk#9, d_year#10, d_dow#11] Condition : ((d_dow#11 IN (6,0) AND d_year#10 IN (1999,2000,2001)) AND isnotnull(d_date_sk#9)) -(106) Project +(107) Project Output [1]: [d_date_sk#9] Input [3]: [d_date_sk#9, d_year#10, d_dow#11] -(107) Exchange +(108) Exchange Input [1]: [d_date_sk#9] Arguments: hashpartitioning(d_date_sk#9, 100), ENSURE_REQUIREMENTS, [plan_id=13] -(108) Sort +(109) Sort Input [1]: [d_date_sk#9] Arguments: [d_date_sk#9 ASC NULLS FIRST], false, 0 -(109) SortMergeJoin +(110) SortMergeJoin Left keys [1]: [ss_sold_date_sk#1] Right keys [1]: [d_date_sk#9] Join type: Inner Join condition: None -(110) Project +(111) Project Output [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Input [9]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8, d_date_sk#9] -(111) Exchange +(112) Exchange Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: hashpartitioning(ss_store_sk#5, 100), ENSURE_REQUIREMENTS, [plan_id=14] -(112) Sort +(113) Sort Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false, 0 -(113) Scan parquet +(114) Scan parquet Output [2]: [s_store_sk#12, s_city#13] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [s_store_sk#12, s_city#13] Condition : (s_city#13 IN (Fairview,Midway) AND isnotnull(s_store_sk#12)) -(115) Project +(116) Project Output [1]: [s_store_sk#12] Input [2]: [s_store_sk#12, s_city#13] -(116) Exchange +(117) Exchange Input [1]: [s_store_sk#12] Arguments: hashpartitioning(s_store_sk#12, 100), ENSURE_REQUIREMENTS, [plan_id=15] -(117) Sort +(118) Sort Input [1]: [s_store_sk#12] Arguments: [s_store_sk#12 ASC NULLS FIRST], false, 0 -(118) SortMergeJoin +(119) SortMergeJoin Left keys [1]: [ss_store_sk#5] Right keys [1]: [s_store_sk#12] Join type: Inner Join condition: None -(119) Project +(120) Project Output [6]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Input [8]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8, s_store_sk#12] -(120) Exchange +(121) Exchange Input [6]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: hashpartitioning(ss_hdemo_sk#3, 100), ENSURE_REQUIREMENTS, [plan_id=16] -(121) Sort +(122) Sort Input [6]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: [ss_hdemo_sk#3 ASC NULLS FIRST], false, 0 -(122) Scan parquet +(123) Scan parquet Output [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(123) Filter +(124) Filter Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] Condition : (((hd_dep_count#15 = 4) OR (hd_vehicle_count#16 = 3)) AND isnotnull(hd_demo_sk#14)) -(124) Project +(125) Project Output [1]: [hd_demo_sk#14] Input [3]: [hd_demo_sk#14, hd_dep_count#15, hd_vehicle_count#16] -(125) Exchange +(126) Exchange Input [1]: [hd_demo_sk#14] Arguments: hashpartitioning(hd_demo_sk#14, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(126) Sort +(127) Sort Input [1]: [hd_demo_sk#14] Arguments: [hd_demo_sk#14 ASC NULLS FIRST], false, 0 -(127) SortMergeJoin +(128) SortMergeJoin Left keys [1]: [ss_hdemo_sk#3] Right keys [1]: [hd_demo_sk#14] Join type: Inner Join condition: None -(128) Project +(129) Project Output [5]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8, hd_demo_sk#14] -(129) Exchange +(130) Exchange Input [5]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: hashpartitioning(ss_addr_sk#4, 100), ENSURE_REQUIREMENTS, [plan_id=18] -(130) Sort +(131) Sort Input [5]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8] Arguments: [ss_addr_sk#4 ASC NULLS FIRST], false, 0 -(131) Scan parquet +(132) Scan parquet Output [2]: [ca_address_sk#17, ca_city#18] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(132) Filter +(133) Filter Input [2]: [ca_address_sk#17, ca_city#18] Condition : (isnotnull(ca_address_sk#17) AND isnotnull(ca_city#18)) -(133) Exchange +(134) Exchange Input [2]: [ca_address_sk#17, ca_city#18] Arguments: hashpartitioning(ca_address_sk#17, 100), ENSURE_REQUIREMENTS, [plan_id=19] -(134) Sort +(135) Sort Input [2]: [ca_address_sk#17, ca_city#18] Arguments: [ca_address_sk#17 ASC NULLS FIRST], false, 0 -(135) SortMergeJoin +(136) SortMergeJoin Left keys [1]: [ss_addr_sk#4] Right keys [1]: [ca_address_sk#17] Join type: Inner Join condition: None -(136) Project +(137) Project Output [6]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8, ca_city#18] Input [7]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8, ca_address_sk#17, ca_city#18] -(137) HashAggregate +(138) HashAggregate Input [6]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_coupon_amt#7, ss_net_profit#8, ca_city#18] Keys [4]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#18] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#7)), partial_sum(UnscaledValue(ss_net_profit#8))] Aggregate Attributes [2]: [sum#21, sum#22] Results [6]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#18, sum#35, sum#36] -(138) HashAggregate +(139) HashAggregate Input [6]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#18, sum#35, sum#36] Keys [4]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#18] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#7)), sum(UnscaledValue(ss_net_profit#8))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#7))#24, sum(UnscaledValue(ss_net_profit#8))#25] Results [5]: [ss_ticket_number#6, ss_customer_sk#2, ca_city#18 AS bought_city#26, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#7))#24,17,2) AS amt#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#8))#25,17,2) AS profit#28] -(139) Exchange +(140) Exchange Input [5]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#26, amt#27, profit#28] Arguments: hashpartitioning(ss_customer_sk#2, 100), ENSURE_REQUIREMENTS, [plan_id=20] -(140) Sort +(141) Sort Input [5]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#26, amt#27, profit#28] Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0 -(141) Scan parquet +(142) Scan parquet Output [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(142) Filter +(143) Filter Input [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Condition : (isnotnull(c_customer_sk#29) AND isnotnull(c_current_addr_sk#30)) -(143) Exchange +(144) Exchange Input [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Arguments: hashpartitioning(c_customer_sk#29, 100), ENSURE_REQUIREMENTS, [plan_id=21] -(144) Sort +(145) Sort Input [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Arguments: [c_customer_sk#29 ASC NULLS FIRST], false, 0 -(145) SortMergeJoin +(146) SortMergeJoin Left keys [1]: [ss_customer_sk#2] Right keys [1]: [c_customer_sk#29] Join type: Inner Join condition: None -(146) Project +(147) Project Output [7]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Input [9]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#26, amt#27, profit#28, c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] -(147) Exchange +(148) Exchange Input [7]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Arguments: hashpartitioning(c_current_addr_sk#30, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(148) Sort +(149) Sort Input [7]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Arguments: [c_current_addr_sk#30 ASC NULLS FIRST], false, 0 -(149) Scan parquet +(150) Scan parquet Output [2]: [ca_address_sk#33, ca_city#34] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(150) Filter +(151) Filter Input [2]: [ca_address_sk#33, ca_city#34] Condition : (isnotnull(ca_address_sk#33) AND isnotnull(ca_city#34)) -(151) Exchange +(152) Exchange Input [2]: [ca_address_sk#33, ca_city#34] Arguments: hashpartitioning(ca_address_sk#33, 100), ENSURE_REQUIREMENTS, [plan_id=23] -(152) Sort +(153) Sort Input [2]: [ca_address_sk#33, ca_city#34] Arguments: [ca_address_sk#33 ASC NULLS FIRST], false, 0 -(153) SortMergeJoin +(154) SortMergeJoin Left keys [1]: [c_current_addr_sk#30] Right keys [1]: [ca_address_sk#33] Join type: Inner Join condition: NOT (ca_city#34 = bought_city#26) -(154) Project +(155) Project Output [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#6, amt#27, profit#28] Input [9]: [ss_ticket_number#6, bought_city#26, amt#27, profit#28, c_current_addr_sk#30, c_first_name#31, c_last_name#32, ca_address_sk#33, ca_city#34] -(155) TakeOrderedAndProject +(156) TakeOrderedAndProject Input [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#6, amt#27, profit#28] Arguments: X, [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, ca_city#34 ASC NULLS FIRST, bought_city#26 ASC NULLS FIRST, ss_ticket_number#6 ASC NULLS FIRST], [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#6, amt#27, profit#28] -(156) AdaptiveSparkPlan +(157) AdaptiveSparkPlan Output [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#6, amt#27, profit#28] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q48.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q48.txt index 47e141b4b..be381617c 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q48.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q48.txt @@ -13,17 +13,17 @@ AdaptiveSparkPlan (112) : +- AQEShuffleRead (54) : +- ShuffleQueryStage (53), Statistics(X) : +- NativeShuffleExchange (52) - : +- ConvertToNative (51) - : +- * Project (50) - : +- * SortMergeJoin Inner (49) + : +- NativeProject (51) + : +- NativeFilter (50) + : +- NativeSortMergeJoin Inner (49) : :- NativeSort (39) : : +- InputAdapter (38) : : +- AQEShuffleRead (37) : : +- ShuffleQueryStage (36), Statistics(X) : : +- NativeShuffleExchange (35) - : : +- ConvertToNative (34) - : : +- * Project (33) - : : +- * SortMergeJoin Inner (32) + : : +- NativeProject (34) + : : +- NativeFilter (33) + : : +- NativeSortMergeJoin Inner (32) : : :- NativeSort (23) : : : +- InputAdapter (22) : : : +- AQEShuffleRead (21) @@ -246,18 +246,19 @@ Input [3]: [#9#9, #10#10, #11#11] Input [3]: [#9#9, #10#10, #11#11] Arguments: [cd_demo_sk#9 ASC NULLS FIRST], false -(32) SortMergeJoin [codegen id : X] +(32) NativeSortMergeJoin Left keys [1]: [ss_cdemo_sk#2] Right keys [1]: [cd_demo_sk#9] Join type: Inner -Join condition: ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree)) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree)) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College)) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))) +Join condition: None -(33) Project [codegen id : X] -Output [4]: [ss_sold_date_sk#1, ss_addr_sk#3, ss_quantity#5, ss_net_profit#7] +(33) NativeFilter Input [9]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_net_profit#7, #9#9, #10#10, #11#11] +Condition : ((((((cd_marital_status#10 = M) AND (cd_education_status#11 = 4 yr Degree)) AND (ss_sales_price#6 >= 100.00)) AND (ss_sales_price#6 <= 150.00)) OR ((((cd_marital_status#10 = D) AND (cd_education_status#11 = 2 yr Degree)) AND (ss_sales_price#6 >= 50.00)) AND (ss_sales_price#6 <= 100.00))) OR ((((cd_marital_status#10 = S) AND (cd_education_status#11 = College)) AND (ss_sales_price#6 >= 150.00)) AND (ss_sales_price#6 <= 200.00))) -(34) ConvertToNative -Input [4]: [ss_sold_date_sk#1, ss_addr_sk#3, ss_quantity#5, ss_net_profit#7] +(34) NativeProject +Output [4]: [ss_sold_date_sk#1, ss_addr_sk#3, ss_quantity#5, ss_net_profit#7] +Input [9]: [ss_sold_date_sk#1, ss_cdemo_sk#2, ss_addr_sk#3, ss_quantity#5, ss_sales_price#6, ss_net_profit#7, #9#9, #10#10, #11#11] (35) NativeShuffleExchange Input [4]: [ss_sold_date_sk#1, ss_addr_sk#3, ss_quantity#5, ss_net_profit#7] @@ -316,18 +317,19 @@ Input [2]: [ca_address_sk#12, ca_state#13] Input [2]: [ca_address_sk#12, ca_state#13] Arguments: [ca_address_sk#12 ASC NULLS FIRST], false -(49) SortMergeJoin [codegen id : X] +(49) NativeSortMergeJoin Left keys [1]: [ss_addr_sk#3] Right keys [1]: [ca_address_sk#12] Join type: Inner -Join condition: ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#7 >= 0.00)) AND (ss_net_profit#7 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#7 >= 150.00)) AND (ss_net_profit#7 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#7 >= 50.00)) AND (ss_net_profit#7 <= 25000.00))) +Join condition: None -(50) Project [codegen id : X] -Output [2]: [ss_sold_date_sk#1, ss_quantity#5] +(50) NativeFilter Input [6]: [ss_sold_date_sk#1, ss_addr_sk#3, ss_quantity#5, ss_net_profit#7, ca_address_sk#12, ca_state#13] +Condition : ((((ca_state#13 IN (CO,OH,TX) AND (ss_net_profit#7 >= 0.00)) AND (ss_net_profit#7 <= 2000.00)) OR ((ca_state#13 IN (OR,MN,KY) AND (ss_net_profit#7 >= 150.00)) AND (ss_net_profit#7 <= 3000.00))) OR ((ca_state#13 IN (VA,CA,MS) AND (ss_net_profit#7 >= 50.00)) AND (ss_net_profit#7 <= 25000.00))) -(51) ConvertToNative -Input [2]: [ss_sold_date_sk#1, ss_quantity#5] +(51) NativeProject +Output [2]: [ss_sold_date_sk#1, ss_quantity#5] +Input [6]: [ss_sold_date_sk#1, ss_addr_sk#3, ss_quantity#5, ss_net_profit#7, ca_address_sk#12, ca_state#13] (52) NativeShuffleExchange Input [2]: [ss_sold_date_sk#1, ss_quantity#5] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q58.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q58.txt index 22236d1b4..e0aa59465 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q58.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q58.txt @@ -1,253 +1,255 @@ == Physical Plan == -AdaptiveSparkPlan (244) +AdaptiveSparkPlan (246) +- == Final Plan == - TakeOrderedAndProject (142) - +- * Project (141) - +- * SortMergeJoin Inner (140) - :- * Project (100) - : +- * SortMergeJoin Inner (99) - : :- NativeSort (59) - : : +- NativeFilter (58) - : : +- NativeProject (57) - : : +- NativeHashAggregate (56) - : : +- InputAdapter (55) - : : +- AQEShuffleRead (54) - : : +- ShuffleQueryStage (53), Statistics(X) - : : +- NativeShuffleExchange (52) - : : +- NativeHashAggregate (51) - : : +- NativeProject (50) - : : +- NativeProject (49) - : : +- NativeSortMergeJoin Inner (48) - : : :- NativeSort (23) - : : : +- InputAdapter (22) - : : : +- AQEShuffleRead (21) - : : : +- ShuffleQueryStage (20), Statistics(X) - : : : +- NativeShuffleExchange (19) - : : : +- NativeProject (18) - : : : +- NativeSortMergeJoin Inner (17) - : : : :- NativeSort (8) - : : : : +- InputAdapter (7) - : : : : +- AQEShuffleRead (6) - : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : +- NativeShuffleExchange (4) - : : : : +- NativeFilter (3) - : : : : +- InputAdapter (2) - : : : : +- NativeParquetScan (1) - : : : +- NativeSort (16) - : : : +- InputAdapter (15) - : : : +- AQEShuffleRead (14) - : : : +- ShuffleQueryStage (13), Statistics(X) - : : : +- NativeShuffleExchange (12) - : : : +- NativeFilter (11) - : : : +- InputAdapter (10) - : : : +- NativeParquetScan (9) - : : +- NativeSort (47) - : : +- InputAdapter (46) - : : +- AQEShuffleRead (45) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- NativeShuffleExchange (43) - : : +- NativeProject (42) - : : +- NativeSortMergeJoin LeftSemi (41) - : : :- NativeSort (31) - : : : +- InputAdapter (30) - : : : +- AQEShuffleRead (29) - : : : +- ShuffleQueryStage (28), Statistics(X) - : : : +- NativeShuffleExchange (27) - : : : +- NativeFilter (26) - : : : +- InputAdapter (25) - : : : +- NativeParquetScan (24) - : : +- NativeSort (40) - : : +- InputAdapter (39) - : : +- AQEShuffleRead (38) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- NativeShuffleExchange (36) - : : +- NativeProject (35) - : : +- NativeFilter (34) - : : +- InputAdapter (33) - : : +- NativeParquetScan (32) - : +- NativeSort (98) - : +- NativeFilter (97) - : +- NativeProject (96) - : +- NativeHashAggregate (95) - : +- InputAdapter (94) - : +- AQEShuffleRead (93) - : +- ShuffleQueryStage (92), Statistics(X) - : +- NativeShuffleExchange (91) - : +- NativeHashAggregate (90) - : +- NativeProject (89) - : +- NativeProject (88) - : +- NativeSortMergeJoin Inner (87) - : :- NativeSort (80) - : : +- InputAdapter (79) - : : +- AQEShuffleRead (78) - : : +- ShuffleQueryStage (77), Statistics(X) - : : +- NativeShuffleExchange (76) - : : +- NativeProject (75) - : : +- NativeSortMergeJoin Inner (74) - : : :- NativeSort (67) - : : : +- InputAdapter (66) - : : : +- AQEShuffleRead (65) - : : : +- ShuffleQueryStage (64), Statistics(X) - : : : +- NativeShuffleExchange (63) - : : : +- NativeFilter (62) - : : : +- InputAdapter (61) - : : : +- NativeParquetScan (60) - : : +- NativeSort (73) - : : +- InputAdapter (72) - : : +- InputAdapter (71) - : : +- AQEShuffleRead (70) - : : +- ShuffleQueryStage (69), Statistics(X) - : : +- ReusedExchange (68) - : +- NativeSort (86) - : +- InputAdapter (85) - : +- InputAdapter (84) - : +- AQEShuffleRead (83) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- NativeSort (139) - +- NativeFilter (138) - +- NativeProject (137) - +- NativeHashAggregate (136) - +- InputAdapter (135) - +- AQEShuffleRead (134) - +- ShuffleQueryStage (133), Statistics(X) - +- NativeShuffleExchange (132) - +- NativeHashAggregate (131) - +- NativeProject (130) - +- NativeProject (129) - +- NativeSortMergeJoin Inner (128) - :- NativeSort (121) - : +- InputAdapter (120) - : +- AQEShuffleRead (119) - : +- ShuffleQueryStage (118), Statistics(X) - : +- NativeShuffleExchange (117) - : +- NativeProject (116) - : +- NativeSortMergeJoin Inner (115) - : :- NativeSort (108) - : : +- InputAdapter (107) - : : +- AQEShuffleRead (106) - : : +- ShuffleQueryStage (105), Statistics(X) - : : +- NativeShuffleExchange (104) - : : +- NativeFilter (103) - : : +- InputAdapter (102) - : : +- NativeParquetScan (101) - : +- NativeSort (114) - : +- InputAdapter (113) - : +- InputAdapter (112) - : +- AQEShuffleRead (111) - : +- ShuffleQueryStage (110), Statistics(X) - : +- ReusedExchange (109) - +- NativeSort (127) - +- InputAdapter (126) - +- InputAdapter (125) - +- AQEShuffleRead (124) - +- ShuffleQueryStage (123), Statistics(X) - +- ReusedExchange (122) + NativeTakeOrdered (144) + +- NativeProject (143) + +- NativeFilter (142) + +- NativeSortMergeJoin Inner (141) + :- NativeProject (101) + : +- NativeFilter (100) + : +- NativeSortMergeJoin Inner (99) + : :- NativeSort (59) + : : +- NativeFilter (58) + : : +- NativeProject (57) + : : +- NativeHashAggregate (56) + : : +- InputAdapter (55) + : : +- AQEShuffleRead (54) + : : +- ShuffleQueryStage (53), Statistics(X) + : : +- NativeShuffleExchange (52) + : : +- NativeHashAggregate (51) + : : +- NativeProject (50) + : : +- NativeProject (49) + : : +- NativeSortMergeJoin Inner (48) + : : :- NativeSort (23) + : : : +- InputAdapter (22) + : : : +- AQEShuffleRead (21) + : : : +- ShuffleQueryStage (20), Statistics(X) + : : : +- NativeShuffleExchange (19) + : : : +- NativeProject (18) + : : : +- NativeSortMergeJoin Inner (17) + : : : :- NativeSort (8) + : : : : +- InputAdapter (7) + : : : : +- AQEShuffleRead (6) + : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : +- NativeShuffleExchange (4) + : : : : +- NativeFilter (3) + : : : : +- InputAdapter (2) + : : : : +- NativeParquetScan (1) + : : : +- NativeSort (16) + : : : +- InputAdapter (15) + : : : +- AQEShuffleRead (14) + : : : +- ShuffleQueryStage (13), Statistics(X) + : : : +- NativeShuffleExchange (12) + : : : +- NativeFilter (11) + : : : +- InputAdapter (10) + : : : +- NativeParquetScan (9) + : : +- NativeSort (47) + : : +- InputAdapter (46) + : : +- AQEShuffleRead (45) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- NativeShuffleExchange (43) + : : +- NativeProject (42) + : : +- NativeSortMergeJoin LeftSemi (41) + : : :- NativeSort (31) + : : : +- InputAdapter (30) + : : : +- AQEShuffleRead (29) + : : : +- ShuffleQueryStage (28), Statistics(X) + : : : +- NativeShuffleExchange (27) + : : : +- NativeFilter (26) + : : : +- InputAdapter (25) + : : : +- NativeParquetScan (24) + : : +- NativeSort (40) + : : +- InputAdapter (39) + : : +- AQEShuffleRead (38) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- NativeShuffleExchange (36) + : : +- NativeProject (35) + : : +- NativeFilter (34) + : : +- InputAdapter (33) + : : +- NativeParquetScan (32) + : +- NativeSort (98) + : +- NativeFilter (97) + : +- NativeProject (96) + : +- NativeHashAggregate (95) + : +- InputAdapter (94) + : +- AQEShuffleRead (93) + : +- ShuffleQueryStage (92), Statistics(X) + : +- NativeShuffleExchange (91) + : +- NativeHashAggregate (90) + : +- NativeProject (89) + : +- NativeProject (88) + : +- NativeSortMergeJoin Inner (87) + : :- NativeSort (80) + : : +- InputAdapter (79) + : : +- AQEShuffleRead (78) + : : +- ShuffleQueryStage (77), Statistics(X) + : : +- NativeShuffleExchange (76) + : : +- NativeProject (75) + : : +- NativeSortMergeJoin Inner (74) + : : :- NativeSort (67) + : : : +- InputAdapter (66) + : : : +- AQEShuffleRead (65) + : : : +- ShuffleQueryStage (64), Statistics(X) + : : : +- NativeShuffleExchange (63) + : : : +- NativeFilter (62) + : : : +- InputAdapter (61) + : : : +- NativeParquetScan (60) + : : +- NativeSort (73) + : : +- InputAdapter (72) + : : +- InputAdapter (71) + : : +- AQEShuffleRead (70) + : : +- ShuffleQueryStage (69), Statistics(X) + : : +- ReusedExchange (68) + : +- NativeSort (86) + : +- InputAdapter (85) + : +- InputAdapter (84) + : +- AQEShuffleRead (83) + : +- ShuffleQueryStage (82), Statistics(X) + : +- ReusedExchange (81) + +- NativeSort (140) + +- NativeFilter (139) + +- NativeProject (138) + +- NativeHashAggregate (137) + +- InputAdapter (136) + +- AQEShuffleRead (135) + +- ShuffleQueryStage (134), Statistics(X) + +- NativeShuffleExchange (133) + +- NativeHashAggregate (132) + +- NativeProject (131) + +- NativeProject (130) + +- NativeSortMergeJoin Inner (129) + :- NativeSort (122) + : +- InputAdapter (121) + : +- AQEShuffleRead (120) + : +- ShuffleQueryStage (119), Statistics(X) + : +- NativeShuffleExchange (118) + : +- NativeProject (117) + : +- NativeSortMergeJoin Inner (116) + : :- NativeSort (109) + : : +- InputAdapter (108) + : : +- AQEShuffleRead (107) + : : +- ShuffleQueryStage (106), Statistics(X) + : : +- NativeShuffleExchange (105) + : : +- NativeFilter (104) + : : +- InputAdapter (103) + : : +- NativeParquetScan (102) + : +- NativeSort (115) + : +- InputAdapter (114) + : +- InputAdapter (113) + : +- AQEShuffleRead (112) + : +- ShuffleQueryStage (111), Statistics(X) + : +- ReusedExchange (110) + +- NativeSort (128) + +- InputAdapter (127) + +- InputAdapter (126) + +- AQEShuffleRead (125) + +- ShuffleQueryStage (124), Statistics(X) + +- ReusedExchange (123) +- == Initial Plan == - TakeOrderedAndProject (243) - +- Project (242) - +- SortMergeJoin Inner (241) - :- Project (208) - : +- SortMergeJoin Inner (207) - : :- Sort (174) - : : +- Filter (173) - : : +- HashAggregate (172) - : : +- Exchange (171) - : : +- HashAggregate (170) - : : +- Project (169) - : : +- SortMergeJoin Inner (168) - : : :- Sort (154) - : : : +- Exchange (153) - : : : +- Project (152) - : : : +- SortMergeJoin Inner (151) - : : : :- Sort (146) - : : : : +- Exchange (145) - : : : : +- Filter (144) - : : : : +- Scan parquet (143) - : : : +- Sort (150) - : : : +- Exchange (149) - : : : +- Filter (148) - : : : +- Scan parquet (147) - : : +- Sort (167) - : : +- Exchange (166) - : : +- Project (165) - : : +- SortMergeJoin LeftSemi (164) - : : :- Sort (158) - : : : +- Exchange (157) - : : : +- Filter (156) - : : : +- Scan parquet (155) - : : +- Sort (163) - : : +- Exchange (162) - : : +- Project (161) - : : +- Filter (160) - : : +- Scan parquet (159) - : +- Sort (206) - : +- Filter (205) - : +- HashAggregate (204) - : +- Exchange (203) - : +- HashAggregate (202) - : +- Project (201) - : +- SortMergeJoin Inner (200) - : :- Sort (186) - : : +- Exchange (185) - : : +- Project (184) - : : +- SortMergeJoin Inner (183) - : : :- Sort (178) - : : : +- Exchange (177) - : : : +- Filter (176) - : : : +- Scan parquet (175) - : : +- Sort (182) - : : +- Exchange (181) - : : +- Filter (180) - : : +- Scan parquet (179) - : +- Sort (199) - : +- Exchange (198) - : +- Project (197) - : +- SortMergeJoin LeftSemi (196) - : :- Sort (190) - : : +- Exchange (189) - : : +- Filter (188) - : : +- Scan parquet (187) - : +- Sort (195) - : +- Exchange (194) - : +- Project (193) - : +- Filter (192) - : +- Scan parquet (191) - +- Sort (240) - +- Filter (239) - +- HashAggregate (238) - +- Exchange (237) - +- HashAggregate (236) - +- Project (235) - +- SortMergeJoin Inner (234) - :- Sort (220) - : +- Exchange (219) - : +- Project (218) - : +- SortMergeJoin Inner (217) - : :- Sort (212) - : : +- Exchange (211) - : : +- Filter (210) - : : +- Scan parquet (209) - : +- Sort (216) - : +- Exchange (215) - : +- Filter (214) - : +- Scan parquet (213) - +- Sort (233) - +- Exchange (232) - +- Project (231) - +- SortMergeJoin LeftSemi (230) - :- Sort (224) - : +- Exchange (223) - : +- Filter (222) - : +- Scan parquet (221) - +- Sort (229) - +- Exchange (228) - +- Project (227) - +- Filter (226) - +- Scan parquet (225) - - -(143) Scan parquet + TakeOrderedAndProject (245) + +- Project (244) + +- SortMergeJoin Inner (243) + :- Project (210) + : +- SortMergeJoin Inner (209) + : :- Sort (176) + : : +- Filter (175) + : : +- HashAggregate (174) + : : +- Exchange (173) + : : +- HashAggregate (172) + : : +- Project (171) + : : +- SortMergeJoin Inner (170) + : : :- Sort (156) + : : : +- Exchange (155) + : : : +- Project (154) + : : : +- SortMergeJoin Inner (153) + : : : :- Sort (148) + : : : : +- Exchange (147) + : : : : +- Filter (146) + : : : : +- Scan parquet (145) + : : : +- Sort (152) + : : : +- Exchange (151) + : : : +- Filter (150) + : : : +- Scan parquet (149) + : : +- Sort (169) + : : +- Exchange (168) + : : +- Project (167) + : : +- SortMergeJoin LeftSemi (166) + : : :- Sort (160) + : : : +- Exchange (159) + : : : +- Filter (158) + : : : +- Scan parquet (157) + : : +- Sort (165) + : : +- Exchange (164) + : : +- Project (163) + : : +- Filter (162) + : : +- Scan parquet (161) + : +- Sort (208) + : +- Filter (207) + : +- HashAggregate (206) + : +- Exchange (205) + : +- HashAggregate (204) + : +- Project (203) + : +- SortMergeJoin Inner (202) + : :- Sort (188) + : : +- Exchange (187) + : : +- Project (186) + : : +- SortMergeJoin Inner (185) + : : :- Sort (180) + : : : +- Exchange (179) + : : : +- Filter (178) + : : : +- Scan parquet (177) + : : +- Sort (184) + : : +- Exchange (183) + : : +- Filter (182) + : : +- Scan parquet (181) + : +- Sort (201) + : +- Exchange (200) + : +- Project (199) + : +- SortMergeJoin LeftSemi (198) + : :- Sort (192) + : : +- Exchange (191) + : : +- Filter (190) + : : +- Scan parquet (189) + : +- Sort (197) + : +- Exchange (196) + : +- Project (195) + : +- Filter (194) + : +- Scan parquet (193) + +- Sort (242) + +- Filter (241) + +- HashAggregate (240) + +- Exchange (239) + +- HashAggregate (238) + +- Project (237) + +- SortMergeJoin Inner (236) + :- Sort (222) + : +- Exchange (221) + : +- Project (220) + : +- SortMergeJoin Inner (219) + : :- Sort (214) + : : +- Exchange (213) + : : +- Filter (212) + : : +- Scan parquet (211) + : +- Sort (218) + : +- Exchange (217) + : +- Filter (216) + : +- Scan parquet (215) + +- Sort (235) + +- Exchange (234) + +- Project (233) + +- SortMergeJoin LeftSemi (232) + :- Sort (226) + : +- Exchange (225) + : +- Filter (224) + : +- Scan parquet (223) + +- Sort (231) + +- Exchange (230) + +- Project (229) + +- Filter (228) + +- Scan parquet (227) + + +(145) Scan parquet Output [3]: [ss_sold_date_sk#1, ss_item_sk#2, ss_ext_sales_price#3] Batched: true Location: InMemoryFileIndex [file:/] @@ -281,7 +283,7 @@ Input [3]: [#1#1, #2#2, #3#3] Input [3]: [#1#1, #2#2, #3#3] Arguments: [ss_item_sk#2 ASC NULLS FIRST], false -(147) Scan parquet +(149) Scan parquet Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location: InMemoryFileIndex [file:/] @@ -344,7 +346,7 @@ Input [3]: [ss_sold_date_sk#1, ss_ext_sales_price#3, i_item_id#5] Input [3]: [ss_sold_date_sk#1, ss_ext_sales_price#3, i_item_id#5] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false -(155) Scan parquet +(157) Scan parquet Output [2]: [d_date_sk#6, d_date#7] Batched: true Location: InMemoryFileIndex [file:/] @@ -378,7 +380,7 @@ Input [2]: [#6#6, #7#7] Input [2]: [#6#6, #7#7] Arguments: [d_date#7 ASC NULLS FIRST], false -(159) Scan parquet +(161) Scan parquet Output [2]: [d_date#8, d_week_seq#9] Batched: true Location: InMemoryFileIndex [file:/] @@ -500,7 +502,7 @@ Condition : isnotnull(ss_item_rev#17) Input [2]: [item_id#16, ss_item_rev#17] Arguments: [item_id#16 ASC NULLS FIRST], false -(175) Scan parquet +(177) Scan parquet Output [3]: [cs_sold_date_sk#18, cs_item_sk#19, cs_ext_sales_price#20] Batched: true Location: InMemoryFileIndex [file:/] @@ -662,726 +664,734 @@ Condition : isnotnull(cs_item_rev#28) Input [2]: [item_id#27, cs_item_rev#28] Arguments: [item_id#27 ASC NULLS FIRST], false -(99) SortMergeJoin [codegen id : X] +(99) NativeSortMergeJoin Left keys [1]: [item_id#16] Right keys [1]: [item_id#27] Join type: Inner -Join condition: ((((cast(ss_item_rev#17 as decimal(19,3)) >= (0.9 * cs_item_rev#28)) AND (cast(ss_item_rev#17 as decimal(20,3)) <= (1.1 * cs_item_rev#28))) AND (cast(cs_item_rev#28 as decimal(19,3)) >= (0.9 * ss_item_rev#17))) AND (cast(cs_item_rev#28 as decimal(20,3)) <= (1.1 * ss_item_rev#17))) +Join condition: None + +(100) NativeFilter +Input [4]: [item_id#16, ss_item_rev#17, item_id#27, cs_item_rev#28] +Condition : ((((cast(ss_item_rev#17 as decimal(19,3)) >= (0.9 * cs_item_rev#28)) AND (cast(ss_item_rev#17 as decimal(20,3)) <= (1.1 * cs_item_rev#28))) AND (cast(cs_item_rev#28 as decimal(19,3)) >= (0.9 * ss_item_rev#17))) AND (cast(cs_item_rev#28 as decimal(20,3)) <= (1.1 * ss_item_rev#17))) -(100) Project [codegen id : X] +(101) NativeProject Output [3]: [item_id#16, ss_item_rev#17, cs_item_rev#28] Input [4]: [item_id#16, ss_item_rev#17, item_id#27, cs_item_rev#28] -(209) Scan parquet +(211) Scan parquet Output [3]: [ws_sold_date_sk#29, ws_item_sk#30, ws_ext_sales_price#31] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(102) InputAdapter +(103) InputAdapter Input [3]: [ws_sold_date_sk#29, ws_item_sk#30, ws_ext_sales_price#31] Arguments: [#29, #30, #31] -(103) NativeFilter +(104) NativeFilter Input [3]: [#29#29, #30#30, #31#31] Condition : (isnotnull(ws_item_sk#30) AND isnotnull(ws_sold_date_sk#29)) -(104) NativeShuffleExchange +(105) NativeShuffleExchange Input [3]: [#29#29, #30#30, #31#31] Arguments: hashpartitioning(ws_item_sk#30, 100), ENSURE_REQUIREMENTS, [plan_id=11] -(105) ShuffleQueryStage +(106) ShuffleQueryStage Output [3]: [#29#29, #30#30, #31#31] Arguments: X -(106) AQEShuffleRead +(107) AQEShuffleRead Input [3]: [#29#29, #30#30, #31#31] Arguments: coalesced -(107) InputAdapter +(108) InputAdapter Input [3]: [#29#29, #30#30, #31#31] -(108) NativeSort +(109) NativeSort Input [3]: [#29#29, #30#30, #31#31] Arguments: [ws_item_sk#30 ASC NULLS FIRST], false -(109) ReusedExchange [Reuses operator id: 12] +(110) ReusedExchange [Reuses operator id: 12] Output [2]: [i_item_sk#32, i_item_id#33] -(110) ShuffleQueryStage +(111) ShuffleQueryStage Output [2]: [i_item_sk#32, i_item_id#33] Arguments: X -(111) AQEShuffleRead +(112) AQEShuffleRead Input [2]: [i_item_sk#32, i_item_id#33] Arguments: coalesced -(112) InputAdapter +(113) InputAdapter Input [2]: [i_item_sk#32, i_item_id#33] Arguments: [#32, #33] -(113) InputAdapter +(114) InputAdapter Input [2]: [#32#32, #33#33] -(114) NativeSort +(115) NativeSort Input [2]: [#32#32, #33#33] Arguments: [i_item_sk#32 ASC NULLS FIRST], false -(115) NativeSortMergeJoin +(116) NativeSortMergeJoin Left keys [1]: [ws_item_sk#30] Right keys [1]: [i_item_sk#32] Join type: Inner Join condition: None -(116) NativeProject +(117) NativeProject Output [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] Input [5]: [#29#29, #30#30, #31#31, #32#32, #33#33] -(117) NativeShuffleExchange +(118) NativeShuffleExchange Input [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] Arguments: hashpartitioning(ws_sold_date_sk#29, 100), ENSURE_REQUIREMENTS, [plan_id=12] -(118) ShuffleQueryStage +(119) ShuffleQueryStage Output [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] Arguments: X -(119) AQEShuffleRead +(120) AQEShuffleRead Input [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] Arguments: coalesced -(120) InputAdapter +(121) InputAdapter Input [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] -(121) NativeSort +(122) NativeSort Input [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] Arguments: [ws_sold_date_sk#29 ASC NULLS FIRST], false -(122) ReusedExchange [Reuses operator id: 43] +(123) ReusedExchange [Reuses operator id: 43] Output [1]: [d_date_sk#34] -(123) ShuffleQueryStage +(124) ShuffleQueryStage Output [1]: [d_date_sk#34] Arguments: X -(124) AQEShuffleRead +(125) AQEShuffleRead Input [1]: [d_date_sk#34] Arguments: coalesced -(125) InputAdapter +(126) InputAdapter Input [1]: [d_date_sk#34] Arguments: [#34] -(126) InputAdapter +(127) InputAdapter Input [1]: [#34#34] -(127) NativeSort +(128) NativeSort Input [1]: [#34#34] Arguments: [d_date_sk#34 ASC NULLS FIRST], false -(128) NativeSortMergeJoin +(129) NativeSortMergeJoin Left keys [1]: [ws_sold_date_sk#29] Right keys [1]: [d_date_sk#34] Join type: Inner Join condition: None -(129) NativeProject +(130) NativeProject Output [2]: [ws_ext_sales_price#31, i_item_id#33] Input [4]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33, #34#34] -(130) NativeProject +(131) NativeProject Output [2]: [i_item_id#33 AS i_item_id#33, UnscaledValue(ws_ext_sales_price#31) AS _c1#35] Input [2]: [ws_ext_sales_price#31, i_item_id#33] -(131) NativeHashAggregate +(132) NativeHashAggregate Input [2]: [i_item_id#33, _c1#35] Keys [1]: [i_item_id#33] Functions [1]: [partial_sum(_c1#35)] Aggregate Attributes [1]: [sum#36] Results [2]: [i_item_id#33, #14] -(132) NativeShuffleExchange +(133) NativeShuffleExchange Input [2]: [i_item_id#33, #14] Arguments: hashpartitioning(i_item_id#33, 100), ENSURE_REQUIREMENTS, [plan_id=13] -(133) ShuffleQueryStage +(134) ShuffleQueryStage Output [2]: [i_item_id#33, #14] Arguments: X -(134) AQEShuffleRead +(135) AQEShuffleRead Input [2]: [i_item_id#33, #14] Arguments: coalesced -(135) InputAdapter +(136) InputAdapter Input [2]: [i_item_id#33, #14] -(136) NativeHashAggregate +(137) NativeHashAggregate Input [2]: [i_item_id#33, #14] Keys [1]: [i_item_id#33] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#31))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#31))#37] Results [2]: [i_item_id#33, sum(UnscaledValue(ws_ext_sales_price#31))#37] -(137) NativeProject +(138) NativeProject Output [2]: [i_item_id#33 AS item_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#31))#37,17,2) AS ws_item_rev#39] Input [2]: [i_item_id#33, sum(UnscaledValue(ws_ext_sales_price#31))#37] -(138) NativeFilter +(139) NativeFilter Input [2]: [item_id#38, ws_item_rev#39] Condition : isnotnull(ws_item_rev#39) -(139) NativeSort +(140) NativeSort Input [2]: [item_id#38, ws_item_rev#39] Arguments: [item_id#38 ASC NULLS FIRST], false -(140) SortMergeJoin [codegen id : X] +(141) NativeSortMergeJoin Left keys [1]: [item_id#16] Right keys [1]: [item_id#38] Join type: Inner -Join condition: ((((((((cast(ss_item_rev#17 as decimal(19,3)) >= (0.9 * ws_item_rev#39)) AND (cast(ss_item_rev#17 as decimal(20,3)) <= (1.1 * ws_item_rev#39))) AND (cast(cs_item_rev#28 as decimal(19,3)) >= (0.9 * ws_item_rev#39))) AND (cast(cs_item_rev#28 as decimal(20,3)) <= (1.1 * ws_item_rev#39))) AND (cast(ws_item_rev#39 as decimal(19,3)) >= (0.9 * ss_item_rev#17))) AND (cast(ws_item_rev#39 as decimal(20,3)) <= (1.1 * ss_item_rev#17))) AND (cast(ws_item_rev#39 as decimal(19,3)) >= (0.9 * cs_item_rev#28))) AND (cast(ws_item_rev#39 as decimal(20,3)) <= (1.1 * cs_item_rev#28))) +Join condition: None + +(142) NativeFilter +Input [5]: [item_id#16, ss_item_rev#17, cs_item_rev#28, item_id#38, ws_item_rev#39] +Condition : ((((((((cast(ss_item_rev#17 as decimal(19,3)) >= (0.9 * ws_item_rev#39)) AND (cast(ss_item_rev#17 as decimal(20,3)) <= (1.1 * ws_item_rev#39))) AND (cast(cs_item_rev#28 as decimal(19,3)) >= (0.9 * ws_item_rev#39))) AND (cast(cs_item_rev#28 as decimal(20,3)) <= (1.1 * ws_item_rev#39))) AND (cast(ws_item_rev#39 as decimal(19,3)) >= (0.9 * ss_item_rev#17))) AND (cast(ws_item_rev#39 as decimal(20,3)) <= (1.1 * ss_item_rev#17))) AND (cast(ws_item_rev#39 as decimal(19,3)) >= (0.9 * cs_item_rev#28))) AND (cast(ws_item_rev#39 as decimal(20,3)) <= (1.1 * cs_item_rev#28))) -(141) Project [codegen id : X] +(143) NativeProject Output [8]: [item_id#16, ss_item_rev#17, (((ss_item_rev#17 / ((ss_item_rev#17 + cs_item_rev#28) + ws_item_rev#39)) / 3) * 100) AS ss_dev#40, cs_item_rev#28, (((cs_item_rev#28 / ((ss_item_rev#17 + cs_item_rev#28) + ws_item_rev#39)) / 3) * 100) AS cs_dev#41, ws_item_rev#39, (((ws_item_rev#39 / ((ss_item_rev#17 + cs_item_rev#28) + ws_item_rev#39)) / 3) * 100) AS ws_dev#42, (((ss_item_rev#17 + cs_item_rev#28) + ws_item_rev#39) / 3) AS average#43] Input [5]: [item_id#16, ss_item_rev#17, cs_item_rev#28, item_id#38, ws_item_rev#39] -(142) TakeOrderedAndProject +(144) NativeTakeOrdered Input [8]: [item_id#16, ss_item_rev#17, ss_dev#40, cs_item_rev#28, cs_dev#41, ws_item_rev#39, ws_dev#42, average#43] -Arguments: X, [item_id#16 ASC NULLS FIRST, ss_item_rev#17 ASC NULLS FIRST], [item_id#16, ss_item_rev#17, ss_dev#40, cs_item_rev#28, cs_dev#41, ws_item_rev#39, ws_dev#42, average#43] +Arguments: X, X, [item_id#16 ASC NULLS FIRST, ss_item_rev#17 ASC NULLS FIRST] -(143) Scan parquet +(145) Scan parquet Output [3]: [ss_sold_date_sk#1, ss_item_sk#2, ss_ext_sales_price#3] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(144) Filter +(146) Filter Input [3]: [ss_sold_date_sk#1, ss_item_sk#2, ss_ext_sales_price#3] Condition : (isnotnull(ss_item_sk#2) AND isnotnull(ss_sold_date_sk#1)) -(145) Exchange +(147) Exchange Input [3]: [ss_sold_date_sk#1, ss_item_sk#2, ss_ext_sales_price#3] Arguments: hashpartitioning(ss_item_sk#2, 100), ENSURE_REQUIREMENTS, [plan_id=14] -(146) Sort +(148) Sort Input [3]: [ss_sold_date_sk#1, ss_item_sk#2, ss_ext_sales_price#3] Arguments: [ss_item_sk#2 ASC NULLS FIRST], false, 0 -(147) Scan parquet +(149) Scan parquet Output [2]: [i_item_sk#4, i_item_id#5] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] ReadSchema: struct -(148) Filter +(150) Filter Input [2]: [i_item_sk#4, i_item_id#5] Condition : (isnotnull(i_item_sk#4) AND isnotnull(i_item_id#5)) -(149) Exchange +(151) Exchange Input [2]: [i_item_sk#4, i_item_id#5] Arguments: hashpartitioning(i_item_sk#4, 100), ENSURE_REQUIREMENTS, [plan_id=15] -(150) Sort +(152) Sort Input [2]: [i_item_sk#4, i_item_id#5] Arguments: [i_item_sk#4 ASC NULLS FIRST], false, 0 -(151) SortMergeJoin +(153) SortMergeJoin Left keys [1]: [ss_item_sk#2] Right keys [1]: [i_item_sk#4] Join type: Inner Join condition: None -(152) Project +(154) Project Output [3]: [ss_sold_date_sk#1, ss_ext_sales_price#3, i_item_id#5] Input [5]: [ss_sold_date_sk#1, ss_item_sk#2, ss_ext_sales_price#3, i_item_sk#4, i_item_id#5] -(153) Exchange +(155) Exchange Input [3]: [ss_sold_date_sk#1, ss_ext_sales_price#3, i_item_id#5] Arguments: hashpartitioning(ss_sold_date_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=16] -(154) Sort +(156) Sort Input [3]: [ss_sold_date_sk#1, ss_ext_sales_price#3, i_item_id#5] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false, 0 -(155) Scan parquet +(157) Scan parquet Output [2]: [d_date_sk#6, d_date#7] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(156) Filter +(158) Filter Input [2]: [d_date_sk#6, d_date#7] Condition : isnotnull(d_date_sk#6) -(157) Exchange +(159) Exchange Input [2]: [d_date_sk#6, d_date#7] Arguments: hashpartitioning(d_date#7, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(158) Sort +(160) Sort Input [2]: [d_date_sk#6, d_date#7] Arguments: [d_date#7 ASC NULLS FIRST], false, 0 -(159) Scan parquet +(161) Scan parquet Output [2]: [d_date#8, d_week_seq#9] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_week_seq)] ReadSchema: struct -(160) Filter +(162) Filter Input [2]: [d_date#8, d_week_seq#9] Condition : (isnotnull(d_week_seq#9) AND (d_week_seq#9 = Subquery subquery#10, [id=#11])) -(161) Project +(163) Project Output [1]: [d_date#8] Input [2]: [d_date#8, d_week_seq#9] -(162) Exchange +(164) Exchange Input [1]: [d_date#8] Arguments: hashpartitioning(d_date#8, 100), ENSURE_REQUIREMENTS, [plan_id=18] -(163) Sort +(165) Sort Input [1]: [d_date#8] Arguments: [d_date#8 ASC NULLS FIRST], false, 0 -(164) SortMergeJoin +(166) SortMergeJoin Left keys [1]: [d_date#7] Right keys [1]: [d_date#8] Join type: LeftSemi Join condition: None -(165) Project +(167) Project Output [1]: [d_date_sk#6] Input [2]: [d_date_sk#6, d_date#7] -(166) Exchange +(168) Exchange Input [1]: [d_date_sk#6] Arguments: hashpartitioning(d_date_sk#6, 100), ENSURE_REQUIREMENTS, [plan_id=19] -(167) Sort +(169) Sort Input [1]: [d_date_sk#6] Arguments: [d_date_sk#6 ASC NULLS FIRST], false, 0 -(168) SortMergeJoin +(170) SortMergeJoin Left keys [1]: [ss_sold_date_sk#1] Right keys [1]: [d_date_sk#6] Join type: Inner Join condition: None -(169) Project +(171) Project Output [2]: [ss_ext_sales_price#3, i_item_id#5] Input [4]: [ss_sold_date_sk#1, ss_ext_sales_price#3, i_item_id#5, d_date_sk#6] -(170) HashAggregate +(172) HashAggregate Input [2]: [ss_ext_sales_price#3, i_item_id#5] Keys [1]: [i_item_id#5] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum#13] Results [2]: [i_item_id#5, sum#44] -(171) Exchange +(173) Exchange Input [2]: [i_item_id#5, sum#44] Arguments: hashpartitioning(i_item_id#5, 100), ENSURE_REQUIREMENTS, [plan_id=20] -(172) HashAggregate +(174) HashAggregate Input [2]: [i_item_id#5, sum#44] Keys [1]: [i_item_id#5] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#15] Results [2]: [i_item_id#5 AS item_id#16, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#15,17,2) AS ss_item_rev#17] -(173) Filter +(175) Filter Input [2]: [item_id#16, ss_item_rev#17] Condition : isnotnull(ss_item_rev#17) -(174) Sort +(176) Sort Input [2]: [item_id#16, ss_item_rev#17] Arguments: [item_id#16 ASC NULLS FIRST], false, 0 -(175) Scan parquet +(177) Scan parquet Output [3]: [cs_sold_date_sk#18, cs_item_sk#19, cs_ext_sales_price#20] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_date_sk)] ReadSchema: struct -(176) Filter +(178) Filter Input [3]: [cs_sold_date_sk#18, cs_item_sk#19, cs_ext_sales_price#20] Condition : (isnotnull(cs_item_sk#19) AND isnotnull(cs_sold_date_sk#18)) -(177) Exchange +(179) Exchange Input [3]: [cs_sold_date_sk#18, cs_item_sk#19, cs_ext_sales_price#20] Arguments: hashpartitioning(cs_item_sk#19, 100), ENSURE_REQUIREMENTS, [plan_id=21] -(178) Sort +(180) Sort Input [3]: [cs_sold_date_sk#18, cs_item_sk#19, cs_ext_sales_price#20] Arguments: [cs_item_sk#19 ASC NULLS FIRST], false, 0 -(179) Scan parquet +(181) Scan parquet Output [2]: [i_item_sk#21, i_item_id#22] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] ReadSchema: struct -(180) Filter +(182) Filter Input [2]: [i_item_sk#21, i_item_id#22] Condition : (isnotnull(i_item_sk#21) AND isnotnull(i_item_id#22)) -(181) Exchange +(183) Exchange Input [2]: [i_item_sk#21, i_item_id#22] Arguments: hashpartitioning(i_item_sk#21, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(182) Sort +(184) Sort Input [2]: [i_item_sk#21, i_item_id#22] Arguments: [i_item_sk#21 ASC NULLS FIRST], false, 0 -(183) SortMergeJoin +(185) SortMergeJoin Left keys [1]: [cs_item_sk#19] Right keys [1]: [i_item_sk#21] Join type: Inner Join condition: None -(184) Project +(186) Project Output [3]: [cs_sold_date_sk#18, cs_ext_sales_price#20, i_item_id#22] Input [5]: [cs_sold_date_sk#18, cs_item_sk#19, cs_ext_sales_price#20, i_item_sk#21, i_item_id#22] -(185) Exchange +(187) Exchange Input [3]: [cs_sold_date_sk#18, cs_ext_sales_price#20, i_item_id#22] Arguments: hashpartitioning(cs_sold_date_sk#18, 100), ENSURE_REQUIREMENTS, [plan_id=23] -(186) Sort +(188) Sort Input [3]: [cs_sold_date_sk#18, cs_ext_sales_price#20, i_item_id#22] Arguments: [cs_sold_date_sk#18 ASC NULLS FIRST], false, 0 -(187) Scan parquet +(189) Scan parquet Output [2]: [d_date_sk#23, d_date#45] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(188) Filter +(190) Filter Input [2]: [d_date_sk#23, d_date#45] Condition : isnotnull(d_date_sk#23) -(189) Exchange +(191) Exchange Input [2]: [d_date_sk#23, d_date#45] Arguments: hashpartitioning(d_date#45, 100), ENSURE_REQUIREMENTS, [plan_id=24] -(190) Sort +(192) Sort Input [2]: [d_date_sk#23, d_date#45] Arguments: [d_date#45 ASC NULLS FIRST], false, 0 -(191) Scan parquet +(193) Scan parquet Output [2]: [d_date#46, d_week_seq#47] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_week_seq)] ReadSchema: struct -(192) Filter +(194) Filter Input [2]: [d_date#46, d_week_seq#47] Condition : (isnotnull(d_week_seq#47) AND (d_week_seq#47 = Subquery subquery#48, [id=#49])) -(193) Project +(195) Project Output [1]: [d_date#46] Input [2]: [d_date#46, d_week_seq#47] -(194) Exchange +(196) Exchange Input [1]: [d_date#46] Arguments: hashpartitioning(d_date#46, 100), ENSURE_REQUIREMENTS, [plan_id=25] -(195) Sort +(197) Sort Input [1]: [d_date#46] Arguments: [d_date#46 ASC NULLS FIRST], false, 0 -(196) SortMergeJoin +(198) SortMergeJoin Left keys [1]: [d_date#45] Right keys [1]: [d_date#46] Join type: LeftSemi Join condition: None -(197) Project +(199) Project Output [1]: [d_date_sk#23] Input [2]: [d_date_sk#23, d_date#45] -(198) Exchange +(200) Exchange Input [1]: [d_date_sk#23] Arguments: hashpartitioning(d_date_sk#23, 100), ENSURE_REQUIREMENTS, [plan_id=26] -(199) Sort +(201) Sort Input [1]: [d_date_sk#23] Arguments: [d_date_sk#23 ASC NULLS FIRST], false, 0 -(200) SortMergeJoin +(202) SortMergeJoin Left keys [1]: [cs_sold_date_sk#18] Right keys [1]: [d_date_sk#23] Join type: Inner Join condition: None -(201) Project +(203) Project Output [2]: [cs_ext_sales_price#20, i_item_id#22] Input [4]: [cs_sold_date_sk#18, cs_ext_sales_price#20, i_item_id#22, d_date_sk#23] -(202) HashAggregate +(204) HashAggregate Input [2]: [cs_ext_sales_price#20, i_item_id#22] Keys [1]: [i_item_id#22] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#20))] Aggregate Attributes [1]: [sum#25] Results [2]: [i_item_id#22, sum#50] -(203) Exchange +(205) Exchange Input [2]: [i_item_id#22, sum#50] Arguments: hashpartitioning(i_item_id#22, 100), ENSURE_REQUIREMENTS, [plan_id=27] -(204) HashAggregate +(206) HashAggregate Input [2]: [i_item_id#22, sum#50] Keys [1]: [i_item_id#22] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#20))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#20))#26] Results [2]: [i_item_id#22 AS item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#20))#26,17,2) AS cs_item_rev#28] -(205) Filter +(207) Filter Input [2]: [item_id#27, cs_item_rev#28] Condition : isnotnull(cs_item_rev#28) -(206) Sort +(208) Sort Input [2]: [item_id#27, cs_item_rev#28] Arguments: [item_id#27 ASC NULLS FIRST], false, 0 -(207) SortMergeJoin +(209) SortMergeJoin Left keys [1]: [item_id#16] Right keys [1]: [item_id#27] Join type: Inner Join condition: ((((cast(ss_item_rev#17 as decimal(19,3)) >= (0.9 * cs_item_rev#28)) AND (cast(ss_item_rev#17 as decimal(20,3)) <= (1.1 * cs_item_rev#28))) AND (cast(cs_item_rev#28 as decimal(19,3)) >= (0.9 * ss_item_rev#17))) AND (cast(cs_item_rev#28 as decimal(20,3)) <= (1.1 * ss_item_rev#17))) -(208) Project +(210) Project Output [3]: [item_id#16, ss_item_rev#17, cs_item_rev#28] Input [4]: [item_id#16, ss_item_rev#17, item_id#27, cs_item_rev#28] -(209) Scan parquet +(211) Scan parquet Output [3]: [ws_sold_date_sk#29, ws_item_sk#30, ws_ext_sales_price#31] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(210) Filter +(212) Filter Input [3]: [ws_sold_date_sk#29, ws_item_sk#30, ws_ext_sales_price#31] Condition : (isnotnull(ws_item_sk#30) AND isnotnull(ws_sold_date_sk#29)) -(211) Exchange +(213) Exchange Input [3]: [ws_sold_date_sk#29, ws_item_sk#30, ws_ext_sales_price#31] Arguments: hashpartitioning(ws_item_sk#30, 100), ENSURE_REQUIREMENTS, [plan_id=28] -(212) Sort +(214) Sort Input [3]: [ws_sold_date_sk#29, ws_item_sk#30, ws_ext_sales_price#31] Arguments: [ws_item_sk#30 ASC NULLS FIRST], false, 0 -(213) Scan parquet +(215) Scan parquet Output [2]: [i_item_sk#32, i_item_id#33] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_item_sk), IsNotNull(i_item_id)] ReadSchema: struct -(214) Filter +(216) Filter Input [2]: [i_item_sk#32, i_item_id#33] Condition : (isnotnull(i_item_sk#32) AND isnotnull(i_item_id#33)) -(215) Exchange +(217) Exchange Input [2]: [i_item_sk#32, i_item_id#33] Arguments: hashpartitioning(i_item_sk#32, 100), ENSURE_REQUIREMENTS, [plan_id=29] -(216) Sort +(218) Sort Input [2]: [i_item_sk#32, i_item_id#33] Arguments: [i_item_sk#32 ASC NULLS FIRST], false, 0 -(217) SortMergeJoin +(219) SortMergeJoin Left keys [1]: [ws_item_sk#30] Right keys [1]: [i_item_sk#32] Join type: Inner Join condition: None -(218) Project +(220) Project Output [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] Input [5]: [ws_sold_date_sk#29, ws_item_sk#30, ws_ext_sales_price#31, i_item_sk#32, i_item_id#33] -(219) Exchange +(221) Exchange Input [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] Arguments: hashpartitioning(ws_sold_date_sk#29, 100), ENSURE_REQUIREMENTS, [plan_id=30] -(220) Sort +(222) Sort Input [3]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33] Arguments: [ws_sold_date_sk#29 ASC NULLS FIRST], false, 0 -(221) Scan parquet +(223) Scan parquet Output [2]: [d_date_sk#34, d_date#51] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(222) Filter +(224) Filter Input [2]: [d_date_sk#34, d_date#51] Condition : isnotnull(d_date_sk#34) -(223) Exchange +(225) Exchange Input [2]: [d_date_sk#34, d_date#51] Arguments: hashpartitioning(d_date#51, 100), ENSURE_REQUIREMENTS, [plan_id=31] -(224) Sort +(226) Sort Input [2]: [d_date_sk#34, d_date#51] Arguments: [d_date#51 ASC NULLS FIRST], false, 0 -(225) Scan parquet +(227) Scan parquet Output [2]: [d_date#52, d_week_seq#53] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_week_seq)] ReadSchema: struct -(226) Filter +(228) Filter Input [2]: [d_date#52, d_week_seq#53] Condition : (isnotnull(d_week_seq#53) AND (d_week_seq#53 = Subquery subquery#54, [id=#55])) -(227) Project +(229) Project Output [1]: [d_date#52] Input [2]: [d_date#52, d_week_seq#53] -(228) Exchange +(230) Exchange Input [1]: [d_date#52] Arguments: hashpartitioning(d_date#52, 100), ENSURE_REQUIREMENTS, [plan_id=32] -(229) Sort +(231) Sort Input [1]: [d_date#52] Arguments: [d_date#52 ASC NULLS FIRST], false, 0 -(230) SortMergeJoin +(232) SortMergeJoin Left keys [1]: [d_date#51] Right keys [1]: [d_date#52] Join type: LeftSemi Join condition: None -(231) Project +(233) Project Output [1]: [d_date_sk#34] Input [2]: [d_date_sk#34, d_date#51] -(232) Exchange +(234) Exchange Input [1]: [d_date_sk#34] Arguments: hashpartitioning(d_date_sk#34, 100), ENSURE_REQUIREMENTS, [plan_id=33] -(233) Sort +(235) Sort Input [1]: [d_date_sk#34] Arguments: [d_date_sk#34 ASC NULLS FIRST], false, 0 -(234) SortMergeJoin +(236) SortMergeJoin Left keys [1]: [ws_sold_date_sk#29] Right keys [1]: [d_date_sk#34] Join type: Inner Join condition: None -(235) Project +(237) Project Output [2]: [ws_ext_sales_price#31, i_item_id#33] Input [4]: [ws_sold_date_sk#29, ws_ext_sales_price#31, i_item_id#33, d_date_sk#34] -(236) HashAggregate +(238) HashAggregate Input [2]: [ws_ext_sales_price#31, i_item_id#33] Keys [1]: [i_item_id#33] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#31))] Aggregate Attributes [1]: [sum#36] Results [2]: [i_item_id#33, sum#56] -(237) Exchange +(239) Exchange Input [2]: [i_item_id#33, sum#56] Arguments: hashpartitioning(i_item_id#33, 100), ENSURE_REQUIREMENTS, [plan_id=34] -(238) HashAggregate +(240) HashAggregate Input [2]: [i_item_id#33, sum#56] Keys [1]: [i_item_id#33] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#31))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#31))#37] Results [2]: [i_item_id#33 AS item_id#38, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#31))#37,17,2) AS ws_item_rev#39] -(239) Filter +(241) Filter Input [2]: [item_id#38, ws_item_rev#39] Condition : isnotnull(ws_item_rev#39) -(240) Sort +(242) Sort Input [2]: [item_id#38, ws_item_rev#39] Arguments: [item_id#38 ASC NULLS FIRST], false, 0 -(241) SortMergeJoin +(243) SortMergeJoin Left keys [1]: [item_id#16] Right keys [1]: [item_id#38] Join type: Inner Join condition: ((((((((cast(ss_item_rev#17 as decimal(19,3)) >= (0.9 * ws_item_rev#39)) AND (cast(ss_item_rev#17 as decimal(20,3)) <= (1.1 * ws_item_rev#39))) AND (cast(cs_item_rev#28 as decimal(19,3)) >= (0.9 * ws_item_rev#39))) AND (cast(cs_item_rev#28 as decimal(20,3)) <= (1.1 * ws_item_rev#39))) AND (cast(ws_item_rev#39 as decimal(19,3)) >= (0.9 * ss_item_rev#17))) AND (cast(ws_item_rev#39 as decimal(20,3)) <= (1.1 * ss_item_rev#17))) AND (cast(ws_item_rev#39 as decimal(19,3)) >= (0.9 * cs_item_rev#28))) AND (cast(ws_item_rev#39 as decimal(20,3)) <= (1.1 * cs_item_rev#28))) -(242) Project +(244) Project Output [8]: [item_id#16, ss_item_rev#17, (((ss_item_rev#17 / ((ss_item_rev#17 + cs_item_rev#28) + ws_item_rev#39)) / 3) * 100) AS ss_dev#40, cs_item_rev#28, (((cs_item_rev#28 / ((ss_item_rev#17 + cs_item_rev#28) + ws_item_rev#39)) / 3) * 100) AS cs_dev#41, ws_item_rev#39, (((ws_item_rev#39 / ((ss_item_rev#17 + cs_item_rev#28) + ws_item_rev#39)) / 3) * 100) AS ws_dev#42, (((ss_item_rev#17 + cs_item_rev#28) + ws_item_rev#39) / 3) AS average#43] Input [5]: [item_id#16, ss_item_rev#17, cs_item_rev#28, item_id#38, ws_item_rev#39] -(243) TakeOrderedAndProject +(245) TakeOrderedAndProject Input [8]: [item_id#16, ss_item_rev#17, ss_dev#40, cs_item_rev#28, cs_dev#41, ws_item_rev#39, ws_dev#42, average#43] Arguments: X, [item_id#16 ASC NULLS FIRST, ss_item_rev#17 ASC NULLS FIRST], [item_id#16, ss_item_rev#17, ss_dev#40, cs_item_rev#28, cs_dev#41, ws_item_rev#39, ws_dev#42, average#43] -(244) AdaptiveSparkPlan +(246) AdaptiveSparkPlan Output [8]: [item_id#16, ss_item_rev#17, ss_dev#40, cs_item_rev#28, cs_dev#41, ws_item_rev#39, ws_dev#42, average#43] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 34 Hosting Expression = Subquery subquery#10, [id=#11] -AdaptiveSparkPlan (252) +AdaptiveSparkPlan (254) +- == Final Plan == - NativeProject (248) - +- NativeFilter (247) - +- InputAdapter (246) - +- NativeParquetScan (245) + NativeProject (250) + +- NativeFilter (249) + +- InputAdapter (248) + +- NativeParquetScan (247) +- == Initial Plan == - Project (251) - +- Filter (250) - +- Scan parquet (249) + Project (253) + +- Filter (252) + +- Scan parquet (251) -(249) Scan parquet +(251) Scan parquet Output [2]: [d_date#57, d_week_seq#58] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(246) InputAdapter +(248) InputAdapter Input [2]: [d_date#57, d_week_seq#58] Arguments: [#57, #58] -(247) NativeFilter +(249) NativeFilter Input [2]: [#57#57, #58#58] Condition : (isnotnull(d_date#57) AND (d_date#57 = 2000-01-03)) -(248) NativeProject +(250) NativeProject Output [1]: [d_week_seq#58] Input [2]: [#57#57, #58#58] -(249) Scan parquet +(251) Scan parquet Output [2]: [d_date#57, d_week_seq#58] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(250) Filter +(252) Filter Input [2]: [d_date#57, d_week_seq#58] Condition : (isnotnull(d_date#57) AND (d_date#57 = 2000-01-03)) -(251) Project +(253) Project Output [1]: [d_week_seq#58] Input [2]: [d_date#57, d_week_seq#58] -(252) AdaptiveSparkPlan +(254) AdaptiveSparkPlan Output [1]: [d_week_seq#58] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q6.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q6.txt index 94c71db2d..486b49821 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q6.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q6.txt @@ -72,9 +72,9 @@ AdaptiveSparkPlan (147) +- AQEShuffleRead (81) +- ShuffleQueryStage (80), Statistics(X) +- NativeShuffleExchange (79) - +- ConvertToNative (78) - +- * Project (77) - +- * SortMergeJoin Inner (76) + +- NativeProject (78) + +- NativeFilter (77) + +- NativeSortMergeJoin Inner (76) :- NativeSort (62) : +- InputAdapter (61) : +- AQEShuffleRead (60) @@ -471,18 +471,19 @@ Condition : isnotnull(avg(i_current_price)#23) Input [2]: [avg(i_current_price)#23, i_category#16] Arguments: [i_category#16 ASC NULLS FIRST], false -(76) SortMergeJoin [codegen id : X] +(76) NativeSortMergeJoin Left keys [1]: [i_category#14] Right keys [1]: [i_category#16] Join type: Inner -Join condition: (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#23)) +Join condition: None -(77) Project [codegen id : X] -Output [1]: [i_item_sk#12] +(77) NativeFilter Input [5]: [#12#12, #13#13, #14#14, avg(i_current_price)#23, i_category#16] +Condition : (cast(i_current_price#13 as decimal(14,7)) > (1.2 * avg(i_current_price)#23)) -(78) ConvertToNative -Input [1]: [i_item_sk#12] +(78) NativeProject +Output [1]: [i_item_sk#12] +Input [5]: [#12#12, #13#13, #14#14, avg(i_current_price)#23, i_category#16] (79) NativeShuffleExchange Input [1]: [i_item_sk#12] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q64.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q64.txt index 830c34425..b418ae0cb 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q64.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q64.txt @@ -6,9 +6,9 @@ AdaptiveSparkPlan (811) +- AQEShuffleRead (494) +- ShuffleQueryStage (493), Statistics(X) +- NativeShuffleExchange (492) - +- ConvertToNative (491) - +- * Project (490) - +- * SortMergeJoin Inner (489) + +- NativeProject (491) + +- NativeFilter (490) + +- NativeSortMergeJoin Inner (489) :- NativeSort (280) : +- InputAdapter (279) : +- AQEShuffleRead (278) @@ -74,9 +74,9 @@ AdaptiveSparkPlan (811) : : : : : : : : : +- AQEShuffleRead (159) : : : : : : : : : +- ShuffleQueryStage (158), Statistics(X) : : : : : : : : : +- NativeShuffleExchange (157) - : : : : : : : : : +- ConvertToNative (156) - : : : : : : : : : +- * Project (155) - : : : : : : : : : +- * SortMergeJoin Inner (154) + : : : : : : : : : +- NativeProject (156) + : : : : : : : : : +- NativeFilter (155) + : : : : : : : : : +- NativeSortMergeJoin Inner (154) : : : : : : : : : :- NativeSort (147) : : : : : : : : : : +- InputAdapter (146) : : : : : : : : : : +- AQEShuffleRead (145) @@ -354,9 +354,9 @@ AdaptiveSparkPlan (811) : : : : : : : : +- AQEShuffleRead (378) : : : : : : : : +- ShuffleQueryStage (377), Statistics(X) : : : : : : : : +- NativeShuffleExchange (376) - : : : : : : : : +- ConvertToNative (375) - : : : : : : : : +- * Project (374) - : : : : : : : : +- * SortMergeJoin Inner (373) + : : : : : : : : +- NativeProject (375) + : : : : : : : : +- NativeFilter (374) + : : : : : : : : +- NativeSortMergeJoin Inner (373) : : : : : : : : :- NativeSort (366) : : : : : : : : : +- InputAdapter (365) : : : : : : : : : +- AQEShuffleRead (364) @@ -1455,18 +1455,19 @@ Input [2]: [#51#51, #52#52] Input [2]: [#51#51, #52#52] Arguments: [cd_demo_sk#51 ASC NULLS FIRST], false -(154) SortMergeJoin [codegen id : X] +(154) NativeSortMergeJoin Left keys [1]: [c_current_cdemo_sk#40] Right keys [1]: [cd_demo_sk#51] Join type: Inner -Join condition: NOT (cd_marital_status#50 = cd_marital_status#52) +Join condition: None -(155) Project [codegen id : X] -Output [14]: [ss_item_sk#2, ss_hdemo_sk#5, ss_addr_sk#6, ss_promo_sk#8, ss_wholesale_cost#10, ss_list_price#11, ss_coupon_amt#12, d_year#35, s_store_name#37, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +(155) NativeFilter Input [18]: [ss_item_sk#2, ss_hdemo_sk#5, ss_addr_sk#6, ss_promo_sk#8, ss_wholesale_cost#10, ss_list_price#11, ss_coupon_amt#12, d_year#35, s_store_name#37, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#50, #51#51, #52#52] +Condition : NOT (cd_marital_status#50 = cd_marital_status#52) -(156) ConvertToNative -Input [14]: [ss_item_sk#2, ss_hdemo_sk#5, ss_addr_sk#6, ss_promo_sk#8, ss_wholesale_cost#10, ss_list_price#11, ss_coupon_amt#12, d_year#35, s_store_name#37, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +(156) NativeProject +Output [14]: [ss_item_sk#2, ss_hdemo_sk#5, ss_addr_sk#6, ss_promo_sk#8, ss_wholesale_cost#10, ss_list_price#11, ss_coupon_amt#12, d_year#35, s_store_name#37, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] +Input [18]: [ss_item_sk#2, ss_hdemo_sk#5, ss_addr_sk#6, ss_promo_sk#8, ss_wholesale_cost#10, ss_list_price#11, ss_coupon_amt#12, d_year#35, s_store_name#37, s_zip#38, c_current_cdemo_sk#40, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48, cd_marital_status#50, #51#51, #52#52] (157) NativeShuffleExchange Input [14]: [ss_item_sk#2, ss_hdemo_sk#5, ss_addr_sk#6, ss_promo_sk#8, ss_wholesale_cost#10, ss_list_price#11, ss_coupon_amt#12, d_year#35, s_store_name#37, s_zip#38, c_current_hdemo_sk#41, c_current_addr_sk#42, d_year#46, d_year#48] @@ -2343,18 +2344,19 @@ Input [2]: [#130#130, #131#131] Input [2]: [#130#130, #131#131] Arguments: [cd_demo_sk#130 ASC NULLS FIRST], false -(373) SortMergeJoin [codegen id : X] +(373) NativeSortMergeJoin Left keys [1]: [c_current_cdemo_sk#119] Right keys [1]: [cd_demo_sk#130] Join type: Inner -Join condition: NOT (cd_marital_status#129 = cd_marital_status#131) +Join condition: None -(374) Project [codegen id : X] -Output [14]: [ss_item_sk#103, ss_hdemo_sk#106, ss_addr_sk#107, ss_promo_sk#109, ss_wholesale_cost#110, ss_list_price#111, ss_coupon_amt#112, d_year#114, s_store_name#116, s_zip#117, c_current_hdemo_sk#120, c_current_addr_sk#121, d_year#125, d_year#127] +(374) NativeFilter Input [18]: [ss_item_sk#103, ss_hdemo_sk#106, ss_addr_sk#107, ss_promo_sk#109, ss_wholesale_cost#110, ss_list_price#111, ss_coupon_amt#112, d_year#114, s_store_name#116, s_zip#117, c_current_cdemo_sk#119, c_current_hdemo_sk#120, c_current_addr_sk#121, d_year#125, d_year#127, cd_marital_status#129, #130#130, #131#131] +Condition : NOT (cd_marital_status#129 = cd_marital_status#131) -(375) ConvertToNative -Input [14]: [ss_item_sk#103, ss_hdemo_sk#106, ss_addr_sk#107, ss_promo_sk#109, ss_wholesale_cost#110, ss_list_price#111, ss_coupon_amt#112, d_year#114, s_store_name#116, s_zip#117, c_current_hdemo_sk#120, c_current_addr_sk#121, d_year#125, d_year#127] +(375) NativeProject +Output [14]: [ss_item_sk#103, ss_hdemo_sk#106, ss_addr_sk#107, ss_promo_sk#109, ss_wholesale_cost#110, ss_list_price#111, ss_coupon_amt#112, d_year#114, s_store_name#116, s_zip#117, c_current_hdemo_sk#120, c_current_addr_sk#121, d_year#125, d_year#127] +Input [18]: [ss_item_sk#103, ss_hdemo_sk#106, ss_addr_sk#107, ss_promo_sk#109, ss_wholesale_cost#110, ss_list_price#111, ss_coupon_amt#112, d_year#114, s_store_name#116, s_zip#117, c_current_cdemo_sk#119, c_current_hdemo_sk#120, c_current_addr_sk#121, d_year#125, d_year#127, cd_marital_status#129, #130#130, #131#131] (376) NativeShuffleExchange Input [14]: [ss_item_sk#103, ss_hdemo_sk#106, ss_addr_sk#107, ss_promo_sk#109, ss_wholesale_cost#110, ss_list_price#111, ss_coupon_amt#112, d_year#114, s_store_name#116, s_zip#117, c_current_hdemo_sk#120, c_current_addr_sk#121, d_year#125, d_year#127] @@ -2805,18 +2807,19 @@ Input [8]: [item_sk#157, store_name#158, store_zip#159, syear#160, cnt#161, s1#1 Input [8]: [item_sk#157, store_name#158, store_zip#159, syear#160, cnt#161, s1#162, s2#163, s3#164] Arguments: [item_sk#157 ASC NULLS FIRST, store_name#158 ASC NULLS FIRST, store_zip#159 ASC NULLS FIRST], false -(489) SortMergeJoin [codegen id : X] +(489) NativeSortMergeJoin Left keys [3]: [item_sk#86, store_name#87, store_zip#88] Right keys [3]: [item_sk#157, store_name#158, store_zip#159] Join type: Inner -Join condition: (cnt#161 <= cnt#98) +Join condition: None -(490) Project [codegen id : X] -Output [21]: [product_name#85, store_name#87, store_zip#88, b_street_number#89, b_streen_name#90, b_city#91, b_zip#92, c_street_number#93, c_street_name#94, c_city#95, c_zip#96, syear#97, cnt#98, s1#99, s2#100, s3#101, s1#162, s2#163, s3#164, syear#160, cnt#161] +(490) NativeFilter Input [25]: [product_name#85, item_sk#86, store_name#87, store_zip#88, b_street_number#89, b_streen_name#90, b_city#91, b_zip#92, c_street_number#93, c_street_name#94, c_city#95, c_zip#96, syear#97, cnt#98, s1#99, s2#100, s3#101, item_sk#157, store_name#158, store_zip#159, syear#160, cnt#161, s1#162, s2#163, s3#164] +Condition : (cnt#161 <= cnt#98) -(491) ConvertToNative -Input [21]: [product_name#85, store_name#87, store_zip#88, b_street_number#89, b_streen_name#90, b_city#91, b_zip#92, c_street_number#93, c_street_name#94, c_city#95, c_zip#96, syear#97, cnt#98, s1#99, s2#100, s3#101, s1#162, s2#163, s3#164, syear#160, cnt#161] +(491) NativeProject +Output [21]: [product_name#85, store_name#87, store_zip#88, b_street_number#89, b_streen_name#90, b_city#91, b_zip#92, c_street_number#93, c_street_name#94, c_city#95, c_zip#96, syear#97, cnt#98, s1#99, s2#100, s3#101, s1#162, s2#163, s3#164, syear#160, cnt#161] +Input [25]: [product_name#85, item_sk#86, store_name#87, store_zip#88, b_street_number#89, b_streen_name#90, b_city#91, b_zip#92, c_street_number#93, c_street_name#94, c_city#95, c_zip#96, syear#97, cnt#98, s1#99, s2#100, s3#101, item_sk#157, store_name#158, store_zip#159, syear#160, cnt#161, s1#162, s2#163, s3#164] (492) NativeShuffleExchange Input [21]: [product_name#85, store_name#87, store_zip#88, b_street_number#89, b_streen_name#90, b_city#91, b_zip#92, c_street_number#93, c_street_name#94, c_city#95, c_zip#96, syear#97, cnt#98, s1#99, s2#100, s3#101, s1#162, s2#163, s3#164, syear#160, cnt#161] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q65.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q65.txt index aa22922d9..4cba19f04 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q65.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q65.txt @@ -1,164 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (155) +AdaptiveSparkPlan (156) +- == Final Plan == - TakeOrderedAndProject (99) - +- * Project (98) - +- * SortMergeJoin Inner (97) - :- NativeSort (63) - : +- InputAdapter (62) - : +- AQEShuffleRead (61) - : +- ShuffleQueryStage (60), Statistics(X) - : +- NativeShuffleExchange (59) - : +- NativeProject (58) - : +- NativeSortMergeJoin Inner (57) - : :- NativeSort (48) - : : +- InputAdapter (47) - : : +- AQEShuffleRead (46) - : : +- ShuffleQueryStage (45), Statistics(X) - : : +- NativeShuffleExchange (44) - : : +- NativeProject (43) - : : +- NativeSortMergeJoin Inner (42) - : : :- NativeSort (8) - : : : +- InputAdapter (7) - : : : +- AQEShuffleRead (6) - : : : +- ShuffleQueryStage (5), Statistics(X) - : : : +- NativeShuffleExchange (4) - : : : +- NativeFilter (3) - : : : +- InputAdapter (2) - : : : +- NativeParquetScan (1) - : : +- NativeSort (41) - : : +- InputAdapter (40) - : : +- AQEShuffleRead (39) - : : +- ShuffleQueryStage (38), Statistics(X) - : : +- NativeShuffleExchange (37) - : : +- NativeFilter (36) - : : +- NativeProject (35) - : : +- NativeHashAggregate (34) - : : +- InputAdapter (33) - : : +- AQEShuffleRead (32) - : : +- ShuffleQueryStage (31), Statistics(X) - : : +- NativeShuffleExchange (30) - : : +- NativeHashAggregate (29) - : : +- NativeProject (28) - : : +- NativeProject (27) - : : +- NativeSortMergeJoin Inner (26) - : : :- NativeSort (16) - : : : +- InputAdapter (15) - : : : +- AQEShuffleRead (14) - : : : +- ShuffleQueryStage (13), Statistics(X) - : : : +- NativeShuffleExchange (12) - : : : +- NativeFilter (11) - : : : +- InputAdapter (10) - : : : +- NativeParquetScan (9) - : : +- NativeSort (25) - : : +- InputAdapter (24) - : : +- AQEShuffleRead (23) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- NativeShuffleExchange (21) - : : +- NativeProject (20) - : : +- NativeFilter (19) - : : +- InputAdapter (18) - : : +- NativeParquetScan (17) - : +- NativeSort (56) - : +- InputAdapter (55) - : +- AQEShuffleRead (54) - : +- ShuffleQueryStage (53), Statistics(X) - : +- NativeShuffleExchange (52) - : +- NativeFilter (51) - : +- InputAdapter (50) - : +- NativeParquetScan (49) - +- NativeSort (96) - +- NativeFilter (95) - +- NativeProject (94) - +- NativeHashAggregate (93) - +- InputAdapter (92) - +- AQEShuffleRead (91) - +- ShuffleQueryStage (90), Statistics(X) - +- NativeShuffleExchange (89) - +- NativeHashAggregate (88) - +- NativeProject (87) - +- NativeHashAggregate (86) - +- InputAdapter (85) - +- AQEShuffleRead (84) - +- ShuffleQueryStage (83), Statistics(X) - +- NativeShuffleExchange (82) - +- NativeHashAggregate (81) - +- NativeProject (80) - +- NativeProject (79) - +- NativeSortMergeJoin Inner (78) - :- NativeSort (71) - : +- InputAdapter (70) - : +- AQEShuffleRead (69) - : +- ShuffleQueryStage (68), Statistics(X) - : +- NativeShuffleExchange (67) - : +- NativeFilter (66) - : +- InputAdapter (65) - : +- NativeParquetScan (64) - +- NativeSort (77) - +- InputAdapter (76) - +- InputAdapter (75) - +- AQEShuffleRead (74) - +- ShuffleQueryStage (73), Statistics(X) - +- ReusedExchange (72) + NativeTakeOrdered (100) + +- NativeProject (99) + +- NativeFilter (98) + +- NativeSortMergeJoin Inner (97) + :- NativeSort (63) + : +- InputAdapter (62) + : +- AQEShuffleRead (61) + : +- ShuffleQueryStage (60), Statistics(X) + : +- NativeShuffleExchange (59) + : +- NativeProject (58) + : +- NativeSortMergeJoin Inner (57) + : :- NativeSort (48) + : : +- InputAdapter (47) + : : +- AQEShuffleRead (46) + : : +- ShuffleQueryStage (45), Statistics(X) + : : +- NativeShuffleExchange (44) + : : +- NativeProject (43) + : : +- NativeSortMergeJoin Inner (42) + : : :- NativeSort (8) + : : : +- InputAdapter (7) + : : : +- AQEShuffleRead (6) + : : : +- ShuffleQueryStage (5), Statistics(X) + : : : +- NativeShuffleExchange (4) + : : : +- NativeFilter (3) + : : : +- InputAdapter (2) + : : : +- NativeParquetScan (1) + : : +- NativeSort (41) + : : +- InputAdapter (40) + : : +- AQEShuffleRead (39) + : : +- ShuffleQueryStage (38), Statistics(X) + : : +- NativeShuffleExchange (37) + : : +- NativeFilter (36) + : : +- NativeProject (35) + : : +- NativeHashAggregate (34) + : : +- InputAdapter (33) + : : +- AQEShuffleRead (32) + : : +- ShuffleQueryStage (31), Statistics(X) + : : +- NativeShuffleExchange (30) + : : +- NativeHashAggregate (29) + : : +- NativeProject (28) + : : +- NativeProject (27) + : : +- NativeSortMergeJoin Inner (26) + : : :- NativeSort (16) + : : : +- InputAdapter (15) + : : : +- AQEShuffleRead (14) + : : : +- ShuffleQueryStage (13), Statistics(X) + : : : +- NativeShuffleExchange (12) + : : : +- NativeFilter (11) + : : : +- InputAdapter (10) + : : : +- NativeParquetScan (9) + : : +- NativeSort (25) + : : +- InputAdapter (24) + : : +- AQEShuffleRead (23) + : : +- ShuffleQueryStage (22), Statistics(X) + : : +- NativeShuffleExchange (21) + : : +- NativeProject (20) + : : +- NativeFilter (19) + : : +- InputAdapter (18) + : : +- NativeParquetScan (17) + : +- NativeSort (56) + : +- InputAdapter (55) + : +- AQEShuffleRead (54) + : +- ShuffleQueryStage (53), Statistics(X) + : +- NativeShuffleExchange (52) + : +- NativeFilter (51) + : +- InputAdapter (50) + : +- NativeParquetScan (49) + +- NativeSort (96) + +- NativeFilter (95) + +- NativeProject (94) + +- NativeHashAggregate (93) + +- InputAdapter (92) + +- AQEShuffleRead (91) + +- ShuffleQueryStage (90), Statistics(X) + +- NativeShuffleExchange (89) + +- NativeHashAggregate (88) + +- NativeProject (87) + +- NativeHashAggregate (86) + +- InputAdapter (85) + +- AQEShuffleRead (84) + +- ShuffleQueryStage (83), Statistics(X) + +- NativeShuffleExchange (82) + +- NativeHashAggregate (81) + +- NativeProject (80) + +- NativeProject (79) + +- NativeSortMergeJoin Inner (78) + :- NativeSort (71) + : +- InputAdapter (70) + : +- AQEShuffleRead (69) + : +- ShuffleQueryStage (68), Statistics(X) + : +- NativeShuffleExchange (67) + : +- NativeFilter (66) + : +- InputAdapter (65) + : +- NativeParquetScan (64) + +- NativeSort (77) + +- InputAdapter (76) + +- InputAdapter (75) + +- AQEShuffleRead (74) + +- ShuffleQueryStage (73), Statistics(X) + +- ReusedExchange (72) +- == Initial Plan == - TakeOrderedAndProject (154) - +- Project (153) - +- SortMergeJoin Inner (152) - :- Sort (132) - : +- Exchange (131) - : +- Project (130) - : +- SortMergeJoin Inner (129) - : :- Sort (124) - : : +- Exchange (123) - : : +- Project (122) - : : +- SortMergeJoin Inner (121) - : : :- Sort (103) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Sort (120) - : : +- Exchange (119) - : : +- Filter (118) - : : +- HashAggregate (117) - : : +- Exchange (116) - : : +- HashAggregate (115) - : : +- Project (114) - : : +- SortMergeJoin Inner (113) - : : :- Sort (107) - : : : +- Exchange (106) - : : : +- Filter (105) - : : : +- Scan parquet (104) - : : +- Sort (112) - : : +- Exchange (111) - : : +- Project (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- Sort (128) - : +- Exchange (127) - : +- Filter (126) - : +- Scan parquet (125) - +- Sort (151) - +- Filter (150) - +- HashAggregate (149) - +- Exchange (148) - +- HashAggregate (147) - +- HashAggregate (146) - +- Exchange (145) - +- HashAggregate (144) - +- Project (143) - +- SortMergeJoin Inner (142) - :- Sort (136) - : +- Exchange (135) - : +- Filter (134) - : +- Scan parquet (133) - +- Sort (141) - +- Exchange (140) - +- Project (139) - +- Filter (138) - +- Scan parquet (137) - - -(100) Scan parquet + TakeOrderedAndProject (155) + +- Project (154) + +- SortMergeJoin Inner (153) + :- Sort (133) + : +- Exchange (132) + : +- Project (131) + : +- SortMergeJoin Inner (130) + : :- Sort (125) + : : +- Exchange (124) + : : +- Project (123) + : : +- SortMergeJoin Inner (122) + : : :- Sort (104) + : : : +- Exchange (103) + : : : +- Filter (102) + : : : +- Scan parquet (101) + : : +- Sort (121) + : : +- Exchange (120) + : : +- Filter (119) + : : +- HashAggregate (118) + : : +- Exchange (117) + : : +- HashAggregate (116) + : : +- Project (115) + : : +- SortMergeJoin Inner (114) + : : :- Sort (108) + : : : +- Exchange (107) + : : : +- Filter (106) + : : : +- Scan parquet (105) + : : +- Sort (113) + : : +- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Sort (129) + : +- Exchange (128) + : +- Filter (127) + : +- Scan parquet (126) + +- Sort (152) + +- Filter (151) + +- HashAggregate (150) + +- Exchange (149) + +- HashAggregate (148) + +- HashAggregate (147) + +- Exchange (146) + +- HashAggregate (145) + +- Project (144) + +- SortMergeJoin Inner (143) + :- Sort (137) + : +- Exchange (136) + : +- Filter (135) + : +- Scan parquet (134) + +- Sort (142) + +- Exchange (141) + +- Project (140) + +- Filter (139) + +- Scan parquet (138) + + +(101) Scan parquet Output [2]: [s_store_sk#1, s_store_name#2] Batched: true Location: InMemoryFileIndex [file:/] @@ -192,7 +193,7 @@ Input [2]: [#1#1, #2#2] Input [2]: [#1#1, #2#2] Arguments: [s_store_sk#1 ASC NULLS FIRST], false -(104) Scan parquet +(105) Scan parquet Output [4]: [ss_sold_date_sk#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6] Batched: true Location: InMemoryFileIndex [file:/] @@ -226,7 +227,7 @@ Input [4]: [#3#3, #4#4, #5#5, #6#6] Input [4]: [#3#3, #4#4, #5#5, #6#6] Arguments: [ss_sold_date_sk#3 ASC NULLS FIRST], false -(108) Scan parquet +(109) Scan parquet Output [2]: [d_date_sk#7, d_month_seq#8] Batched: true Location: InMemoryFileIndex [file:/] @@ -363,7 +364,7 @@ Input [4]: [s_store_name#2, ss_store_sk#5, ss_item_sk#4, revenue#13] Input [4]: [s_store_name#2, ss_store_sk#5, ss_item_sk#4, revenue#13] Arguments: [ss_item_sk#4 ASC NULLS FIRST], false -(125) Scan parquet +(126) Scan parquet Output [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Batched: true Location: InMemoryFileIndex [file:/] @@ -426,7 +427,7 @@ Input [7]: [s_store_name#2, ss_store_sk#5, revenue#13, i_item_desc#15, i_current Input [7]: [s_store_name#2, ss_store_sk#5, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false -(133) Scan parquet +(134) Scan parquet Output [4]: [ss_sold_date_sk#19, ss_item_sk#20, ss_store_sk#21, ss_sales_price#22] Batched: true Location: InMemoryFileIndex [file:/] @@ -570,287 +571,291 @@ Condition : isnotnull(ave#32) Input [2]: [ss_store_sk#21, ave#32] Arguments: [ss_store_sk#21 ASC NULLS FIRST], false -(97) SortMergeJoin [codegen id : X] +(97) NativeSortMergeJoin Left keys [1]: [ss_store_sk#5] Right keys [1]: [ss_store_sk#21] Join type: Inner -Join condition: (cast(revenue#13 as decimal(23,7)) <= (0.1 * ave#32)) +Join condition: None + +(98) NativeFilter +Input [9]: [s_store_name#2, ss_store_sk#5, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18, ss_store_sk#21, ave#32] +Condition : (cast(revenue#13 as decimal(23,7)) <= (0.1 * ave#32)) -(98) Project [codegen id : X] +(99) NativeProject Output [6]: [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] Input [9]: [s_store_name#2, ss_store_sk#5, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18, ss_store_sk#21, ave#32] -(99) TakeOrderedAndProject +(100) NativeTakeOrdered Input [6]: [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] -Arguments: X, [s_store_name#2 ASC NULLS FIRST, i_item_desc#15 ASC NULLS FIRST], [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] +Arguments: X, X, [s_store_name#2 ASC NULLS FIRST, i_item_desc#15 ASC NULLS FIRST] -(100) Scan parquet +(101) Scan parquet Output [2]: [s_store_sk#1, s_store_name#2] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct -(101) Filter +(102) Filter Input [2]: [s_store_sk#1, s_store_name#2] Condition : isnotnull(s_store_sk#1) -(102) Exchange +(103) Exchange Input [2]: [s_store_sk#1, s_store_name#2] Arguments: hashpartitioning(s_store_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=12] -(103) Sort +(104) Sort Input [2]: [s_store_sk#1, s_store_name#2] Arguments: [s_store_sk#1 ASC NULLS FIRST], false, 0 -(104) Scan parquet +(105) Scan parquet Output [4]: [ss_sold_date_sk#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(105) Filter +(106) Filter Input [4]: [ss_sold_date_sk#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6] Condition : ((isnotnull(ss_sold_date_sk#3) AND isnotnull(ss_store_sk#5)) AND isnotnull(ss_item_sk#4)) -(106) Exchange +(107) Exchange Input [4]: [ss_sold_date_sk#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6] Arguments: hashpartitioning(ss_sold_date_sk#3, 100), ENSURE_REQUIREMENTS, [plan_id=13] -(107) Sort +(108) Sort Input [4]: [ss_sold_date_sk#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6] Arguments: [ss_sold_date_sk#3 ASC NULLS FIRST], false, 0 -(108) Scan parquet +(109) Scan parquet Output [2]: [d_date_sk#7, d_month_seq#8] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [d_date_sk#7, d_month_seq#8] Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1176)) AND (d_month_seq#8 <= 1187)) AND isnotnull(d_date_sk#7)) -(110) Project +(111) Project Output [1]: [d_date_sk#7] Input [2]: [d_date_sk#7, d_month_seq#8] -(111) Exchange +(112) Exchange Input [1]: [d_date_sk#7] Arguments: hashpartitioning(d_date_sk#7, 100), ENSURE_REQUIREMENTS, [plan_id=14] -(112) Sort +(113) Sort Input [1]: [d_date_sk#7] Arguments: [d_date_sk#7 ASC NULLS FIRST], false, 0 -(113) SortMergeJoin +(114) SortMergeJoin Left keys [1]: [ss_sold_date_sk#3] Right keys [1]: [d_date_sk#7] Join type: Inner Join condition: None -(114) Project +(115) Project Output [3]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6] Input [5]: [ss_sold_date_sk#3, ss_item_sk#4, ss_store_sk#5, ss_sales_price#6, d_date_sk#7] -(115) HashAggregate +(116) HashAggregate Input [3]: [ss_item_sk#4, ss_store_sk#5, ss_sales_price#6] Keys [2]: [ss_store_sk#5, ss_item_sk#4] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] Aggregate Attributes [1]: [sum#10] Results [3]: [ss_store_sk#5, ss_item_sk#4, sum#33] -(116) Exchange +(117) Exchange Input [3]: [ss_store_sk#5, ss_item_sk#4, sum#33] Arguments: hashpartitioning(ss_store_sk#5, ss_item_sk#4, 100), ENSURE_REQUIREMENTS, [plan_id=15] -(117) HashAggregate +(118) HashAggregate Input [3]: [ss_store_sk#5, ss_item_sk#4, sum#33] Keys [2]: [ss_store_sk#5, ss_item_sk#4] Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#12] Results [3]: [ss_store_sk#5, ss_item_sk#4, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#12,17,2) AS revenue#13] -(118) Filter +(119) Filter Input [3]: [ss_store_sk#5, ss_item_sk#4, revenue#13] Condition : isnotnull(revenue#13) -(119) Exchange +(120) Exchange Input [3]: [ss_store_sk#5, ss_item_sk#4, revenue#13] Arguments: hashpartitioning(ss_store_sk#5, 100), ENSURE_REQUIREMENTS, [plan_id=16] -(120) Sort +(121) Sort Input [3]: [ss_store_sk#5, ss_item_sk#4, revenue#13] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false, 0 -(121) SortMergeJoin +(122) SortMergeJoin Left keys [1]: [s_store_sk#1] Right keys [1]: [ss_store_sk#5] Join type: Inner Join condition: None -(122) Project +(123) Project Output [4]: [s_store_name#2, ss_store_sk#5, ss_item_sk#4, revenue#13] Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#5, ss_item_sk#4, revenue#13] -(123) Exchange +(124) Exchange Input [4]: [s_store_name#2, ss_store_sk#5, ss_item_sk#4, revenue#13] Arguments: hashpartitioning(ss_item_sk#4, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(124) Sort +(125) Sort Input [4]: [s_store_name#2, ss_store_sk#5, ss_item_sk#4, revenue#13] Arguments: [ss_item_sk#4 ASC NULLS FIRST], false, 0 -(125) Scan parquet +(126) Scan parquet Output [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(126) Filter +(127) Filter Input [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Condition : isnotnull(i_item_sk#14) -(127) Exchange +(128) Exchange Input [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: hashpartitioning(i_item_sk#14, 100), ENSURE_REQUIREMENTS, [plan_id=18] -(128) Sort +(129) Sort Input [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: [i_item_sk#14 ASC NULLS FIRST], false, 0 -(129) SortMergeJoin +(130) SortMergeJoin Left keys [1]: [ss_item_sk#4] Right keys [1]: [i_item_sk#14] Join type: Inner Join condition: None -(130) Project +(131) Project Output [7]: [s_store_name#2, ss_store_sk#5, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Input [9]: [s_store_name#2, ss_store_sk#5, ss_item_sk#4, revenue#13, i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] -(131) Exchange +(132) Exchange Input [7]: [s_store_name#2, ss_store_sk#5, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: hashpartitioning(ss_store_sk#5, 100), ENSURE_REQUIREMENTS, [plan_id=19] -(132) Sort +(133) Sort Input [7]: [s_store_name#2, ss_store_sk#5, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false, 0 -(133) Scan parquet +(134) Scan parquet Output [4]: [ss_sold_date_sk#19, ss_item_sk#20, ss_store_sk#21, ss_sales_price#22] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_store_sk)] ReadSchema: struct -(134) Filter +(135) Filter Input [4]: [ss_sold_date_sk#19, ss_item_sk#20, ss_store_sk#21, ss_sales_price#22] Condition : (isnotnull(ss_sold_date_sk#19) AND isnotnull(ss_store_sk#21)) -(135) Exchange +(136) Exchange Input [4]: [ss_sold_date_sk#19, ss_item_sk#20, ss_store_sk#21, ss_sales_price#22] Arguments: hashpartitioning(ss_sold_date_sk#19, 100), ENSURE_REQUIREMENTS, [plan_id=20] -(136) Sort +(137) Sort Input [4]: [ss_sold_date_sk#19, ss_item_sk#20, ss_store_sk#21, ss_sales_price#22] Arguments: [ss_sold_date_sk#19 ASC NULLS FIRST], false, 0 -(137) Scan parquet +(138) Scan parquet Output [2]: [d_date_sk#23, d_month_seq#34] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct -(138) Filter +(139) Filter Input [2]: [d_date_sk#23, d_month_seq#34] Condition : (((isnotnull(d_month_seq#34) AND (d_month_seq#34 >= 1176)) AND (d_month_seq#34 <= 1187)) AND isnotnull(d_date_sk#23)) -(139) Project +(140) Project Output [1]: [d_date_sk#23] Input [2]: [d_date_sk#23, d_month_seq#34] -(140) Exchange +(141) Exchange Input [1]: [d_date_sk#23] Arguments: hashpartitioning(d_date_sk#23, 100), ENSURE_REQUIREMENTS, [plan_id=21] -(141) Sort +(142) Sort Input [1]: [d_date_sk#23] Arguments: [d_date_sk#23 ASC NULLS FIRST], false, 0 -(142) SortMergeJoin +(143) SortMergeJoin Left keys [1]: [ss_sold_date_sk#19] Right keys [1]: [d_date_sk#23] Join type: Inner Join condition: None -(143) Project +(144) Project Output [3]: [ss_item_sk#20, ss_store_sk#21, ss_sales_price#22] Input [5]: [ss_sold_date_sk#19, ss_item_sk#20, ss_store_sk#21, ss_sales_price#22, d_date_sk#23] -(144) HashAggregate +(145) HashAggregate Input [3]: [ss_item_sk#20, ss_store_sk#21, ss_sales_price#22] Keys [2]: [ss_store_sk#21, ss_item_sk#20] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#22))] Aggregate Attributes [1]: [sum#25] Results [3]: [ss_store_sk#21, ss_item_sk#20, sum#35] -(145) Exchange +(146) Exchange Input [3]: [ss_store_sk#21, ss_item_sk#20, sum#35] Arguments: hashpartitioning(ss_store_sk#21, ss_item_sk#20, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(146) HashAggregate +(147) HashAggregate Input [3]: [ss_store_sk#21, ss_item_sk#20, sum#35] Keys [2]: [ss_store_sk#21, ss_item_sk#20] Functions [1]: [sum(UnscaledValue(ss_sales_price#22))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#22))#26] Results [2]: [ss_store_sk#21, MakeDecimal(sum(UnscaledValue(ss_sales_price#22))#26,17,2) AS revenue#27] -(147) HashAggregate +(148) HashAggregate Input [2]: [ss_store_sk#21, revenue#27] Keys [1]: [ss_store_sk#21] Functions [1]: [partial_avg(revenue#27)] Aggregate Attributes [2]: [sum#28, count#29] Results [3]: [ss_store_sk#21, sum#36, count#37] -(148) Exchange +(149) Exchange Input [3]: [ss_store_sk#21, sum#36, count#37] Arguments: hashpartitioning(ss_store_sk#21, 100), ENSURE_REQUIREMENTS, [plan_id=23] -(149) HashAggregate +(150) HashAggregate Input [3]: [ss_store_sk#21, sum#36, count#37] Keys [1]: [ss_store_sk#21] Functions [1]: [avg(revenue#27)] Aggregate Attributes [1]: [avg(revenue#27)#31] Results [2]: [ss_store_sk#21, avg(revenue#27)#31 AS ave#32] -(150) Filter +(151) Filter Input [2]: [ss_store_sk#21, ave#32] Condition : isnotnull(ave#32) -(151) Sort +(152) Sort Input [2]: [ss_store_sk#21, ave#32] Arguments: [ss_store_sk#21 ASC NULLS FIRST], false, 0 -(152) SortMergeJoin +(153) SortMergeJoin Left keys [1]: [ss_store_sk#5] Right keys [1]: [ss_store_sk#21] Join type: Inner Join condition: (cast(revenue#13 as decimal(23,7)) <= (0.1 * ave#32)) -(153) Project +(154) Project Output [6]: [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] Input [9]: [s_store_name#2, ss_store_sk#5, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18, ss_store_sk#21, ave#32] -(154) TakeOrderedAndProject +(155) TakeOrderedAndProject Input [6]: [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: X, [s_store_name#2 ASC NULLS FIRST, i_item_desc#15 ASC NULLS FIRST], [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] -(155) AdaptiveSparkPlan +(156) AdaptiveSparkPlan Output [6]: [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q68.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q68.txt index b5f7ab751..39af5de52 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q68.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q68.txt @@ -1,165 +1,166 @@ == Physical Plan == -AdaptiveSparkPlan (156) +AdaptiveSparkPlan (157) +- == Final Plan == - TakeOrderedAndProject (99) - +- * Project (98) - +- * SortMergeJoin Inner (97) - :- NativeSort (90) - : +- InputAdapter (89) - : +- AQEShuffleRead (88) - : +- ShuffleQueryStage (87), Statistics(X) - : +- NativeShuffleExchange (86) - : +- NativeProject (85) - : +- NativeSortMergeJoin Inner (84) - : :- NativeSort (75) - : : +- InputAdapter (74) - : : +- AQEShuffleRead (73) - : : +- ShuffleQueryStage (72), Statistics(X) - : : +- NativeShuffleExchange (71) - : : +- NativeProject (70) - : : +- NativeHashAggregate (69) - : : +- NativeHashAggregate (68) - : : +- NativeProject (67) - : : +- NativeProject (66) - : : +- NativeSortMergeJoin Inner (65) - : : :- NativeSort (56) - : : : +- InputAdapter (55) - : : : +- AQEShuffleRead (54) - : : : +- ShuffleQueryStage (53), Statistics(X) - : : : +- NativeShuffleExchange (52) - : : : +- NativeProject (51) - : : : +- NativeSortMergeJoin Inner (50) - : : : :- NativeSort (40) - : : : : +- InputAdapter (39) - : : : : +- AQEShuffleRead (38) - : : : : +- ShuffleQueryStage (37), Statistics(X) - : : : : +- NativeShuffleExchange (36) - : : : : +- NativeProject (35) - : : : : +- NativeSortMergeJoin Inner (34) - : : : : :- NativeSort (24) - : : : : : +- InputAdapter (23) - : : : : : +- AQEShuffleRead (22) - : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : +- NativeShuffleExchange (20) - : : : : : +- NativeProject (19) - : : : : : +- NativeSortMergeJoin Inner (18) - : : : : : :- NativeSort (8) - : : : : : : +- InputAdapter (7) - : : : : : : +- AQEShuffleRead (6) - : : : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : : : +- NativeShuffleExchange (4) - : : : : : : +- NativeFilter (3) - : : : : : : +- InputAdapter (2) - : : : : : : +- NativeParquetScan (1) - : : : : : +- NativeSort (17) - : : : : : +- InputAdapter (16) - : : : : : +- AQEShuffleRead (15) - : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : +- NativeShuffleExchange (13) - : : : : : +- NativeProject (12) - : : : : : +- NativeFilter (11) - : : : : : +- InputAdapter (10) - : : : : : +- NativeParquetScan (9) - : : : : +- NativeSort (33) - : : : : +- InputAdapter (32) - : : : : +- AQEShuffleRead (31) - : : : : +- ShuffleQueryStage (30), Statistics(X) - : : : : +- NativeShuffleExchange (29) - : : : : +- NativeProject (28) - : : : : +- NativeFilter (27) - : : : : +- InputAdapter (26) - : : : : +- NativeParquetScan (25) - : : : +- NativeSort (49) - : : : +- InputAdapter (48) - : : : +- AQEShuffleRead (47) - : : : +- ShuffleQueryStage (46), Statistics(X) - : : : +- NativeShuffleExchange (45) - : : : +- NativeProject (44) - : : : +- NativeFilter (43) - : : : +- InputAdapter (42) - : : : +- NativeParquetScan (41) - : : +- NativeSort (64) - : : +- InputAdapter (63) - : : +- AQEShuffleRead (62) - : : +- ShuffleQueryStage (61), Statistics(X) - : : +- NativeShuffleExchange (60) - : : +- NativeFilter (59) - : : +- InputAdapter (58) - : : +- NativeParquetScan (57) - : +- NativeSort (83) - : +- InputAdapter (82) - : +- AQEShuffleRead (81) - : +- ShuffleQueryStage (80), Statistics(X) - : +- NativeShuffleExchange (79) - : +- NativeFilter (78) - : +- InputAdapter (77) - : +- NativeParquetScan (76) - +- NativeSort (96) - +- InputAdapter (95) - +- InputAdapter (94) - +- AQEShuffleRead (93) - +- ShuffleQueryStage (92), Statistics(X) - +- ReusedExchange (91) + NativeTakeOrdered (100) + +- NativeProject (99) + +- NativeFilter (98) + +- NativeSortMergeJoin Inner (97) + :- NativeSort (90) + : +- InputAdapter (89) + : +- AQEShuffleRead (88) + : +- ShuffleQueryStage (87), Statistics(X) + : +- NativeShuffleExchange (86) + : +- NativeProject (85) + : +- NativeSortMergeJoin Inner (84) + : :- NativeSort (75) + : : +- InputAdapter (74) + : : +- AQEShuffleRead (73) + : : +- ShuffleQueryStage (72), Statistics(X) + : : +- NativeShuffleExchange (71) + : : +- NativeProject (70) + : : +- NativeHashAggregate (69) + : : +- NativeHashAggregate (68) + : : +- NativeProject (67) + : : +- NativeProject (66) + : : +- NativeSortMergeJoin Inner (65) + : : :- NativeSort (56) + : : : +- InputAdapter (55) + : : : +- AQEShuffleRead (54) + : : : +- ShuffleQueryStage (53), Statistics(X) + : : : +- NativeShuffleExchange (52) + : : : +- NativeProject (51) + : : : +- NativeSortMergeJoin Inner (50) + : : : :- NativeSort (40) + : : : : +- InputAdapter (39) + : : : : +- AQEShuffleRead (38) + : : : : +- ShuffleQueryStage (37), Statistics(X) + : : : : +- NativeShuffleExchange (36) + : : : : +- NativeProject (35) + : : : : +- NativeSortMergeJoin Inner (34) + : : : : :- NativeSort (24) + : : : : : +- InputAdapter (23) + : : : : : +- AQEShuffleRead (22) + : : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : : +- NativeShuffleExchange (20) + : : : : : +- NativeProject (19) + : : : : : +- NativeSortMergeJoin Inner (18) + : : : : : :- NativeSort (8) + : : : : : : +- InputAdapter (7) + : : : : : : +- AQEShuffleRead (6) + : : : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : : : +- NativeShuffleExchange (4) + : : : : : : +- NativeFilter (3) + : : : : : : +- InputAdapter (2) + : : : : : : +- NativeParquetScan (1) + : : : : : +- NativeSort (17) + : : : : : +- InputAdapter (16) + : : : : : +- AQEShuffleRead (15) + : : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : : +- NativeShuffleExchange (13) + : : : : : +- NativeProject (12) + : : : : : +- NativeFilter (11) + : : : : : +- InputAdapter (10) + : : : : : +- NativeParquetScan (9) + : : : : +- NativeSort (33) + : : : : +- InputAdapter (32) + : : : : +- AQEShuffleRead (31) + : : : : +- ShuffleQueryStage (30), Statistics(X) + : : : : +- NativeShuffleExchange (29) + : : : : +- NativeProject (28) + : : : : +- NativeFilter (27) + : : : : +- InputAdapter (26) + : : : : +- NativeParquetScan (25) + : : : +- NativeSort (49) + : : : +- InputAdapter (48) + : : : +- AQEShuffleRead (47) + : : : +- ShuffleQueryStage (46), Statistics(X) + : : : +- NativeShuffleExchange (45) + : : : +- NativeProject (44) + : : : +- NativeFilter (43) + : : : +- InputAdapter (42) + : : : +- NativeParquetScan (41) + : : +- NativeSort (64) + : : +- InputAdapter (63) + : : +- AQEShuffleRead (62) + : : +- ShuffleQueryStage (61), Statistics(X) + : : +- NativeShuffleExchange (60) + : : +- NativeFilter (59) + : : +- InputAdapter (58) + : : +- NativeParquetScan (57) + : +- NativeSort (83) + : +- InputAdapter (82) + : +- AQEShuffleRead (81) + : +- ShuffleQueryStage (80), Statistics(X) + : +- NativeShuffleExchange (79) + : +- NativeFilter (78) + : +- InputAdapter (77) + : +- NativeParquetScan (76) + +- NativeSort (96) + +- InputAdapter (95) + +- InputAdapter (94) + +- AQEShuffleRead (93) + +- ShuffleQueryStage (92), Statistics(X) + +- ReusedExchange (91) +- == Initial Plan == - TakeOrderedAndProject (155) - +- Project (154) - +- SortMergeJoin Inner (153) - :- Sort (148) - : +- Exchange (147) - : +- Project (146) - : +- SortMergeJoin Inner (145) - : :- Sort (140) - : : +- Exchange (139) - : : +- HashAggregate (138) - : : +- HashAggregate (137) - : : +- Project (136) - : : +- SortMergeJoin Inner (135) - : : :- Sort (130) - : : : +- Exchange (129) - : : : +- Project (128) - : : : +- SortMergeJoin Inner (127) - : : : :- Sort (121) - : : : : +- Exchange (120) - : : : : +- Project (119) - : : : : +- SortMergeJoin Inner (118) - : : : : :- Sort (112) - : : : : : +- Exchange (111) - : : : : : +- Project (110) - : : : : : +- SortMergeJoin Inner (109) - : : : : : :- Sort (103) - : : : : : : +- Exchange (102) - : : : : : : +- Filter (101) - : : : : : : +- Scan parquet (100) - : : : : : +- Sort (108) - : : : : : +- Exchange (107) - : : : : : +- Project (106) - : : : : : +- Filter (105) - : : : : : +- Scan parquet (104) - : : : : +- Sort (117) - : : : : +- Exchange (116) - : : : : +- Project (115) - : : : : +- Filter (114) - : : : : +- Scan parquet (113) - : : : +- Sort (126) - : : : +- Exchange (125) - : : : +- Project (124) - : : : +- Filter (123) - : : : +- Scan parquet (122) - : : +- Sort (134) - : : +- Exchange (133) - : : +- Filter (132) - : : +- Scan parquet (131) - : +- Sort (144) - : +- Exchange (143) - : +- Filter (142) - : +- Scan parquet (141) - +- Sort (152) - +- Exchange (151) - +- Filter (150) - +- Scan parquet (149) - - -(100) Scan parquet + TakeOrderedAndProject (156) + +- Project (155) + +- SortMergeJoin Inner (154) + :- Sort (149) + : +- Exchange (148) + : +- Project (147) + : +- SortMergeJoin Inner (146) + : :- Sort (141) + : : +- Exchange (140) + : : +- HashAggregate (139) + : : +- HashAggregate (138) + : : +- Project (137) + : : +- SortMergeJoin Inner (136) + : : :- Sort (131) + : : : +- Exchange (130) + : : : +- Project (129) + : : : +- SortMergeJoin Inner (128) + : : : :- Sort (122) + : : : : +- Exchange (121) + : : : : +- Project (120) + : : : : +- SortMergeJoin Inner (119) + : : : : :- Sort (113) + : : : : : +- Exchange (112) + : : : : : +- Project (111) + : : : : : +- SortMergeJoin Inner (110) + : : : : : :- Sort (104) + : : : : : : +- Exchange (103) + : : : : : : +- Filter (102) + : : : : : : +- Scan parquet (101) + : : : : : +- Sort (109) + : : : : : +- Exchange (108) + : : : : : +- Project (107) + : : : : : +- Filter (106) + : : : : : +- Scan parquet (105) + : : : : +- Sort (118) + : : : : +- Exchange (117) + : : : : +- Project (116) + : : : : +- Filter (115) + : : : : +- Scan parquet (114) + : : : +- Sort (127) + : : : +- Exchange (126) + : : : +- Project (125) + : : : +- Filter (124) + : : : +- Scan parquet (123) + : : +- Sort (135) + : : +- Exchange (134) + : : +- Filter (133) + : : +- Scan parquet (132) + : +- Sort (145) + : +- Exchange (144) + : +- Filter (143) + : +- Scan parquet (142) + +- Sort (153) + +- Exchange (152) + +- Filter (151) + +- Scan parquet (150) + + +(101) Scan parquet Output [9]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Batched: true Location: InMemoryFileIndex [file:/] @@ -193,7 +194,7 @@ Input [9]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8, #9#9] Input [9]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8, #9#9] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false -(104) Scan parquet +(105) Scan parquet Output [3]: [d_date_sk#10, d_year#11, d_dom#12] Batched: true Location: InMemoryFileIndex [file:/] @@ -260,7 +261,7 @@ Input [8]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_tic Input [8]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false -(113) Scan parquet +(114) Scan parquet Output [2]: [s_store_sk#13, s_city#14] Batched: true Location: InMemoryFileIndex [file:/] @@ -327,7 +328,7 @@ Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, s Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: [ss_hdemo_sk#3 ASC NULLS FIRST], false -(122) Scan parquet +(123) Scan parquet Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location: InMemoryFileIndex [file:/] @@ -394,7 +395,7 @@ Input [6]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_pri Input [6]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: [ss_addr_sk#4 ASC NULLS FIRST], false -(131) Scan parquet +(132) Scan parquet Output [2]: [ca_address_sk#18, ca_city#19] Batched: true Location: InMemoryFileIndex [file:/] @@ -479,7 +480,7 @@ Input [6]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#30, extended_price Input [6]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#30, extended_price#31, list_price#32, extended_tax#33] Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false -(141) Scan parquet +(142) Scan parquet Output [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Batched: true Location: InMemoryFileIndex [file:/] @@ -564,284 +565,288 @@ Input [2]: [#38#38, #39#39] Input [2]: [#38#38, #39#39] Arguments: [ca_address_sk#38 ASC NULLS FIRST], false -(97) SortMergeJoin [codegen id : X] +(97) NativeSortMergeJoin Left keys [1]: [c_current_addr_sk#35] Right keys [1]: [ca_address_sk#38] Join type: Inner -Join condition: NOT (ca_city#39 = bought_city#30) +Join condition: None + +(98) NativeFilter +Input [10]: [ss_ticket_number#6, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37, #38#38, #39#39] +Condition : NOT (ca_city#39 = bought_city#30) -(98) Project [codegen id : X] +(99) NativeProject Output [8]: [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#6, extended_price#31, extended_tax#33, list_price#32] Input [10]: [ss_ticket_number#6, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37, #38#38, #39#39] -(99) TakeOrderedAndProject +(100) NativeTakeOrdered Input [8]: [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#6, extended_price#31, extended_tax#33, list_price#32] -Arguments: X, [c_last_name#37 ASC NULLS FIRST, ss_ticket_number#6 ASC NULLS FIRST], [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#6, extended_price#31, extended_tax#33, list_price#32] +Arguments: X, X, [c_last_name#37 ASC NULLS FIRST, ss_ticket_number#6 ASC NULLS FIRST] -(100) Scan parquet +(101) Scan parquet Output [9]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_sold_date_sk), IsNotNull(ss_store_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk), IsNotNull(ss_customer_sk)] ReadSchema: struct -(101) Filter +(102) Filter Input [9]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Condition : ((((isnotnull(ss_sold_date_sk#1) AND isnotnull(ss_store_sk#5)) AND isnotnull(ss_hdemo_sk#3)) AND isnotnull(ss_addr_sk#4)) AND isnotnull(ss_customer_sk#2)) -(102) Exchange +(103) Exchange Input [9]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: hashpartitioning(ss_sold_date_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=12] -(103) Sort +(104) Sort Input [9]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: [ss_sold_date_sk#1 ASC NULLS FIRST], false, 0 -(104) Scan parquet +(105) Scan parquet Output [3]: [d_date_sk#10, d_year#11, d_dom#12] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(105) Filter +(106) Filter Input [3]: [d_date_sk#10, d_year#11, d_dom#12] Condition : ((((isnotnull(d_dom#12) AND (d_dom#12 >= 1)) AND (d_dom#12 <= 2)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) -(106) Project +(107) Project Output [1]: [d_date_sk#10] Input [3]: [d_date_sk#10, d_year#11, d_dom#12] -(107) Exchange +(108) Exchange Input [1]: [d_date_sk#10] Arguments: hashpartitioning(d_date_sk#10, 100), ENSURE_REQUIREMENTS, [plan_id=13] -(108) Sort +(109) Sort Input [1]: [d_date_sk#10] Arguments: [d_date_sk#10 ASC NULLS FIRST], false, 0 -(109) SortMergeJoin +(110) SortMergeJoin Left keys [1]: [ss_sold_date_sk#1] Right keys [1]: [d_date_sk#10] Join type: Inner Join condition: None -(110) Project +(111) Project Output [8]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Input [10]: [ss_sold_date_sk#1, ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9, d_date_sk#10] -(111) Exchange +(112) Exchange Input [8]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: hashpartitioning(ss_store_sk#5, 100), ENSURE_REQUIREMENTS, [plan_id=14] -(112) Sort +(113) Sort Input [8]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: [ss_store_sk#5 ASC NULLS FIRST], false, 0 -(113) Scan parquet +(114) Scan parquet Output [2]: [s_store_sk#13, s_city#14] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [In(s_city, [Fairview,Midway]), IsNotNull(s_store_sk)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [s_store_sk#13, s_city#14] Condition : (s_city#14 IN (Midway,Fairview) AND isnotnull(s_store_sk#13)) -(115) Project +(116) Project Output [1]: [s_store_sk#13] Input [2]: [s_store_sk#13, s_city#14] -(116) Exchange +(117) Exchange Input [1]: [s_store_sk#13] Arguments: hashpartitioning(s_store_sk#13, 100), ENSURE_REQUIREMENTS, [plan_id=15] -(117) Sort +(118) Sort Input [1]: [s_store_sk#13] Arguments: [s_store_sk#13 ASC NULLS FIRST], false, 0 -(118) SortMergeJoin +(119) SortMergeJoin Left keys [1]: [ss_store_sk#5] Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None -(119) Project +(120) Project Output [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Input [9]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_store_sk#5, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9, s_store_sk#13] -(120) Exchange +(121) Exchange Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: hashpartitioning(ss_hdemo_sk#3, 100), ENSURE_REQUIREMENTS, [plan_id=16] -(121) Sort +(122) Sort Input [7]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: [ss_hdemo_sk#3 ASC NULLS FIRST], false, 0 -(122) Scan parquet +(123) Scan parquet Output [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [Or(EqualTo(hd_dep_count,4),EqualTo(hd_vehicle_count,3)), IsNotNull(hd_demo_sk)] ReadSchema: struct -(123) Filter +(124) Filter Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] Condition : (((hd_dep_count#16 = 4) OR (hd_vehicle_count#17 = 3)) AND isnotnull(hd_demo_sk#15)) -(124) Project +(125) Project Output [1]: [hd_demo_sk#15] Input [3]: [hd_demo_sk#15, hd_dep_count#16, hd_vehicle_count#17] -(125) Exchange +(126) Exchange Input [1]: [hd_demo_sk#15] Arguments: hashpartitioning(hd_demo_sk#15, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(126) Sort +(127) Sort Input [1]: [hd_demo_sk#15] Arguments: [hd_demo_sk#15 ASC NULLS FIRST], false, 0 -(127) SortMergeJoin +(128) SortMergeJoin Left keys [1]: [ss_hdemo_sk#3] Right keys [1]: [hd_demo_sk#15] Join type: Inner Join condition: None -(128) Project +(129) Project Output [6]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Input [8]: [ss_customer_sk#2, ss_hdemo_sk#3, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9, hd_demo_sk#15] -(129) Exchange +(130) Exchange Input [6]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: hashpartitioning(ss_addr_sk#4, 100), ENSURE_REQUIREMENTS, [plan_id=18] -(130) Sort +(131) Sort Input [6]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9] Arguments: [ss_addr_sk#4 ASC NULLS FIRST], false, 0 -(131) Scan parquet +(132) Scan parquet Output [2]: [ca_address_sk#18, ca_city#19] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(132) Filter +(133) Filter Input [2]: [ca_address_sk#18, ca_city#19] Condition : (isnotnull(ca_address_sk#18) AND isnotnull(ca_city#19)) -(133) Exchange +(134) Exchange Input [2]: [ca_address_sk#18, ca_city#19] Arguments: hashpartitioning(ca_address_sk#18, 100), ENSURE_REQUIREMENTS, [plan_id=19] -(134) Sort +(135) Sort Input [2]: [ca_address_sk#18, ca_city#19] Arguments: [ca_address_sk#18 ASC NULLS FIRST], false, 0 -(135) SortMergeJoin +(136) SortMergeJoin Left keys [1]: [ss_addr_sk#4] Right keys [1]: [ca_address_sk#18] Join type: Inner Join condition: None -(136) Project +(137) Project Output [7]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9, ca_city#19] Input [8]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9, ca_address_sk#18, ca_city#19] -(137) HashAggregate +(138) HashAggregate Input [7]: [ss_customer_sk#2, ss_addr_sk#4, ss_ticket_number#6, ss_ext_sales_price#7, ss_ext_list_price#8, ss_ext_tax#9, ca_city#19] Keys [4]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#19] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#7)), partial_sum(UnscaledValue(ss_ext_list_price#8)), partial_sum(UnscaledValue(ss_ext_tax#9))] Aggregate Attributes [3]: [sum#23, sum#24, sum#25] Results [7]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#19, sum#40, sum#41, sum#42] -(138) HashAggregate +(139) HashAggregate Input [7]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#19, sum#40, sum#41, sum#42] Keys [4]: [ss_ticket_number#6, ss_customer_sk#2, ss_addr_sk#4, ca_city#19] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#7)), sum(UnscaledValue(ss_ext_list_price#8)), sum(UnscaledValue(ss_ext_tax#9))] Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#7))#27, sum(UnscaledValue(ss_ext_list_price#8))#28, sum(UnscaledValue(ss_ext_tax#9))#29] Results [6]: [ss_ticket_number#6, ss_customer_sk#2, ca_city#19 AS bought_city#30, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#27,17,2) AS extended_price#31, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#8))#28,17,2) AS list_price#32, MakeDecimal(sum(UnscaledValue(ss_ext_tax#9))#29,17,2) AS extended_tax#33] -(139) Exchange +(140) Exchange Input [6]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#30, extended_price#31, list_price#32, extended_tax#33] Arguments: hashpartitioning(ss_customer_sk#2, 100), ENSURE_REQUIREMENTS, [plan_id=20] -(140) Sort +(141) Sort Input [6]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#30, extended_price#31, list_price#32, extended_tax#33] Arguments: [ss_customer_sk#2 ASC NULLS FIRST], false, 0 -(141) Scan parquet +(142) Scan parquet Output [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(142) Filter +(143) Filter Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Condition : (isnotnull(c_customer_sk#34) AND isnotnull(c_current_addr_sk#35)) -(143) Exchange +(144) Exchange Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Arguments: hashpartitioning(c_customer_sk#34, 100), ENSURE_REQUIREMENTS, [plan_id=21] -(144) Sort +(145) Sort Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Arguments: [c_customer_sk#34 ASC NULLS FIRST], false, 0 -(145) SortMergeJoin +(146) SortMergeJoin Left keys [1]: [ss_customer_sk#2] Right keys [1]: [c_customer_sk#34] Join type: Inner Join condition: None -(146) Project +(147) Project Output [8]: [ss_ticket_number#6, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Input [10]: [ss_ticket_number#6, ss_customer_sk#2, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] -(147) Exchange +(148) Exchange Input [8]: [ss_ticket_number#6, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Arguments: hashpartitioning(c_current_addr_sk#35, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(148) Sort +(149) Sort Input [8]: [ss_ticket_number#6, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Arguments: [c_current_addr_sk#35 ASC NULLS FIRST], false, 0 -(149) Scan parquet +(150) Scan parquet Output [2]: [ca_address_sk#38, ca_city#39] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(150) Filter +(151) Filter Input [2]: [ca_address_sk#38, ca_city#39] Condition : (isnotnull(ca_address_sk#38) AND isnotnull(ca_city#39)) -(151) Exchange +(152) Exchange Input [2]: [ca_address_sk#38, ca_city#39] Arguments: hashpartitioning(ca_address_sk#38, 100), ENSURE_REQUIREMENTS, [plan_id=23] -(152) Sort +(153) Sort Input [2]: [ca_address_sk#38, ca_city#39] Arguments: [ca_address_sk#38 ASC NULLS FIRST], false, 0 -(153) SortMergeJoin +(154) SortMergeJoin Left keys [1]: [c_current_addr_sk#35] Right keys [1]: [ca_address_sk#38] Join type: Inner Join condition: NOT (ca_city#39 = bought_city#30) -(154) Project +(155) Project Output [8]: [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#6, extended_price#31, extended_tax#33, list_price#32] Input [10]: [ss_ticket_number#6, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37, ca_address_sk#38, ca_city#39] -(155) TakeOrderedAndProject +(156) TakeOrderedAndProject Input [8]: [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#6, extended_price#31, extended_tax#33, list_price#32] Arguments: X, [c_last_name#37 ASC NULLS FIRST, ss_ticket_number#6 ASC NULLS FIRST], [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#6, extended_price#31, extended_tax#33, list_price#32] -(156) AdaptiveSparkPlan +(157) AdaptiveSparkPlan Output [8]: [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#6, extended_price#31, extended_tax#33, list_price#32] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q72.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q72.txt index 746601c52..7774e0c81 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q72.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q72.txt @@ -23,9 +23,9 @@ AdaptiveSparkPlan (256) : : +- AQEShuffleRead (131) : : +- ShuffleQueryStage (130), Statistics(X) : : +- NativeShuffleExchange (129) - : : +- ConvertToNative (128) - : : +- * Project (127) - : : +- * SortMergeJoin Inner (126) + : : +- NativeProject (128) + : : +- NativeFilter (127) + : : +- NativeSortMergeJoin Inner (126) : : :- NativeSort (117) : : : +- InputAdapter (116) : : : +- AQEShuffleRead (115) @@ -73,9 +73,9 @@ AdaptiveSparkPlan (256) : : : : : : : : : +- AQEShuffleRead (22) : : : : : : : : : +- ShuffleQueryStage (21), Statistics(X) : : : : : : : : : +- NativeShuffleExchange (20) - : : : : : : : : : +- ConvertToNative (19) - : : : : : : : : : +- * Project (18) - : : : : : : : : : +- * SortMergeJoin Inner (17) + : : : : : : : : : +- NativeProject (19) + : : : : : : : : : +- NativeFilter (18) + : : : : : : : : : +- NativeSortMergeJoin Inner (17) : : : : : : : : : :- NativeSort (8) : : : : : : : : : : +- InputAdapter (7) : : : : : : : : : : +- AQEShuffleRead (6) @@ -327,18 +327,19 @@ Input [4]: [#9#9, #10#10, #11#11, #12#12] Input [4]: [#9#9, #10#10, #11#11, #12#12] Arguments: [inv_item_sk#10 ASC NULLS FIRST], false -(17) SortMergeJoin [codegen id : X] +(17) NativeSortMergeJoin Left keys [1]: [cs_item_sk#5] Right keys [1]: [inv_item_sk#10] Join type: Inner -Join condition: (inv_quantity_on_hand#12 < cs_quantity#8) +Join condition: None -(18) Project [codegen id : X] -Output [9]: [cs_sold_date_sk#1, cs_ship_date_sk#2, cs_bill_cdemo_sk#3, cs_bill_hdemo_sk#4, cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, inv_date_sk#9, inv_warehouse_sk#11] +(18) NativeFilter Input [12]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8, #9#9, #10#10, #11#11, #12#12] +Condition : (inv_quantity_on_hand#12 < cs_quantity#8) -(19) ConvertToNative -Input [9]: [cs_sold_date_sk#1, cs_ship_date_sk#2, cs_bill_cdemo_sk#3, cs_bill_hdemo_sk#4, cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, inv_date_sk#9, inv_warehouse_sk#11] +(19) NativeProject +Output [9]: [cs_sold_date_sk#1, cs_ship_date_sk#2, cs_bill_cdemo_sk#3, cs_bill_hdemo_sk#4, cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, inv_date_sk#9, inv_warehouse_sk#11] +Input [12]: [#1#1, #2#2, #3#3, #4#4, #5#5, #6#6, #7#7, #8#8, #9#9, #10#10, #11#11, #12#12] (20) NativeShuffleExchange Input [9]: [cs_sold_date_sk#1, cs_ship_date_sk#2, cs_bill_cdemo_sk#3, cs_bill_hdemo_sk#4, cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, inv_date_sk#9, inv_warehouse_sk#11] @@ -783,18 +784,19 @@ Input [2]: [#27#27, #28#28] Input [2]: [#27#27, #28#28] Arguments: [d_date_sk#27 ASC NULLS FIRST], false -(126) SortMergeJoin [codegen id : X] +(126) NativeSortMergeJoin Left keys [1]: [cs_ship_date_sk#2] Right keys [1]: [d_date_sk#27] Join type: Inner -Join condition: (d_date#28 > date_add(d_date#22, 5)) +Join condition: None -(127) Project [codegen id : X] -Output [6]: [cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(127) NativeFilter Input [10]: [cs_ship_date_sk#2, cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, #27#27, #28#28] +Condition : (d_date#28 > date_add(d_date#22, 5)) -(128) ConvertToNative -Input [6]: [cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +(128) NativeProject +Output [6]: [cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] +Input [10]: [cs_ship_date_sk#2, cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, w_warehouse_name#14, i_item_desc#16, d_date#22, d_week_seq#23, #27#27, #28#28] (129) NativeShuffleExchange Input [6]: [cs_item_sk#5, cs_promo_sk#6, cs_order_number#7, w_warehouse_name#14, i_item_desc#16, d_week_seq#23] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q74.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q74.txt index 29e2f0dea..c01214844 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q74.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q74.txt @@ -1,262 +1,263 @@ == Physical Plan == -AdaptiveSparkPlan (253) +AdaptiveSparkPlan (254) +- == Final Plan == - TakeOrderedAndProject (152) - +- * Project (151) - +- * SortMergeJoin Inner (150) - :- NativeProject (122) - : +- NativeSortMergeJoin Inner (121) - : :- NativeSortMergeJoin Inner (77) - : : :- NativeSort (47) - : : : +- InputAdapter (46) - : : : +- AQEShuffleRead (45) - : : : +- ShuffleQueryStage (44), Statistics(X) - : : : +- NativeShuffleExchange (43) - : : : +- NativeFilter (42) - : : : +- NativeProject (41) - : : : +- NativeHashAggregate (40) - : : : +- InputAdapter (39) - : : : +- AQEShuffleRead (38) - : : : +- ShuffleQueryStage (37), Statistics(X) - : : : +- NativeShuffleExchange (36) - : : : +- NativeHashAggregate (35) - : : : +- NativeProject (34) - : : : +- NativeProject (33) - : : : +- NativeSortMergeJoin Inner (32) - : : : :- NativeSort (23) - : : : : +- InputAdapter (22) - : : : : +- AQEShuffleRead (21) - : : : : +- ShuffleQueryStage (20), Statistics(X) - : : : : +- NativeShuffleExchange (19) - : : : : +- NativeProject (18) - : : : : +- NativeSortMergeJoin Inner (17) - : : : : :- NativeSort (8) - : : : : : +- InputAdapter (7) - : : : : : +- AQEShuffleRead (6) - : : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : : +- NativeShuffleExchange (4) - : : : : : +- NativeFilter (3) - : : : : : +- InputAdapter (2) - : : : : : +- NativeParquetScan (1) - : : : : +- NativeSort (16) - : : : : +- InputAdapter (15) - : : : : +- AQEShuffleRead (14) - : : : : +- ShuffleQueryStage (13), Statistics(X) - : : : : +- NativeShuffleExchange (12) - : : : : +- NativeFilter (11) - : : : : +- InputAdapter (10) - : : : : +- NativeParquetScan (9) - : : : +- NativeSort (31) - : : : +- InputAdapter (30) - : : : +- AQEShuffleRead (29) - : : : +- ShuffleQueryStage (28), Statistics(X) - : : : +- NativeShuffleExchange (27) - : : : +- NativeFilter (26) - : : : +- InputAdapter (25) - : : : +- NativeParquetScan (24) - : : +- NativeSort (76) - : : +- InputAdapter (75) - : : +- AQEShuffleRead (74) - : : +- ShuffleQueryStage (73), Statistics(X) - : : +- NativeShuffleExchange (72) - : : +- NativeProject (71) - : : +- NativeHashAggregate (70) - : : +- InputAdapter (69) - : : +- AQEShuffleRead (68) - : : +- ShuffleQueryStage (67), Statistics(X) - : : +- NativeShuffleExchange (66) - : : +- NativeHashAggregate (65) - : : +- NativeProject (64) - : : +- NativeProject (63) - : : +- NativeSortMergeJoin Inner (62) - : : :- NativeSort (53) - : : : +- InputAdapter (52) - : : : +- InputAdapter (51) - : : : +- AQEShuffleRead (50) - : : : +- ShuffleQueryStage (49), Statistics(X) - : : : +- ReusedExchange (48) - : : +- NativeSort (61) - : : +- InputAdapter (60) - : : +- AQEShuffleRead (59) - : : +- ShuffleQueryStage (58), Statistics(X) - : : +- NativeShuffleExchange (57) - : : +- NativeFilter (56) - : : +- InputAdapter (55) - : : +- NativeParquetScan (54) - : +- NativeSort (120) - : +- InputAdapter (119) - : +- AQEShuffleRead (118) - : +- ShuffleQueryStage (117), Statistics(X) - : +- NativeShuffleExchange (116) - : +- NativeFilter (115) - : +- NativeProject (114) - : +- NativeHashAggregate (113) - : +- InputAdapter (112) - : +- AQEShuffleRead (111) - : +- ShuffleQueryStage (110), Statistics(X) - : +- NativeShuffleExchange (109) - : +- NativeHashAggregate (108) - : +- NativeProject (107) - : +- NativeProject (106) - : +- NativeSortMergeJoin Inner (105) - : :- NativeSort (98) - : : +- InputAdapter (97) - : : +- AQEShuffleRead (96) - : : +- ShuffleQueryStage (95), Statistics(X) - : : +- NativeShuffleExchange (94) - : : +- NativeProject (93) - : : +- NativeSortMergeJoin Inner (92) - : : :- NativeSort (83) - : : : +- InputAdapter (82) - : : : +- InputAdapter (81) - : : : +- AQEShuffleRead (80) - : : : +- ShuffleQueryStage (79), Statistics(X) - : : : +- ReusedExchange (78) - : : +- NativeSort (91) - : : +- InputAdapter (90) - : : +- AQEShuffleRead (89) - : : +- ShuffleQueryStage (88), Statistics(X) - : : +- NativeShuffleExchange (87) - : : +- NativeFilter (86) - : : +- InputAdapter (85) - : : +- NativeParquetScan (84) - : +- NativeSort (104) - : +- InputAdapter (103) - : +- InputAdapter (102) - : +- AQEShuffleRead (101) - : +- ShuffleQueryStage (100), Statistics(X) - : +- ReusedExchange (99) - +- NativeSort (149) - +- InputAdapter (148) - +- AQEShuffleRead (147) - +- ShuffleQueryStage (146), Statistics(X) - +- NativeShuffleExchange (145) - +- NativeProject (144) - +- NativeHashAggregate (143) - +- InputAdapter (142) - +- AQEShuffleRead (141) - +- ShuffleQueryStage (140), Statistics(X) - +- NativeShuffleExchange (139) - +- NativeHashAggregate (138) - +- NativeProject (137) - +- NativeProject (136) - +- NativeSortMergeJoin Inner (135) - :- NativeSort (128) - : +- InputAdapter (127) - : +- InputAdapter (126) - : +- AQEShuffleRead (125) - : +- ShuffleQueryStage (124), Statistics(X) - : +- ReusedExchange (123) - +- NativeSort (134) - +- InputAdapter (133) - +- InputAdapter (132) - +- AQEShuffleRead (131) - +- ShuffleQueryStage (130), Statistics(X) - +- ReusedExchange (129) + NativeTakeOrdered (153) + +- NativeProject (152) + +- NativeFilter (151) + +- NativeSortMergeJoin Inner (150) + :- NativeProject (122) + : +- NativeSortMergeJoin Inner (121) + : :- NativeSortMergeJoin Inner (77) + : : :- NativeSort (47) + : : : +- InputAdapter (46) + : : : +- AQEShuffleRead (45) + : : : +- ShuffleQueryStage (44), Statistics(X) + : : : +- NativeShuffleExchange (43) + : : : +- NativeFilter (42) + : : : +- NativeProject (41) + : : : +- NativeHashAggregate (40) + : : : +- InputAdapter (39) + : : : +- AQEShuffleRead (38) + : : : +- ShuffleQueryStage (37), Statistics(X) + : : : +- NativeShuffleExchange (36) + : : : +- NativeHashAggregate (35) + : : : +- NativeProject (34) + : : : +- NativeProject (33) + : : : +- NativeSortMergeJoin Inner (32) + : : : :- NativeSort (23) + : : : : +- InputAdapter (22) + : : : : +- AQEShuffleRead (21) + : : : : +- ShuffleQueryStage (20), Statistics(X) + : : : : +- NativeShuffleExchange (19) + : : : : +- NativeProject (18) + : : : : +- NativeSortMergeJoin Inner (17) + : : : : :- NativeSort (8) + : : : : : +- InputAdapter (7) + : : : : : +- AQEShuffleRead (6) + : : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : : +- NativeShuffleExchange (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (16) + : : : : +- InputAdapter (15) + : : : : +- AQEShuffleRead (14) + : : : : +- ShuffleQueryStage (13), Statistics(X) + : : : : +- NativeShuffleExchange (12) + : : : : +- NativeFilter (11) + : : : : +- InputAdapter (10) + : : : : +- NativeParquetScan (9) + : : : +- NativeSort (31) + : : : +- InputAdapter (30) + : : : +- AQEShuffleRead (29) + : : : +- ShuffleQueryStage (28), Statistics(X) + : : : +- NativeShuffleExchange (27) + : : : +- NativeFilter (26) + : : : +- InputAdapter (25) + : : : +- NativeParquetScan (24) + : : +- NativeSort (76) + : : +- InputAdapter (75) + : : +- AQEShuffleRead (74) + : : +- ShuffleQueryStage (73), Statistics(X) + : : +- NativeShuffleExchange (72) + : : +- NativeProject (71) + : : +- NativeHashAggregate (70) + : : +- InputAdapter (69) + : : +- AQEShuffleRead (68) + : : +- ShuffleQueryStage (67), Statistics(X) + : : +- NativeShuffleExchange (66) + : : +- NativeHashAggregate (65) + : : +- NativeProject (64) + : : +- NativeProject (63) + : : +- NativeSortMergeJoin Inner (62) + : : :- NativeSort (53) + : : : +- InputAdapter (52) + : : : +- InputAdapter (51) + : : : +- AQEShuffleRead (50) + : : : +- ShuffleQueryStage (49), Statistics(X) + : : : +- ReusedExchange (48) + : : +- NativeSort (61) + : : +- InputAdapter (60) + : : +- AQEShuffleRead (59) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- NativeShuffleExchange (57) + : : +- NativeFilter (56) + : : +- InputAdapter (55) + : : +- NativeParquetScan (54) + : +- NativeSort (120) + : +- InputAdapter (119) + : +- AQEShuffleRead (118) + : +- ShuffleQueryStage (117), Statistics(X) + : +- NativeShuffleExchange (116) + : +- NativeFilter (115) + : +- NativeProject (114) + : +- NativeHashAggregate (113) + : +- InputAdapter (112) + : +- AQEShuffleRead (111) + : +- ShuffleQueryStage (110), Statistics(X) + : +- NativeShuffleExchange (109) + : +- NativeHashAggregate (108) + : +- NativeProject (107) + : +- NativeProject (106) + : +- NativeSortMergeJoin Inner (105) + : :- NativeSort (98) + : : +- InputAdapter (97) + : : +- AQEShuffleRead (96) + : : +- ShuffleQueryStage (95), Statistics(X) + : : +- NativeShuffleExchange (94) + : : +- NativeProject (93) + : : +- NativeSortMergeJoin Inner (92) + : : :- NativeSort (83) + : : : +- InputAdapter (82) + : : : +- InputAdapter (81) + : : : +- AQEShuffleRead (80) + : : : +- ShuffleQueryStage (79), Statistics(X) + : : : +- ReusedExchange (78) + : : +- NativeSort (91) + : : +- InputAdapter (90) + : : +- AQEShuffleRead (89) + : : +- ShuffleQueryStage (88), Statistics(X) + : : +- NativeShuffleExchange (87) + : : +- NativeFilter (86) + : : +- InputAdapter (85) + : : +- NativeParquetScan (84) + : +- NativeSort (104) + : +- InputAdapter (103) + : +- InputAdapter (102) + : +- AQEShuffleRead (101) + : +- ShuffleQueryStage (100), Statistics(X) + : +- ReusedExchange (99) + +- NativeSort (149) + +- InputAdapter (148) + +- AQEShuffleRead (147) + +- ShuffleQueryStage (146), Statistics(X) + +- NativeShuffleExchange (145) + +- NativeProject (144) + +- NativeHashAggregate (143) + +- InputAdapter (142) + +- AQEShuffleRead (141) + +- ShuffleQueryStage (140), Statistics(X) + +- NativeShuffleExchange (139) + +- NativeHashAggregate (138) + +- NativeProject (137) + +- NativeProject (136) + +- NativeSortMergeJoin Inner (135) + :- NativeSort (128) + : +- InputAdapter (127) + : +- InputAdapter (126) + : +- AQEShuffleRead (125) + : +- ShuffleQueryStage (124), Statistics(X) + : +- ReusedExchange (123) + +- NativeSort (134) + +- InputAdapter (133) + +- InputAdapter (132) + +- AQEShuffleRead (131) + +- ShuffleQueryStage (130), Statistics(X) + +- ReusedExchange (129) +- == Initial Plan == - TakeOrderedAndProject (252) - +- Project (251) - +- SortMergeJoin Inner (250) - :- Project (226) - : +- SortMergeJoin Inner (225) - : :- SortMergeJoin Inner (200) - : : :- Sort (176) - : : : +- Exchange (175) - : : : +- Filter (174) - : : : +- HashAggregate (173) - : : : +- Exchange (172) - : : : +- HashAggregate (171) - : : : +- Project (170) - : : : +- SortMergeJoin Inner (169) - : : : :- Sort (164) - : : : : +- Exchange (163) - : : : : +- Project (162) - : : : : +- SortMergeJoin Inner (161) - : : : : :- Sort (156) - : : : : : +- Exchange (155) - : : : : : +- Filter (154) - : : : : : +- Scan parquet (153) - : : : : +- Sort (160) - : : : : +- Exchange (159) - : : : : +- Filter (158) - : : : : +- Scan parquet (157) - : : : +- Sort (168) - : : : +- Exchange (167) - : : : +- Filter (166) - : : : +- Scan parquet (165) - : : +- Sort (199) - : : +- Exchange (198) - : : +- HashAggregate (197) - : : +- Exchange (196) - : : +- HashAggregate (195) - : : +- Project (194) - : : +- SortMergeJoin Inner (193) - : : :- Sort (188) - : : : +- Exchange (187) - : : : +- Project (186) - : : : +- SortMergeJoin Inner (185) - : : : :- Sort (180) - : : : : +- Exchange (179) - : : : : +- Filter (178) - : : : : +- Scan parquet (177) - : : : +- Sort (184) - : : : +- Exchange (183) - : : : +- Filter (182) - : : : +- Scan parquet (181) - : : +- Sort (192) - : : +- Exchange (191) - : : +- Filter (190) - : : +- Scan parquet (189) - : +- Sort (224) - : +- Exchange (223) - : +- Filter (222) - : +- HashAggregate (221) - : +- Exchange (220) - : +- HashAggregate (219) - : +- Project (218) - : +- SortMergeJoin Inner (217) - : :- Sort (212) - : : +- Exchange (211) - : : +- Project (210) - : : +- SortMergeJoin Inner (209) - : : :- Sort (204) - : : : +- Exchange (203) - : : : +- Filter (202) - : : : +- Scan parquet (201) - : : +- Sort (208) - : : +- Exchange (207) - : : +- Filter (206) - : : +- Scan parquet (205) - : +- Sort (216) - : +- Exchange (215) - : +- Filter (214) - : +- Scan parquet (213) - +- Sort (249) - +- Exchange (248) - +- HashAggregate (247) - +- Exchange (246) - +- HashAggregate (245) - +- Project (244) - +- SortMergeJoin Inner (243) - :- Sort (238) - : +- Exchange (237) - : +- Project (236) - : +- SortMergeJoin Inner (235) - : :- Sort (230) - : : +- Exchange (229) - : : +- Filter (228) - : : +- Scan parquet (227) - : +- Sort (234) - : +- Exchange (233) - : +- Filter (232) - : +- Scan parquet (231) - +- Sort (242) - +- Exchange (241) - +- Filter (240) - +- Scan parquet (239) - - -(153) Scan parquet + TakeOrderedAndProject (253) + +- Project (252) + +- SortMergeJoin Inner (251) + :- Project (227) + : +- SortMergeJoin Inner (226) + : :- SortMergeJoin Inner (201) + : : :- Sort (177) + : : : +- Exchange (176) + : : : +- Filter (175) + : : : +- HashAggregate (174) + : : : +- Exchange (173) + : : : +- HashAggregate (172) + : : : +- Project (171) + : : : +- SortMergeJoin Inner (170) + : : : :- Sort (165) + : : : : +- Exchange (164) + : : : : +- Project (163) + : : : : +- SortMergeJoin Inner (162) + : : : : :- Sort (157) + : : : : : +- Exchange (156) + : : : : : +- Filter (155) + : : : : : +- Scan parquet (154) + : : : : +- Sort (161) + : : : : +- Exchange (160) + : : : : +- Filter (159) + : : : : +- Scan parquet (158) + : : : +- Sort (169) + : : : +- Exchange (168) + : : : +- Filter (167) + : : : +- Scan parquet (166) + : : +- Sort (200) + : : +- Exchange (199) + : : +- HashAggregate (198) + : : +- Exchange (197) + : : +- HashAggregate (196) + : : +- Project (195) + : : +- SortMergeJoin Inner (194) + : : :- Sort (189) + : : : +- Exchange (188) + : : : +- Project (187) + : : : +- SortMergeJoin Inner (186) + : : : :- Sort (181) + : : : : +- Exchange (180) + : : : : +- Filter (179) + : : : : +- Scan parquet (178) + : : : +- Sort (185) + : : : +- Exchange (184) + : : : +- Filter (183) + : : : +- Scan parquet (182) + : : +- Sort (193) + : : +- Exchange (192) + : : +- Filter (191) + : : +- Scan parquet (190) + : +- Sort (225) + : +- Exchange (224) + : +- Filter (223) + : +- HashAggregate (222) + : +- Exchange (221) + : +- HashAggregate (220) + : +- Project (219) + : +- SortMergeJoin Inner (218) + : :- Sort (213) + : : +- Exchange (212) + : : +- Project (211) + : : +- SortMergeJoin Inner (210) + : : :- Sort (205) + : : : +- Exchange (204) + : : : +- Filter (203) + : : : +- Scan parquet (202) + : : +- Sort (209) + : : +- Exchange (208) + : : +- Filter (207) + : : +- Scan parquet (206) + : +- Sort (217) + : +- Exchange (216) + : +- Filter (215) + : +- Scan parquet (214) + +- Sort (250) + +- Exchange (249) + +- HashAggregate (248) + +- Exchange (247) + +- HashAggregate (246) + +- Project (245) + +- SortMergeJoin Inner (244) + :- Sort (239) + : +- Exchange (238) + : +- Project (237) + : +- SortMergeJoin Inner (236) + : :- Sort (231) + : : +- Exchange (230) + : : +- Filter (229) + : : +- Scan parquet (228) + : +- Sort (235) + : +- Exchange (234) + : +- Filter (233) + : +- Scan parquet (232) + +- Sort (243) + +- Exchange (242) + +- Filter (241) + +- Scan parquet (240) + + +(154) Scan parquet Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location: InMemoryFileIndex [file:/] @@ -290,7 +291,7 @@ Input [4]: [#1#1, #2#2, #3#3, #4#4] Input [4]: [#1#1, #2#2, #3#3, #4#4] Arguments: [c_customer_sk#1 ASC NULLS FIRST], false -(157) Scan parquet +(158) Scan parquet Output [3]: [ss_sold_date_sk#5, ss_customer_sk#6, ss_net_paid#7] Batched: true Location: InMemoryFileIndex [file:/] @@ -353,7 +354,7 @@ Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_sold_date_sk#5, s Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_sold_date_sk#5, ss_net_paid#7] Arguments: [ss_sold_date_sk#5 ASC NULLS FIRST], false -(165) Scan parquet +(166) Scan parquet Output [2]: [d_date_sk#8, d_year#9] Batched: true Location: InMemoryFileIndex [file:/] @@ -479,7 +480,7 @@ Input [5]: [#16#16, #17#17, #18#18, #19#19, #20#20] Input [5]: [#16#16, #17#17, #18#18, #19#19, #20#20] Arguments: [ss_sold_date_sk#19 ASC NULLS FIRST], false -(189) Scan parquet +(190) Scan parquet Output [2]: [d_date_sk#21, d_year#22] Batched: true Location: InMemoryFileIndex [file:/] @@ -607,7 +608,7 @@ Input [4]: [#29#29, #30#30, #31#31, #32#32] Input [4]: [#29#29, #30#30, #31#31, #32#32] Arguments: [c_customer_sk#29 ASC NULLS FIRST], false -(205) Scan parquet +(206) Scan parquet Output [3]: [ws_sold_date_sk#33, ws_bill_customer_sk#34, ws_net_paid#35] Batched: true Location: InMemoryFileIndex [file:/] @@ -882,503 +883,507 @@ Input [2]: [customer_id#52, year_total#53] Input [2]: [customer_id#52, year_total#53] Arguments: [customer_id#52 ASC NULLS FIRST], false -(150) SortMergeJoin [codegen id : X] +(150) NativeSortMergeJoin Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#52] Join type: Inner -Join condition: (CASE WHEN (year_total#42 > 0.00) THEN (year_total#53 / year_total#42) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#28 / year_total#15) END) +Join condition: None -(151) Project [codegen id : X] +(151) NativeFilter +Input [9]: [customer_id#14, year_total#15, customer_id#25, customer_first_name#26, customer_last_name#27, year_total#28, year_total#42, customer_id#52, year_total#53] +Condition : (CASE WHEN (year_total#42 > 0.00) THEN (year_total#53 / year_total#42) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#28 / year_total#15) END) + +(152) NativeProject Output [3]: [customer_id#25, customer_first_name#26, customer_last_name#27] Input [9]: [customer_id#14, year_total#15, customer_id#25, customer_first_name#26, customer_last_name#27, year_total#28, year_total#42, customer_id#52, year_total#53] -(152) TakeOrderedAndProject +(153) NativeTakeOrdered Input [3]: [customer_id#25, customer_first_name#26, customer_last_name#27] -Arguments: X, [customer_id#25 ASC NULLS FIRST, customer_id#25 ASC NULLS FIRST, customer_id#25 ASC NULLS FIRST], [customer_id#25, customer_first_name#26, customer_last_name#27] +Arguments: X, X, [customer_id#25 ASC NULLS FIRST, customer_id#25 ASC NULLS FIRST, customer_id#25 ASC NULLS FIRST] -(153) Scan parquet +(154) Scan parquet Output [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(154) Filter +(155) Filter Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Condition : (isnotnull(c_customer_sk#1) AND isnotnull(c_customer_id#2)) -(155) Exchange +(156) Exchange Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Arguments: hashpartitioning(c_customer_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=16] -(156) Sort +(157) Sort Input [4]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4] Arguments: [c_customer_sk#1 ASC NULLS FIRST], false, 0 -(157) Scan parquet +(158) Scan parquet Output [3]: [ss_sold_date_sk#5, ss_customer_sk#6, ss_net_paid#7] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(158) Filter +(159) Filter Input [3]: [ss_sold_date_sk#5, ss_customer_sk#6, ss_net_paid#7] Condition : (isnotnull(ss_customer_sk#6) AND isnotnull(ss_sold_date_sk#5)) -(159) Exchange +(160) Exchange Input [3]: [ss_sold_date_sk#5, ss_customer_sk#6, ss_net_paid#7] Arguments: hashpartitioning(ss_customer_sk#6, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(160) Sort +(161) Sort Input [3]: [ss_sold_date_sk#5, ss_customer_sk#6, ss_net_paid#7] Arguments: [ss_customer_sk#6 ASC NULLS FIRST], false, 0 -(161) SortMergeJoin +(162) SortMergeJoin Left keys [1]: [c_customer_sk#1] Right keys [1]: [ss_customer_sk#6] Join type: Inner Join condition: None -(162) Project +(163) Project Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_sold_date_sk#5, ss_net_paid#7] Input [7]: [c_customer_sk#1, c_customer_id#2, c_first_name#3, c_last_name#4, ss_sold_date_sk#5, ss_customer_sk#6, ss_net_paid#7] -(163) Exchange +(164) Exchange Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_sold_date_sk#5, ss_net_paid#7] Arguments: hashpartitioning(ss_sold_date_sk#5, 100), ENSURE_REQUIREMENTS, [plan_id=18] -(164) Sort +(165) Sort Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_sold_date_sk#5, ss_net_paid#7] Arguments: [ss_sold_date_sk#5 ASC NULLS FIRST], false, 0 -(165) Scan parquet +(166) Scan parquet Output [2]: [d_date_sk#8, d_year#9] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(166) Filter +(167) Filter Input [2]: [d_date_sk#8, d_year#9] Condition : (((isnotnull(d_year#9) AND (d_year#9 = 2001)) AND d_year#9 IN (2001,2002)) AND isnotnull(d_date_sk#8)) -(167) Exchange +(168) Exchange Input [2]: [d_date_sk#8, d_year#9] Arguments: hashpartitioning(d_date_sk#8, 100), ENSURE_REQUIREMENTS, [plan_id=19] -(168) Sort +(169) Sort Input [2]: [d_date_sk#8, d_year#9] Arguments: [d_date_sk#8 ASC NULLS FIRST], false, 0 -(169) SortMergeJoin +(170) SortMergeJoin Left keys [1]: [ss_sold_date_sk#5] Right keys [1]: [d_date_sk#8] Join type: Inner Join condition: None -(170) Project +(171) Project Output [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#7, d_year#9] Input [7]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_sold_date_sk#5, ss_net_paid#7, d_date_sk#8, d_year#9] -(171) HashAggregate +(172) HashAggregate Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#7, d_year#9] Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#7))] Aggregate Attributes [1]: [sum#11] Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#54] -(172) Exchange +(173) Exchange Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#54] Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, 100), ENSURE_REQUIREMENTS, [plan_id=20] -(173) HashAggregate +(174) HashAggregate Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9, sum#54] Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#9] Functions [1]: [sum(UnscaledValue(ss_net_paid#7))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#7))#13] Results [2]: [c_customer_id#2 AS customer_id#14, MakeDecimal(sum(UnscaledValue(ss_net_paid#7))#13,17,2) AS year_total#15] -(174) Filter +(175) Filter Input [2]: [customer_id#14, year_total#15] Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) -(175) Exchange +(176) Exchange Input [2]: [customer_id#14, year_total#15] Arguments: hashpartitioning(customer_id#14, 100), ENSURE_REQUIREMENTS, [plan_id=21] -(176) Sort +(177) Sort Input [2]: [customer_id#14, year_total#15] Arguments: [customer_id#14 ASC NULLS FIRST], false, 0 -(177) Scan parquet +(178) Scan parquet Output [4]: [c_customer_sk#55, c_customer_id#16, c_first_name#17, c_last_name#18] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(178) Filter +(179) Filter Input [4]: [c_customer_sk#55, c_customer_id#16, c_first_name#17, c_last_name#18] Condition : (isnotnull(c_customer_sk#55) AND isnotnull(c_customer_id#16)) -(179) Exchange +(180) Exchange Input [4]: [c_customer_sk#55, c_customer_id#16, c_first_name#17, c_last_name#18] Arguments: hashpartitioning(c_customer_sk#55, 100), ENSURE_REQUIREMENTS, [plan_id=22] -(180) Sort +(181) Sort Input [4]: [c_customer_sk#55, c_customer_id#16, c_first_name#17, c_last_name#18] Arguments: [c_customer_sk#55 ASC NULLS FIRST], false, 0 -(181) Scan parquet +(182) Scan parquet Output [3]: [ss_sold_date_sk#19, ss_customer_sk#56, ss_net_paid#20] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_customer_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(182) Filter +(183) Filter Input [3]: [ss_sold_date_sk#19, ss_customer_sk#56, ss_net_paid#20] Condition : (isnotnull(ss_customer_sk#56) AND isnotnull(ss_sold_date_sk#19)) -(183) Exchange +(184) Exchange Input [3]: [ss_sold_date_sk#19, ss_customer_sk#56, ss_net_paid#20] Arguments: hashpartitioning(ss_customer_sk#56, 100), ENSURE_REQUIREMENTS, [plan_id=23] -(184) Sort +(185) Sort Input [3]: [ss_sold_date_sk#19, ss_customer_sk#56, ss_net_paid#20] Arguments: [ss_customer_sk#56 ASC NULLS FIRST], false, 0 -(185) SortMergeJoin +(186) SortMergeJoin Left keys [1]: [c_customer_sk#55] Right keys [1]: [ss_customer_sk#56] Join type: Inner Join condition: None -(186) Project +(187) Project Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_sold_date_sk#19, ss_net_paid#20] Input [7]: [c_customer_sk#55, c_customer_id#16, c_first_name#17, c_last_name#18, ss_sold_date_sk#19, ss_customer_sk#56, ss_net_paid#20] -(187) Exchange +(188) Exchange Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_sold_date_sk#19, ss_net_paid#20] Arguments: hashpartitioning(ss_sold_date_sk#19, 100), ENSURE_REQUIREMENTS, [plan_id=24] -(188) Sort +(189) Sort Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_sold_date_sk#19, ss_net_paid#20] Arguments: [ss_sold_date_sk#19 ASC NULLS FIRST], false, 0 -(189) Scan parquet +(190) Scan parquet Output [2]: [d_date_sk#21, d_year#22] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(190) Filter +(191) Filter Input [2]: [d_date_sk#21, d_year#22] Condition : (((isnotnull(d_year#22) AND (d_year#22 = 2002)) AND d_year#22 IN (2001,2002)) AND isnotnull(d_date_sk#21)) -(191) Exchange +(192) Exchange Input [2]: [d_date_sk#21, d_year#22] Arguments: hashpartitioning(d_date_sk#21, 100), ENSURE_REQUIREMENTS, [plan_id=25] -(192) Sort +(193) Sort Input [2]: [d_date_sk#21, d_year#22] Arguments: [d_date_sk#21 ASC NULLS FIRST], false, 0 -(193) SortMergeJoin +(194) SortMergeJoin Left keys [1]: [ss_sold_date_sk#19] Right keys [1]: [d_date_sk#21] Join type: Inner Join condition: None -(194) Project +(195) Project Output [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#22] Input [7]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_sold_date_sk#19, ss_net_paid#20, d_date_sk#21, d_year#22] -(195) HashAggregate +(196) HashAggregate Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, ss_net_paid#20, d_year#22] Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#22] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#20))] Aggregate Attributes [1]: [sum#24] Results [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#22, sum#57] -(196) Exchange +(197) Exchange Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#22, sum#57] Arguments: hashpartitioning(c_customer_id#16, c_first_name#17, c_last_name#18, d_year#22, 100), ENSURE_REQUIREMENTS, [plan_id=26] -(197) HashAggregate +(198) HashAggregate Input [5]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#22, sum#57] Keys [4]: [c_customer_id#16, c_first_name#17, c_last_name#18, d_year#22] Functions [1]: [sum(UnscaledValue(ss_net_paid#20))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#20))#13] Results [4]: [c_customer_id#16 AS customer_id#25, c_first_name#17 AS customer_first_name#26, c_last_name#18 AS customer_last_name#27, MakeDecimal(sum(UnscaledValue(ss_net_paid#20))#13,17,2) AS year_total#28] -(198) Exchange +(199) Exchange Input [4]: [customer_id#25, customer_first_name#26, customer_last_name#27, year_total#28] Arguments: hashpartitioning(customer_id#25, 100), ENSURE_REQUIREMENTS, [plan_id=27] -(199) Sort +(200) Sort Input [4]: [customer_id#25, customer_first_name#26, customer_last_name#27, year_total#28] Arguments: [customer_id#25 ASC NULLS FIRST], false, 0 -(200) SortMergeJoin +(201) SortMergeJoin Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#25] Join type: Inner Join condition: None -(201) Scan parquet +(202) Scan parquet Output [4]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(202) Filter +(203) Filter Input [4]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32] Condition : (isnotnull(c_customer_sk#29) AND isnotnull(c_customer_id#30)) -(203) Exchange +(204) Exchange Input [4]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32] Arguments: hashpartitioning(c_customer_sk#29, 100), ENSURE_REQUIREMENTS, [plan_id=28] -(204) Sort +(205) Sort Input [4]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32] Arguments: [c_customer_sk#29 ASC NULLS FIRST], false, 0 -(205) Scan parquet +(206) Scan parquet Output [3]: [ws_sold_date_sk#33, ws_bill_customer_sk#34, ws_net_paid#35] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(206) Filter +(207) Filter Input [3]: [ws_sold_date_sk#33, ws_bill_customer_sk#34, ws_net_paid#35] Condition : (isnotnull(ws_bill_customer_sk#34) AND isnotnull(ws_sold_date_sk#33)) -(207) Exchange +(208) Exchange Input [3]: [ws_sold_date_sk#33, ws_bill_customer_sk#34, ws_net_paid#35] Arguments: hashpartitioning(ws_bill_customer_sk#34, 100), ENSURE_REQUIREMENTS, [plan_id=29] -(208) Sort +(209) Sort Input [3]: [ws_sold_date_sk#33, ws_bill_customer_sk#34, ws_net_paid#35] Arguments: [ws_bill_customer_sk#34 ASC NULLS FIRST], false, 0 -(209) SortMergeJoin +(210) SortMergeJoin Left keys [1]: [c_customer_sk#29] Right keys [1]: [ws_bill_customer_sk#34] Join type: Inner Join condition: None -(210) Project +(211) Project Output [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_sold_date_sk#33, ws_net_paid#35] Input [7]: [c_customer_sk#29, c_customer_id#30, c_first_name#31, c_last_name#32, ws_sold_date_sk#33, ws_bill_customer_sk#34, ws_net_paid#35] -(211) Exchange +(212) Exchange Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_sold_date_sk#33, ws_net_paid#35] Arguments: hashpartitioning(ws_sold_date_sk#33, 100), ENSURE_REQUIREMENTS, [plan_id=30] -(212) Sort +(213) Sort Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_sold_date_sk#33, ws_net_paid#35] Arguments: [ws_sold_date_sk#33 ASC NULLS FIRST], false, 0 -(213) Scan parquet +(214) Scan parquet Output [2]: [d_date_sk#36, d_year#37] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(214) Filter +(215) Filter Input [2]: [d_date_sk#36, d_year#37] Condition : (((isnotnull(d_year#37) AND (d_year#37 = 2001)) AND d_year#37 IN (2001,2002)) AND isnotnull(d_date_sk#36)) -(215) Exchange +(216) Exchange Input [2]: [d_date_sk#36, d_year#37] Arguments: hashpartitioning(d_date_sk#36, 100), ENSURE_REQUIREMENTS, [plan_id=31] -(216) Sort +(217) Sort Input [2]: [d_date_sk#36, d_year#37] Arguments: [d_date_sk#36 ASC NULLS FIRST], false, 0 -(217) SortMergeJoin +(218) SortMergeJoin Left keys [1]: [ws_sold_date_sk#33] Right keys [1]: [d_date_sk#36] Join type: Inner Join condition: None -(218) Project +(219) Project Output [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_net_paid#35, d_year#37] Input [7]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_sold_date_sk#33, ws_net_paid#35, d_date_sk#36, d_year#37] -(219) HashAggregate +(220) HashAggregate Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_net_paid#35, d_year#37] Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#37] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#35))] Aggregate Attributes [1]: [sum#39] Results [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#37, sum#58] -(220) Exchange +(221) Exchange Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#37, sum#58] Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#37, 100), ENSURE_REQUIREMENTS, [plan_id=32] -(221) HashAggregate +(222) HashAggregate Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#37, sum#58] Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#37] Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#40] Results [2]: [c_customer_id#30 AS customer_id#41, MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#40,17,2) AS year_total#42] -(222) Filter +(223) Filter Input [2]: [customer_id#41, year_total#42] Condition : (isnotnull(year_total#42) AND (year_total#42 > 0.00)) -(223) Exchange +(224) Exchange Input [2]: [customer_id#41, year_total#42] Arguments: hashpartitioning(customer_id#41, 100), ENSURE_REQUIREMENTS, [plan_id=33] -(224) Sort +(225) Sort Input [2]: [customer_id#41, year_total#42] Arguments: [customer_id#41 ASC NULLS FIRST], false, 0 -(225) SortMergeJoin +(226) SortMergeJoin Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#41] Join type: Inner Join condition: None -(226) Project +(227) Project Output [7]: [customer_id#14, year_total#15, customer_id#25, customer_first_name#26, customer_last_name#27, year_total#28, year_total#42] Input [8]: [customer_id#14, year_total#15, customer_id#25, customer_first_name#26, customer_last_name#27, year_total#28, customer_id#41, year_total#42] -(227) Scan parquet +(228) Scan parquet Output [4]: [c_customer_sk#59, c_customer_id#43, c_first_name#44, c_last_name#45] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(228) Filter +(229) Filter Input [4]: [c_customer_sk#59, c_customer_id#43, c_first_name#44, c_last_name#45] Condition : (isnotnull(c_customer_sk#59) AND isnotnull(c_customer_id#43)) -(229) Exchange +(230) Exchange Input [4]: [c_customer_sk#59, c_customer_id#43, c_first_name#44, c_last_name#45] Arguments: hashpartitioning(c_customer_sk#59, 100), ENSURE_REQUIREMENTS, [plan_id=34] -(230) Sort +(231) Sort Input [4]: [c_customer_sk#59, c_customer_id#43, c_first_name#44, c_last_name#45] Arguments: [c_customer_sk#59 ASC NULLS FIRST], false, 0 -(231) Scan parquet +(232) Scan parquet Output [3]: [ws_sold_date_sk#46, ws_bill_customer_sk#60, ws_net_paid#47] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_bill_customer_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(232) Filter +(233) Filter Input [3]: [ws_sold_date_sk#46, ws_bill_customer_sk#60, ws_net_paid#47] Condition : (isnotnull(ws_bill_customer_sk#60) AND isnotnull(ws_sold_date_sk#46)) -(233) Exchange +(234) Exchange Input [3]: [ws_sold_date_sk#46, ws_bill_customer_sk#60, ws_net_paid#47] Arguments: hashpartitioning(ws_bill_customer_sk#60, 100), ENSURE_REQUIREMENTS, [plan_id=35] -(234) Sort +(235) Sort Input [3]: [ws_sold_date_sk#46, ws_bill_customer_sk#60, ws_net_paid#47] Arguments: [ws_bill_customer_sk#60 ASC NULLS FIRST], false, 0 -(235) SortMergeJoin +(236) SortMergeJoin Left keys [1]: [c_customer_sk#59] Right keys [1]: [ws_bill_customer_sk#60] Join type: Inner Join condition: None -(236) Project +(237) Project Output [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_sold_date_sk#46, ws_net_paid#47] Input [7]: [c_customer_sk#59, c_customer_id#43, c_first_name#44, c_last_name#45, ws_sold_date_sk#46, ws_bill_customer_sk#60, ws_net_paid#47] -(237) Exchange +(238) Exchange Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_sold_date_sk#46, ws_net_paid#47] Arguments: hashpartitioning(ws_sold_date_sk#46, 100), ENSURE_REQUIREMENTS, [plan_id=36] -(238) Sort +(239) Sort Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_sold_date_sk#46, ws_net_paid#47] Arguments: [ws_sold_date_sk#46 ASC NULLS FIRST], false, 0 -(239) Scan parquet +(240) Scan parquet Output [2]: [d_date_sk#48, d_year#49] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(240) Filter +(241) Filter Input [2]: [d_date_sk#48, d_year#49] Condition : (((isnotnull(d_year#49) AND (d_year#49 = 2002)) AND d_year#49 IN (2001,2002)) AND isnotnull(d_date_sk#48)) -(241) Exchange +(242) Exchange Input [2]: [d_date_sk#48, d_year#49] Arguments: hashpartitioning(d_date_sk#48, 100), ENSURE_REQUIREMENTS, [plan_id=37] -(242) Sort +(243) Sort Input [2]: [d_date_sk#48, d_year#49] Arguments: [d_date_sk#48 ASC NULLS FIRST], false, 0 -(243) SortMergeJoin +(244) SortMergeJoin Left keys [1]: [ws_sold_date_sk#46] Right keys [1]: [d_date_sk#48] Join type: Inner Join condition: None -(244) Project +(245) Project Output [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_net_paid#47, d_year#49] Input [7]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_sold_date_sk#46, ws_net_paid#47, d_date_sk#48, d_year#49] -(245) HashAggregate +(246) HashAggregate Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_net_paid#47, d_year#49] Keys [4]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#49] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#47))] Aggregate Attributes [1]: [sum#51] Results [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#49, sum#61] -(246) Exchange +(247) Exchange Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#49, sum#61] Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#49, 100), ENSURE_REQUIREMENTS, [plan_id=38] -(247) HashAggregate +(248) HashAggregate Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#49, sum#61] Keys [4]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#49] Functions [1]: [sum(UnscaledValue(ws_net_paid#47))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#47))#40] Results [2]: [c_customer_id#43 AS customer_id#52, MakeDecimal(sum(UnscaledValue(ws_net_paid#47))#40,17,2) AS year_total#53] -(248) Exchange +(249) Exchange Input [2]: [customer_id#52, year_total#53] Arguments: hashpartitioning(customer_id#52, 100), ENSURE_REQUIREMENTS, [plan_id=39] -(249) Sort +(250) Sort Input [2]: [customer_id#52, year_total#53] Arguments: [customer_id#52 ASC NULLS FIRST], false, 0 -(250) SortMergeJoin +(251) SortMergeJoin Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#52] Join type: Inner Join condition: (CASE WHEN (year_total#42 > 0.00) THEN (year_total#53 / year_total#42) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#28 / year_total#15) END) -(251) Project +(252) Project Output [3]: [customer_id#25, customer_first_name#26, customer_last_name#27] Input [9]: [customer_id#14, year_total#15, customer_id#25, customer_first_name#26, customer_last_name#27, year_total#28, year_total#42, customer_id#52, year_total#53] -(252) TakeOrderedAndProject +(253) TakeOrderedAndProject Input [3]: [customer_id#25, customer_first_name#26, customer_last_name#27] Arguments: X, [customer_id#25 ASC NULLS FIRST, customer_id#25 ASC NULLS FIRST, customer_id#25 ASC NULLS FIRST], [customer_id#25, customer_first_name#26, customer_last_name#27] -(253) AdaptiveSparkPlan +(254) AdaptiveSparkPlan Output [3]: [customer_id#25, customer_first_name#26, customer_last_name#27] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q75.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q75.txt index 3a569e622..69d9a196e 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q75.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q75.txt @@ -1,464 +1,465 @@ == Physical Plan == -AdaptiveSparkPlan (455) +AdaptiveSparkPlan (456) +- == Final Plan == - TakeOrderedAndProject (269) - +- * Project (268) - +- * SortMergeJoin Inner (267) - :- NativeSort (160) - : +- InputAdapter (159) - : +- AQEShuffleRead (158) - : +- ShuffleQueryStage (157), Statistics(X) - : +- NativeShuffleExchange (156) - : +- NativeFilter (155) - : +- NativeProject (154) - : +- NativeHashAggregate (153) - : +- InputAdapter (152) - : +- AQEShuffleRead (151) - : +- ShuffleQueryStage (150), Statistics(X) - : +- NativeShuffleExchange (149) - : +- NativeHashAggregate (148) - : +- NativeProject (147) - : +- NativeProject (146) - : +- NativeHashAggregate (145) - : +- InputAdapter (144) - : +- AQEShuffleRead (143) - : +- ShuffleQueryStage (142), Statistics(X) - : +- NativeShuffleExchange (141) - : +- NativeHashAggregate (140) - : +- InputAdapter (139) - : +- NativeUnion (138) - : :- NativeProject (49) - : : +- NativeSortMergeJoin LeftOuter (48) - : : :- NativeSort (39) - : : : +- InputAdapter (38) - : : : +- AQEShuffleRead (37) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- NativeShuffleExchange (35) - : : : +- NativeProject (34) - : : : +- NativeSortMergeJoin Inner (33) - : : : :- NativeSort (24) - : : : : +- InputAdapter (23) - : : : : +- AQEShuffleRead (22) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- NativeShuffleExchange (20) - : : : : +- NativeProject (19) - : : : : +- NativeSortMergeJoin Inner (18) - : : : : :- NativeSort (8) - : : : : : +- InputAdapter (7) - : : : : : +- AQEShuffleRead (6) - : : : : : +- ShuffleQueryStage (5), Statistics(X) - : : : : : +- NativeShuffleExchange (4) - : : : : : +- NativeFilter (3) - : : : : : +- InputAdapter (2) - : : : : : +- NativeParquetScan (1) - : : : : +- NativeSort (17) - : : : : +- InputAdapter (16) - : : : : +- AQEShuffleRead (15) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- NativeShuffleExchange (13) - : : : : +- NativeProject (12) - : : : : +- NativeFilter (11) - : : : : +- InputAdapter (10) - : : : : +- NativeParquetScan (9) - : : : +- NativeSort (32) - : : : +- InputAdapter (31) - : : : +- AQEShuffleRead (30) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- NativeShuffleExchange (28) - : : : +- NativeFilter (27) - : : : +- InputAdapter (26) - : : : +- NativeParquetScan (25) - : : +- NativeSort (47) - : : +- InputAdapter (46) - : : +- AQEShuffleRead (45) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- NativeShuffleExchange (43) - : : +- NativeFilter (42) - : : +- InputAdapter (41) - : : +- NativeParquetScan (40) - : :- NativeProject (93) - : : +- NativeSortMergeJoin LeftOuter (92) - : : :- NativeSort (83) - : : : +- InputAdapter (82) - : : : +- AQEShuffleRead (81) - : : : +- ShuffleQueryStage (80), Statistics(X) - : : : +- NativeShuffleExchange (79) - : : : +- NativeProject (78) - : : : +- NativeSortMergeJoin Inner (77) - : : : :- NativeSort (70) - : : : : +- InputAdapter (69) - : : : : +- AQEShuffleRead (68) - : : : : +- ShuffleQueryStage (67), Statistics(X) - : : : : +- NativeShuffleExchange (66) - : : : : +- NativeProject (65) - : : : : +- NativeSortMergeJoin Inner (64) - : : : : :- NativeSort (57) - : : : : : +- InputAdapter (56) - : : : : : +- AQEShuffleRead (55) - : : : : : +- ShuffleQueryStage (54), Statistics(X) - : : : : : +- NativeShuffleExchange (53) - : : : : : +- NativeFilter (52) - : : : : : +- InputAdapter (51) - : : : : : +- NativeParquetScan (50) - : : : : +- NativeSort (63) - : : : : +- InputAdapter (62) - : : : : +- InputAdapter (61) - : : : : +- AQEShuffleRead (60) - : : : : +- ShuffleQueryStage (59), Statistics(X) - : : : : +- ReusedExchange (58) - : : : +- NativeSort (76) - : : : +- InputAdapter (75) - : : : +- InputAdapter (74) - : : : +- AQEShuffleRead (73) - : : : +- ShuffleQueryStage (72), Statistics(X) - : : : +- ReusedExchange (71) - : : +- NativeSort (91) - : : +- InputAdapter (90) - : : +- AQEShuffleRead (89) - : : +- ShuffleQueryStage (88), Statistics(X) - : : +- NativeShuffleExchange (87) - : : +- NativeFilter (86) - : : +- InputAdapter (85) - : : +- NativeParquetScan (84) - : +- NativeProject (137) - : +- NativeSortMergeJoin LeftOuter (136) - : :- NativeSort (127) - : : +- InputAdapter (126) - : : +- AQEShuffleRead (125) - : : +- ShuffleQueryStage (124), Statistics(X) - : : +- NativeShuffleExchange (123) - : : +- NativeProject (122) - : : +- NativeSortMergeJoin Inner (121) - : : :- NativeSort (114) - : : : +- InputAdapter (113) - : : : +- AQEShuffleRead (112) - : : : +- ShuffleQueryStage (111), Statistics(X) - : : : +- NativeShuffleExchange (110) - : : : +- NativeProject (109) - : : : +- NativeSortMergeJoin Inner (108) - : : : :- NativeSort (101) - : : : : +- InputAdapter (100) - : : : : +- AQEShuffleRead (99) - : : : : +- ShuffleQueryStage (98), Statistics(X) - : : : : +- NativeShuffleExchange (97) - : : : : +- NativeFilter (96) - : : : : +- InputAdapter (95) - : : : : +- NativeParquetScan (94) - : : : +- NativeSort (107) - : : : +- InputAdapter (106) - : : : +- InputAdapter (105) - : : : +- AQEShuffleRead (104) - : : : +- ShuffleQueryStage (103), Statistics(X) - : : : +- ReusedExchange (102) - : : +- NativeSort (120) - : : +- InputAdapter (119) - : : +- InputAdapter (118) - : : +- AQEShuffleRead (117) - : : +- ShuffleQueryStage (116), Statistics(X) - : : +- ReusedExchange (115) - : +- NativeSort (135) - : +- InputAdapter (134) - : +- AQEShuffleRead (133) - : +- ShuffleQueryStage (132), Statistics(X) - : +- NativeShuffleExchange (131) - : +- NativeFilter (130) - : +- InputAdapter (129) - : +- NativeParquetScan (128) - +- NativeSort (266) - +- InputAdapter (265) - +- AQEShuffleRead (264) - +- ShuffleQueryStage (263), Statistics(X) - +- NativeShuffleExchange (262) - +- NativeFilter (261) - +- NativeProject (260) - +- NativeHashAggregate (259) - +- InputAdapter (258) - +- AQEShuffleRead (257) - +- ShuffleQueryStage (256), Statistics(X) - +- NativeShuffleExchange (255) - +- NativeHashAggregate (254) - +- NativeProject (253) - +- NativeProject (252) - +- NativeHashAggregate (251) - +- InputAdapter (250) - +- AQEShuffleRead (249) - +- ShuffleQueryStage (248), Statistics(X) - +- NativeShuffleExchange (247) - +- NativeHashAggregate (246) - +- InputAdapter (245) - +- NativeUnion (244) - :- NativeProject (189) - : +- NativeSortMergeJoin LeftOuter (188) - : :- NativeSort (181) - : : +- InputAdapter (180) - : : +- AQEShuffleRead (179) - : : +- ShuffleQueryStage (178), Statistics(X) - : : +- NativeShuffleExchange (177) - : : +- NativeProject (176) - : : +- NativeSortMergeJoin Inner (175) - : : :- NativeSort (166) - : : : +- InputAdapter (165) - : : : +- InputAdapter (164) - : : : +- AQEShuffleRead (163) - : : : +- ShuffleQueryStage (162), Statistics(X) - : : : +- ReusedExchange (161) - : : +- NativeSort (174) - : : +- InputAdapter (173) - : : +- AQEShuffleRead (172) - : : +- ShuffleQueryStage (171), Statistics(X) - : : +- NativeShuffleExchange (170) - : : +- NativeFilter (169) - : : +- InputAdapter (168) - : : +- NativeParquetScan (167) - : +- NativeSort (187) - : +- InputAdapter (186) - : +- InputAdapter (185) - : +- AQEShuffleRead (184) - : +- ShuffleQueryStage (183), Statistics(X) - : +- ReusedExchange (182) - :- NativeProject (216) - : +- NativeSortMergeJoin LeftOuter (215) - : :- NativeSort (208) - : : +- InputAdapter (207) - : : +- AQEShuffleRead (206) - : : +- ShuffleQueryStage (205), Statistics(X) - : : +- NativeShuffleExchange (204) - : : +- NativeProject (203) - : : +- NativeSortMergeJoin Inner (202) - : : :- NativeSort (195) - : : : +- InputAdapter (194) - : : : +- InputAdapter (193) - : : : +- AQEShuffleRead (192) - : : : +- ShuffleQueryStage (191), Statistics(X) - : : : +- ReusedExchange (190) - : : +- NativeSort (201) - : : +- InputAdapter (200) - : : +- InputAdapter (199) - : : +- AQEShuffleRead (198) - : : +- ShuffleQueryStage (197), Statistics(X) - : : +- ReusedExchange (196) - : +- NativeSort (214) - : +- InputAdapter (213) - : +- InputAdapter (212) - : +- AQEShuffleRead (211) - : +- ShuffleQueryStage (210), Statistics(X) - : +- ReusedExchange (209) - +- NativeProject (243) - +- NativeSortMergeJoin LeftOuter (242) - :- NativeSort (235) - : +- InputAdapter (234) - : +- AQEShuffleRead (233) - : +- ShuffleQueryStage (232), Statistics(X) - : +- NativeShuffleExchange (231) - : +- NativeProject (230) - : +- NativeSortMergeJoin Inner (229) - : :- NativeSort (222) - : : +- InputAdapter (221) - : : +- InputAdapter (220) - : : +- AQEShuffleRead (219) - : : +- ShuffleQueryStage (218), Statistics(X) - : : +- ReusedExchange (217) - : +- NativeSort (228) - : +- InputAdapter (227) - : +- InputAdapter (226) - : +- AQEShuffleRead (225) - : +- ShuffleQueryStage (224), Statistics(X) - : +- ReusedExchange (223) - +- NativeSort (241) - +- InputAdapter (240) - +- InputAdapter (239) - +- AQEShuffleRead (238) - +- ShuffleQueryStage (237), Statistics(X) - +- ReusedExchange (236) + NativeTakeOrdered (270) + +- NativeProject (269) + +- NativeFilter (268) + +- NativeSortMergeJoin Inner (267) + :- NativeSort (160) + : +- InputAdapter (159) + : +- AQEShuffleRead (158) + : +- ShuffleQueryStage (157), Statistics(X) + : +- NativeShuffleExchange (156) + : +- NativeFilter (155) + : +- NativeProject (154) + : +- NativeHashAggregate (153) + : +- InputAdapter (152) + : +- AQEShuffleRead (151) + : +- ShuffleQueryStage (150), Statistics(X) + : +- NativeShuffleExchange (149) + : +- NativeHashAggregate (148) + : +- NativeProject (147) + : +- NativeProject (146) + : +- NativeHashAggregate (145) + : +- InputAdapter (144) + : +- AQEShuffleRead (143) + : +- ShuffleQueryStage (142), Statistics(X) + : +- NativeShuffleExchange (141) + : +- NativeHashAggregate (140) + : +- InputAdapter (139) + : +- NativeUnion (138) + : :- NativeProject (49) + : : +- NativeSortMergeJoin LeftOuter (48) + : : :- NativeSort (39) + : : : +- InputAdapter (38) + : : : +- AQEShuffleRead (37) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- NativeShuffleExchange (35) + : : : +- NativeProject (34) + : : : +- NativeSortMergeJoin Inner (33) + : : : :- NativeSort (24) + : : : : +- InputAdapter (23) + : : : : +- AQEShuffleRead (22) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- NativeShuffleExchange (20) + : : : : +- NativeProject (19) + : : : : +- NativeSortMergeJoin Inner (18) + : : : : :- NativeSort (8) + : : : : : +- InputAdapter (7) + : : : : : +- AQEShuffleRead (6) + : : : : : +- ShuffleQueryStage (5), Statistics(X) + : : : : : +- NativeShuffleExchange (4) + : : : : : +- NativeFilter (3) + : : : : : +- InputAdapter (2) + : : : : : +- NativeParquetScan (1) + : : : : +- NativeSort (17) + : : : : +- InputAdapter (16) + : : : : +- AQEShuffleRead (15) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- NativeShuffleExchange (13) + : : : : +- NativeProject (12) + : : : : +- NativeFilter (11) + : : : : +- InputAdapter (10) + : : : : +- NativeParquetScan (9) + : : : +- NativeSort (32) + : : : +- InputAdapter (31) + : : : +- AQEShuffleRead (30) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- NativeShuffleExchange (28) + : : : +- NativeFilter (27) + : : : +- InputAdapter (26) + : : : +- NativeParquetScan (25) + : : +- NativeSort (47) + : : +- InputAdapter (46) + : : +- AQEShuffleRead (45) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- NativeShuffleExchange (43) + : : +- NativeFilter (42) + : : +- InputAdapter (41) + : : +- NativeParquetScan (40) + : :- NativeProject (93) + : : +- NativeSortMergeJoin LeftOuter (92) + : : :- NativeSort (83) + : : : +- InputAdapter (82) + : : : +- AQEShuffleRead (81) + : : : +- ShuffleQueryStage (80), Statistics(X) + : : : +- NativeShuffleExchange (79) + : : : +- NativeProject (78) + : : : +- NativeSortMergeJoin Inner (77) + : : : :- NativeSort (70) + : : : : +- InputAdapter (69) + : : : : +- AQEShuffleRead (68) + : : : : +- ShuffleQueryStage (67), Statistics(X) + : : : : +- NativeShuffleExchange (66) + : : : : +- NativeProject (65) + : : : : +- NativeSortMergeJoin Inner (64) + : : : : :- NativeSort (57) + : : : : : +- InputAdapter (56) + : : : : : +- AQEShuffleRead (55) + : : : : : +- ShuffleQueryStage (54), Statistics(X) + : : : : : +- NativeShuffleExchange (53) + : : : : : +- NativeFilter (52) + : : : : : +- InputAdapter (51) + : : : : : +- NativeParquetScan (50) + : : : : +- NativeSort (63) + : : : : +- InputAdapter (62) + : : : : +- InputAdapter (61) + : : : : +- AQEShuffleRead (60) + : : : : +- ShuffleQueryStage (59), Statistics(X) + : : : : +- ReusedExchange (58) + : : : +- NativeSort (76) + : : : +- InputAdapter (75) + : : : +- InputAdapter (74) + : : : +- AQEShuffleRead (73) + : : : +- ShuffleQueryStage (72), Statistics(X) + : : : +- ReusedExchange (71) + : : +- NativeSort (91) + : : +- InputAdapter (90) + : : +- AQEShuffleRead (89) + : : +- ShuffleQueryStage (88), Statistics(X) + : : +- NativeShuffleExchange (87) + : : +- NativeFilter (86) + : : +- InputAdapter (85) + : : +- NativeParquetScan (84) + : +- NativeProject (137) + : +- NativeSortMergeJoin LeftOuter (136) + : :- NativeSort (127) + : : +- InputAdapter (126) + : : +- AQEShuffleRead (125) + : : +- ShuffleQueryStage (124), Statistics(X) + : : +- NativeShuffleExchange (123) + : : +- NativeProject (122) + : : +- NativeSortMergeJoin Inner (121) + : : :- NativeSort (114) + : : : +- InputAdapter (113) + : : : +- AQEShuffleRead (112) + : : : +- ShuffleQueryStage (111), Statistics(X) + : : : +- NativeShuffleExchange (110) + : : : +- NativeProject (109) + : : : +- NativeSortMergeJoin Inner (108) + : : : :- NativeSort (101) + : : : : +- InputAdapter (100) + : : : : +- AQEShuffleRead (99) + : : : : +- ShuffleQueryStage (98), Statistics(X) + : : : : +- NativeShuffleExchange (97) + : : : : +- NativeFilter (96) + : : : : +- InputAdapter (95) + : : : : +- NativeParquetScan (94) + : : : +- NativeSort (107) + : : : +- InputAdapter (106) + : : : +- InputAdapter (105) + : : : +- AQEShuffleRead (104) + : : : +- ShuffleQueryStage (103), Statistics(X) + : : : +- ReusedExchange (102) + : : +- NativeSort (120) + : : +- InputAdapter (119) + : : +- InputAdapter (118) + : : +- AQEShuffleRead (117) + : : +- ShuffleQueryStage (116), Statistics(X) + : : +- ReusedExchange (115) + : +- NativeSort (135) + : +- InputAdapter (134) + : +- AQEShuffleRead (133) + : +- ShuffleQueryStage (132), Statistics(X) + : +- NativeShuffleExchange (131) + : +- NativeFilter (130) + : +- InputAdapter (129) + : +- NativeParquetScan (128) + +- NativeSort (266) + +- InputAdapter (265) + +- AQEShuffleRead (264) + +- ShuffleQueryStage (263), Statistics(X) + +- NativeShuffleExchange (262) + +- NativeFilter (261) + +- NativeProject (260) + +- NativeHashAggregate (259) + +- InputAdapter (258) + +- AQEShuffleRead (257) + +- ShuffleQueryStage (256), Statistics(X) + +- NativeShuffleExchange (255) + +- NativeHashAggregate (254) + +- NativeProject (253) + +- NativeProject (252) + +- NativeHashAggregate (251) + +- InputAdapter (250) + +- AQEShuffleRead (249) + +- ShuffleQueryStage (248), Statistics(X) + +- NativeShuffleExchange (247) + +- NativeHashAggregate (246) + +- InputAdapter (245) + +- NativeUnion (244) + :- NativeProject (189) + : +- NativeSortMergeJoin LeftOuter (188) + : :- NativeSort (181) + : : +- InputAdapter (180) + : : +- AQEShuffleRead (179) + : : +- ShuffleQueryStage (178), Statistics(X) + : : +- NativeShuffleExchange (177) + : : +- NativeProject (176) + : : +- NativeSortMergeJoin Inner (175) + : : :- NativeSort (166) + : : : +- InputAdapter (165) + : : : +- InputAdapter (164) + : : : +- AQEShuffleRead (163) + : : : +- ShuffleQueryStage (162), Statistics(X) + : : : +- ReusedExchange (161) + : : +- NativeSort (174) + : : +- InputAdapter (173) + : : +- AQEShuffleRead (172) + : : +- ShuffleQueryStage (171), Statistics(X) + : : +- NativeShuffleExchange (170) + : : +- NativeFilter (169) + : : +- InputAdapter (168) + : : +- NativeParquetScan (167) + : +- NativeSort (187) + : +- InputAdapter (186) + : +- InputAdapter (185) + : +- AQEShuffleRead (184) + : +- ShuffleQueryStage (183), Statistics(X) + : +- ReusedExchange (182) + :- NativeProject (216) + : +- NativeSortMergeJoin LeftOuter (215) + : :- NativeSort (208) + : : +- InputAdapter (207) + : : +- AQEShuffleRead (206) + : : +- ShuffleQueryStage (205), Statistics(X) + : : +- NativeShuffleExchange (204) + : : +- NativeProject (203) + : : +- NativeSortMergeJoin Inner (202) + : : :- NativeSort (195) + : : : +- InputAdapter (194) + : : : +- InputAdapter (193) + : : : +- AQEShuffleRead (192) + : : : +- ShuffleQueryStage (191), Statistics(X) + : : : +- ReusedExchange (190) + : : +- NativeSort (201) + : : +- InputAdapter (200) + : : +- InputAdapter (199) + : : +- AQEShuffleRead (198) + : : +- ShuffleQueryStage (197), Statistics(X) + : : +- ReusedExchange (196) + : +- NativeSort (214) + : +- InputAdapter (213) + : +- InputAdapter (212) + : +- AQEShuffleRead (211) + : +- ShuffleQueryStage (210), Statistics(X) + : +- ReusedExchange (209) + +- NativeProject (243) + +- NativeSortMergeJoin LeftOuter (242) + :- NativeSort (235) + : +- InputAdapter (234) + : +- AQEShuffleRead (233) + : +- ShuffleQueryStage (232), Statistics(X) + : +- NativeShuffleExchange (231) + : +- NativeProject (230) + : +- NativeSortMergeJoin Inner (229) + : :- NativeSort (222) + : : +- InputAdapter (221) + : : +- InputAdapter (220) + : : +- AQEShuffleRead (219) + : : +- ShuffleQueryStage (218), Statistics(X) + : : +- ReusedExchange (217) + : +- NativeSort (228) + : +- InputAdapter (227) + : +- InputAdapter (226) + : +- AQEShuffleRead (225) + : +- ShuffleQueryStage (224), Statistics(X) + : +- ReusedExchange (223) + +- NativeSort (241) + +- InputAdapter (240) + +- InputAdapter (239) + +- AQEShuffleRead (238) + +- ShuffleQueryStage (237), Statistics(X) + +- ReusedExchange (236) +- == Initial Plan == - TakeOrderedAndProject (454) - +- Project (453) - +- SortMergeJoin Inner (452) - :- Sort (360) - : +- Exchange (359) - : +- Filter (358) - : +- HashAggregate (357) - : +- Exchange (356) - : +- HashAggregate (355) - : +- HashAggregate (354) - : +- Exchange (353) - : +- HashAggregate (352) - : +- Union (351) - : :- Project (296) - : : +- SortMergeJoin LeftOuter (295) - : : :- Sort (290) - : : : +- Exchange (289) - : : : +- Project (288) - : : : +- SortMergeJoin Inner (287) - : : : :- Sort (282) - : : : : +- Exchange (281) - : : : : +- Project (280) - : : : : +- SortMergeJoin Inner (279) - : : : : :- Sort (273) - : : : : : +- Exchange (272) - : : : : : +- Filter (271) - : : : : : +- Scan parquet (270) - : : : : +- Sort (278) - : : : : +- Exchange (277) - : : : : +- Project (276) - : : : : +- Filter (275) - : : : : +- Scan parquet (274) - : : : +- Sort (286) - : : : +- Exchange (285) - : : : +- Filter (284) - : : : +- Scan parquet (283) - : : +- Sort (294) - : : +- Exchange (293) - : : +- Filter (292) - : : +- Scan parquet (291) - : :- Project (323) - : : +- SortMergeJoin LeftOuter (322) - : : :- Sort (317) - : : : +- Exchange (316) - : : : +- Project (315) - : : : +- SortMergeJoin Inner (314) - : : : :- Sort (309) - : : : : +- Exchange (308) - : : : : +- Project (307) - : : : : +- SortMergeJoin Inner (306) - : : : : :- Sort (300) - : : : : : +- Exchange (299) - : : : : : +- Filter (298) - : : : : : +- Scan parquet (297) - : : : : +- Sort (305) - : : : : +- Exchange (304) - : : : : +- Project (303) - : : : : +- Filter (302) - : : : : +- Scan parquet (301) - : : : +- Sort (313) - : : : +- Exchange (312) - : : : +- Filter (311) - : : : +- Scan parquet (310) - : : +- Sort (321) - : : +- Exchange (320) - : : +- Filter (319) - : : +- Scan parquet (318) - : +- Project (350) - : +- SortMergeJoin LeftOuter (349) - : :- Sort (344) - : : +- Exchange (343) - : : +- Project (342) - : : +- SortMergeJoin Inner (341) - : : :- Sort (336) - : : : +- Exchange (335) - : : : +- Project (334) - : : : +- SortMergeJoin Inner (333) - : : : :- Sort (327) - : : : : +- Exchange (326) - : : : : +- Filter (325) - : : : : +- Scan parquet (324) - : : : +- Sort (332) - : : : +- Exchange (331) - : : : +- Project (330) - : : : +- Filter (329) - : : : +- Scan parquet (328) - : : +- Sort (340) - : : +- Exchange (339) - : : +- Filter (338) - : : +- Scan parquet (337) - : +- Sort (348) - : +- Exchange (347) - : +- Filter (346) - : +- Scan parquet (345) - +- Sort (451) - +- Exchange (450) - +- Filter (449) - +- HashAggregate (448) - +- Exchange (447) - +- HashAggregate (446) - +- HashAggregate (445) - +- Exchange (444) - +- HashAggregate (443) - +- Union (442) - :- Project (387) - : +- SortMergeJoin LeftOuter (386) - : :- Sort (381) - : : +- Exchange (380) - : : +- Project (379) - : : +- SortMergeJoin Inner (378) - : : :- Sort (373) - : : : +- Exchange (372) - : : : +- Project (371) - : : : +- SortMergeJoin Inner (370) - : : : :- Sort (364) - : : : : +- Exchange (363) - : : : : +- Filter (362) - : : : : +- Scan parquet (361) - : : : +- Sort (369) - : : : +- Exchange (368) - : : : +- Project (367) - : : : +- Filter (366) - : : : +- Scan parquet (365) - : : +- Sort (377) - : : +- Exchange (376) - : : +- Filter (375) - : : +- Scan parquet (374) - : +- Sort (385) - : +- Exchange (384) - : +- Filter (383) - : +- Scan parquet (382) - :- Project (414) - : +- SortMergeJoin LeftOuter (413) - : :- Sort (408) - : : +- Exchange (407) - : : +- Project (406) - : : +- SortMergeJoin Inner (405) - : : :- Sort (400) - : : : +- Exchange (399) - : : : +- Project (398) - : : : +- SortMergeJoin Inner (397) - : : : :- Sort (391) - : : : : +- Exchange (390) - : : : : +- Filter (389) - : : : : +- Scan parquet (388) - : : : +- Sort (396) - : : : +- Exchange (395) - : : : +- Project (394) - : : : +- Filter (393) - : : : +- Scan parquet (392) - : : +- Sort (404) - : : +- Exchange (403) - : : +- Filter (402) - : : +- Scan parquet (401) - : +- Sort (412) - : +- Exchange (411) - : +- Filter (410) - : +- Scan parquet (409) - +- Project (441) - +- SortMergeJoin LeftOuter (440) - :- Sort (435) - : +- Exchange (434) - : +- Project (433) - : +- SortMergeJoin Inner (432) - : :- Sort (427) - : : +- Exchange (426) - : : +- Project (425) - : : +- SortMergeJoin Inner (424) - : : :- Sort (418) - : : : +- Exchange (417) - : : : +- Filter (416) - : : : +- Scan parquet (415) - : : +- Sort (423) - : : +- Exchange (422) - : : +- Project (421) - : : +- Filter (420) - : : +- Scan parquet (419) - : +- Sort (431) - : +- Exchange (430) - : +- Filter (429) - : +- Scan parquet (428) - +- Sort (439) - +- Exchange (438) - +- Filter (437) - +- Scan parquet (436) - - -(270) Scan parquet + TakeOrderedAndProject (455) + +- Project (454) + +- SortMergeJoin Inner (453) + :- Sort (361) + : +- Exchange (360) + : +- Filter (359) + : +- HashAggregate (358) + : +- Exchange (357) + : +- HashAggregate (356) + : +- HashAggregate (355) + : +- Exchange (354) + : +- HashAggregate (353) + : +- Union (352) + : :- Project (297) + : : +- SortMergeJoin LeftOuter (296) + : : :- Sort (291) + : : : +- Exchange (290) + : : : +- Project (289) + : : : +- SortMergeJoin Inner (288) + : : : :- Sort (283) + : : : : +- Exchange (282) + : : : : +- Project (281) + : : : : +- SortMergeJoin Inner (280) + : : : : :- Sort (274) + : : : : : +- Exchange (273) + : : : : : +- Filter (272) + : : : : : +- Scan parquet (271) + : : : : +- Sort (279) + : : : : +- Exchange (278) + : : : : +- Project (277) + : : : : +- Filter (276) + : : : : +- Scan parquet (275) + : : : +- Sort (287) + : : : +- Exchange (286) + : : : +- Filter (285) + : : : +- Scan parquet (284) + : : +- Sort (295) + : : +- Exchange (294) + : : +- Filter (293) + : : +- Scan parquet (292) + : :- Project (324) + : : +- SortMergeJoin LeftOuter (323) + : : :- Sort (318) + : : : +- Exchange (317) + : : : +- Project (316) + : : : +- SortMergeJoin Inner (315) + : : : :- Sort (310) + : : : : +- Exchange (309) + : : : : +- Project (308) + : : : : +- SortMergeJoin Inner (307) + : : : : :- Sort (301) + : : : : : +- Exchange (300) + : : : : : +- Filter (299) + : : : : : +- Scan parquet (298) + : : : : +- Sort (306) + : : : : +- Exchange (305) + : : : : +- Project (304) + : : : : +- Filter (303) + : : : : +- Scan parquet (302) + : : : +- Sort (314) + : : : +- Exchange (313) + : : : +- Filter (312) + : : : +- Scan parquet (311) + : : +- Sort (322) + : : +- Exchange (321) + : : +- Filter (320) + : : +- Scan parquet (319) + : +- Project (351) + : +- SortMergeJoin LeftOuter (350) + : :- Sort (345) + : : +- Exchange (344) + : : +- Project (343) + : : +- SortMergeJoin Inner (342) + : : :- Sort (337) + : : : +- Exchange (336) + : : : +- Project (335) + : : : +- SortMergeJoin Inner (334) + : : : :- Sort (328) + : : : : +- Exchange (327) + : : : : +- Filter (326) + : : : : +- Scan parquet (325) + : : : +- Sort (333) + : : : +- Exchange (332) + : : : +- Project (331) + : : : +- Filter (330) + : : : +- Scan parquet (329) + : : +- Sort (341) + : : +- Exchange (340) + : : +- Filter (339) + : : +- Scan parquet (338) + : +- Sort (349) + : +- Exchange (348) + : +- Filter (347) + : +- Scan parquet (346) + +- Sort (452) + +- Exchange (451) + +- Filter (450) + +- HashAggregate (449) + +- Exchange (448) + +- HashAggregate (447) + +- HashAggregate (446) + +- Exchange (445) + +- HashAggregate (444) + +- Union (443) + :- Project (388) + : +- SortMergeJoin LeftOuter (387) + : :- Sort (382) + : : +- Exchange (381) + : : +- Project (380) + : : +- SortMergeJoin Inner (379) + : : :- Sort (374) + : : : +- Exchange (373) + : : : +- Project (372) + : : : +- SortMergeJoin Inner (371) + : : : :- Sort (365) + : : : : +- Exchange (364) + : : : : +- Filter (363) + : : : : +- Scan parquet (362) + : : : +- Sort (370) + : : : +- Exchange (369) + : : : +- Project (368) + : : : +- Filter (367) + : : : +- Scan parquet (366) + : : +- Sort (378) + : : +- Exchange (377) + : : +- Filter (376) + : : +- Scan parquet (375) + : +- Sort (386) + : +- Exchange (385) + : +- Filter (384) + : +- Scan parquet (383) + :- Project (415) + : +- SortMergeJoin LeftOuter (414) + : :- Sort (409) + : : +- Exchange (408) + : : +- Project (407) + : : +- SortMergeJoin Inner (406) + : : :- Sort (401) + : : : +- Exchange (400) + : : : +- Project (399) + : : : +- SortMergeJoin Inner (398) + : : : :- Sort (392) + : : : : +- Exchange (391) + : : : : +- Filter (390) + : : : : +- Scan parquet (389) + : : : +- Sort (397) + : : : +- Exchange (396) + : : : +- Project (395) + : : : +- Filter (394) + : : : +- Scan parquet (393) + : : +- Sort (405) + : : +- Exchange (404) + : : +- Filter (403) + : : +- Scan parquet (402) + : +- Sort (413) + : +- Exchange (412) + : +- Filter (411) + : +- Scan parquet (410) + +- Project (442) + +- SortMergeJoin LeftOuter (441) + :- Sort (436) + : +- Exchange (435) + : +- Project (434) + : +- SortMergeJoin Inner (433) + : :- Sort (428) + : : +- Exchange (427) + : : +- Project (426) + : : +- SortMergeJoin Inner (425) + : : :- Sort (419) + : : : +- Exchange (418) + : : : +- Filter (417) + : : : +- Scan parquet (416) + : : +- Sort (424) + : : +- Exchange (423) + : : +- Project (422) + : : +- Filter (421) + : : +- Scan parquet (420) + : +- Sort (432) + : +- Exchange (431) + : +- Filter (430) + : +- Scan parquet (429) + +- Sort (440) + +- Exchange (439) + +- Filter (438) + +- Scan parquet (437) + + +(271) Scan parquet Output [5]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5] Batched: true Location: InMemoryFileIndex [file:/] @@ -492,7 +493,7 @@ Input [5]: [#1#1, #2#2, #3#3, #4#4, #5#5] Input [5]: [#1#1, #2#2, #3#3, #4#4, #5#5] Arguments: [cs_item_sk#2 ASC NULLS FIRST], false -(274) Scan parquet +(275) Scan parquet Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location: InMemoryFileIndex [file:/] @@ -559,7 +560,7 @@ Input [9]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, c Input [9]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: [cs_sold_date_sk#1 ASC NULLS FIRST], false -(283) Scan parquet +(284) Scan parquet Output [2]: [d_date_sk#12, d_year#13] Batched: true Location: InMemoryFileIndex [file:/] @@ -622,7 +623,7 @@ Input [9]: [cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5 Input [9]: [cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST], false -(291) Scan parquet +(292) Scan parquet Output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Batched: true Location: InMemoryFileIndex [file:/] @@ -666,7 +667,7 @@ Join condition: None Output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#4 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#18, (cs_ext_sales_price#5 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#19] Input [13]: [cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, #14#14, #15#15, #16#16, #17#17] -(297) Scan parquet +(298) Scan parquet Output [5]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24] Batched: true Location: InMemoryFileIndex [file:/] @@ -802,7 +803,7 @@ Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_pri Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, d_year#31] Arguments: [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST], false -(318) Scan parquet +(319) Scan parquet Output [4]: [sr_item_sk#32, sr_ticket_number#33, sr_return_quantity#34, sr_return_amt#35] Batched: true Location: InMemoryFileIndex [file:/] @@ -846,7 +847,7 @@ Join condition: None Output [7]: [d_year#31, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, (ss_quantity#23 - coalesce(sr_return_quantity#34, 0)) AS sales_cnt#36, (ss_ext_sales_price#24 - coalesce(sr_return_amt#35, 0.00)) AS sales_amt#37] Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, d_year#31, #32#32, #33#33, #34#34, #35#35] -(324) Scan parquet +(325) Scan parquet Output [5]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42] Batched: true Location: InMemoryFileIndex [file:/] @@ -982,7 +983,7 @@ Input [9]: [ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_pric Input [9]: [ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47, d_year#49] Arguments: [ws_order_number#40 ASC NULLS FIRST, ws_item_sk#39 ASC NULLS FIRST], false -(345) Scan parquet +(346) Scan parquet Output [4]: [wr_item_sk#50, wr_order_number#51, wr_return_quantity#52, wr_return_amt#53] Batched: true Location: InMemoryFileIndex [file:/] @@ -1148,7 +1149,7 @@ Input [9]: [#65#65, #66#66, #67#67, #68#68, #69#69, #70#70, #71#71, #72#72, #73# Input [9]: [#65#65, #66#66, #67#67, #68#68, #69#69, #70#70, #71#71, #72#72, #73#73] Arguments: [cs_sold_date_sk#65 ASC NULLS FIRST], false -(374) Scan parquet +(375) Scan parquet Output [2]: [d_date_sk#74, d_year#75] Batched: true Location: InMemoryFileIndex [file:/] @@ -1553,891 +1554,895 @@ Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufac Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#113, sales_amt#114] Arguments: [i_brand_id#70 ASC NULLS FIRST, i_class_id#71 ASC NULLS FIRST, i_category_id#72 ASC NULLS FIRST, i_manufact_id#73 ASC NULLS FIRST], false -(267) SortMergeJoin [codegen id : X] +(267) NativeSortMergeJoin Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Right keys [4]: [i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Join type: Inner -Join condition: ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#113 as decimal(17,2))) < 0.90000000000000000000) +Join condition: None -(268) Project [codegen id : X] +(268) NativeFilter +Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#113, sales_amt#114] +Condition : ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#113 as decimal(17,2))) < 0.90000000000000000000) + +(269) NativeProject Output [10]: [d_year#75 AS prev_year#115, d_year#13 AS year#116, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#113 AS prev_yr_cnt#117, sales_cnt#63 AS curr_yr_cnt#118, (sales_cnt#63 - sales_cnt#113) AS sales_cnt_diff#119, (sales_amt#64 - sales_amt#114) AS sales_amt_diff#120] Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#113, sales_amt#114] -(269) TakeOrderedAndProject +(270) NativeTakeOrdered Input [10]: [prev_year#115, year#116, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#117, curr_yr_cnt#118, sales_cnt_diff#119, sales_amt_diff#120] -Arguments: X, [sales_cnt_diff#119 ASC NULLS FIRST, sales_amt_diff#120 ASC NULLS FIRST], [prev_year#115, year#116, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#117, curr_yr_cnt#118, sales_cnt_diff#119, sales_amt_diff#120] +Arguments: X, X, [sales_cnt_diff#119 ASC NULLS FIRST, sales_amt_diff#120 ASC NULLS FIRST] -(270) Scan parquet +(271) Scan parquet Output [5]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_date_sk)] ReadSchema: struct -(271) Filter +(272) Filter Input [5]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5] Condition : (isnotnull(cs_item_sk#2) AND isnotnull(cs_sold_date_sk#1)) -(272) Exchange +(273) Exchange Input [5]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5] Arguments: hashpartitioning(cs_item_sk#2, 100), ENSURE_REQUIREMENTS, [plan_id=25] -(273) Sort +(274) Sort Input [5]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5] Arguments: [cs_item_sk#2 ASC NULLS FIRST], false, 0 -(274) Scan parquet +(275) Scan parquet Output [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(275) Filter +(276) Filter Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] Condition : ((((((isnotnull(i_category#10) AND (i_category#10 = Books)) AND isnotnull(i_item_sk#6)) AND isnotnull(i_brand_id#7)) AND isnotnull(i_class_id#8)) AND isnotnull(i_category_id#9)) AND isnotnull(i_manufact_id#11)) -(276) Project +(277) Project Output [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Input [6]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_category#10, i_manufact_id#11] -(277) Exchange +(278) Exchange Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: hashpartitioning(i_item_sk#6, 100), ENSURE_REQUIREMENTS, [plan_id=26] -(278) Sort +(279) Sort Input [5]: [i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: [i_item_sk#6 ASC NULLS FIRST], false, 0 -(279) SortMergeJoin +(280) SortMergeJoin Left keys [1]: [cs_item_sk#2] Right keys [1]: [i_item_sk#6] Join type: Inner Join condition: None -(280) Project +(281) Project Output [9]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Input [10]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_item_sk#6, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] -(281) Exchange +(282) Exchange Input [9]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: hashpartitioning(cs_sold_date_sk#1, 100), ENSURE_REQUIREMENTS, [plan_id=27] -(282) Sort +(283) Sort Input [9]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Arguments: [cs_sold_date_sk#1 ASC NULLS FIRST], false, 0 -(283) Scan parquet +(284) Scan parquet Output [2]: [d_date_sk#12, d_year#13] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(284) Filter +(285) Filter Input [2]: [d_date_sk#12, d_year#13] Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2002)) AND isnotnull(d_date_sk#12)) -(285) Exchange +(286) Exchange Input [2]: [d_date_sk#12, d_year#13] Arguments: hashpartitioning(d_date_sk#12, 100), ENSURE_REQUIREMENTS, [plan_id=28] -(286) Sort +(287) Sort Input [2]: [d_date_sk#12, d_year#13] Arguments: [d_date_sk#12 ASC NULLS FIRST], false, 0 -(287) SortMergeJoin +(288) SortMergeJoin Left keys [1]: [cs_sold_date_sk#1] Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None -(288) Project +(289) Project Output [9]: [cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Input [11]: [cs_sold_date_sk#1, cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_date_sk#12, d_year#13] -(289) Exchange +(290) Exchange Input [9]: [cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: hashpartitioning(cs_order_number#3, cs_item_sk#2, 100), ENSURE_REQUIREMENTS, [plan_id=29] -(290) Sort +(291) Sort Input [9]: [cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13] Arguments: [cs_order_number#3 ASC NULLS FIRST, cs_item_sk#2 ASC NULLS FIRST], false, 0 -(291) Scan parquet +(292) Scan parquet Output [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(292) Filter +(293) Filter Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Condition : (isnotnull(cr_order_number#15) AND isnotnull(cr_item_sk#14)) -(293) Exchange +(294) Exchange Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Arguments: hashpartitioning(cr_order_number#15, cr_item_sk#14, 100), ENSURE_REQUIREMENTS, [plan_id=30] -(294) Sort +(295) Sort Input [4]: [cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] Arguments: [cr_order_number#15 ASC NULLS FIRST, cr_item_sk#14 ASC NULLS FIRST], false, 0 -(295) SortMergeJoin +(296) SortMergeJoin Left keys [2]: [cs_order_number#3, cs_item_sk#2] Right keys [2]: [cr_order_number#15, cr_item_sk#14] Join type: LeftOuter Join condition: None -(296) Project +(297) Project Output [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, (cs_quantity#4 - coalesce(cr_return_quantity#16, 0)) AS sales_cnt#18, (cs_ext_sales_price#5 - coalesce(cr_return_amount#17, 0.00)) AS sales_amt#19] Input [13]: [cs_item_sk#2, cs_order_number#3, cs_quantity#4, cs_ext_sales_price#5, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, d_year#13, cr_item_sk#14, cr_order_number#15, cr_return_quantity#16, cr_return_amount#17] -(297) Scan parquet +(298) Scan parquet Output [5]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(298) Filter +(299) Filter Input [5]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24] Condition : (isnotnull(ss_item_sk#21) AND isnotnull(ss_sold_date_sk#20)) -(299) Exchange +(300) Exchange Input [5]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24] Arguments: hashpartitioning(ss_item_sk#21, 100), ENSURE_REQUIREMENTS, [plan_id=31] -(300) Sort +(301) Sort Input [5]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24] Arguments: [ss_item_sk#21 ASC NULLS FIRST], false, 0 -(301) Scan parquet +(302) Scan parquet Output [6]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28, i_category#121, i_manufact_id#29] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(302) Filter +(303) Filter Input [6]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28, i_category#121, i_manufact_id#29] Condition : ((((((isnotnull(i_category#121) AND (i_category#121 = Books)) AND isnotnull(i_item_sk#25)) AND isnotnull(i_brand_id#26)) AND isnotnull(i_class_id#27)) AND isnotnull(i_category_id#28)) AND isnotnull(i_manufact_id#29)) -(303) Project +(304) Project Output [5]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29] Input [6]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28, i_category#121, i_manufact_id#29] -(304) Exchange +(305) Exchange Input [5]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29] Arguments: hashpartitioning(i_item_sk#25, 100), ENSURE_REQUIREMENTS, [plan_id=32] -(305) Sort +(306) Sort Input [5]: [i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29] Arguments: [i_item_sk#25 ASC NULLS FIRST], false, 0 -(306) SortMergeJoin +(307) SortMergeJoin Left keys [1]: [ss_item_sk#21] Right keys [1]: [i_item_sk#25] Join type: Inner Join condition: None -(307) Project +(308) Project Output [9]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29] Input [10]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_item_sk#25, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29] -(308) Exchange +(309) Exchange Input [9]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29] Arguments: hashpartitioning(ss_sold_date_sk#20, 100), ENSURE_REQUIREMENTS, [plan_id=33] -(309) Sort +(310) Sort Input [9]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29] Arguments: [ss_sold_date_sk#20 ASC NULLS FIRST], false, 0 -(310) Scan parquet +(311) Scan parquet Output [2]: [d_date_sk#30, d_year#31] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(311) Filter +(312) Filter Input [2]: [d_date_sk#30, d_year#31] Condition : ((isnotnull(d_year#31) AND (d_year#31 = 2002)) AND isnotnull(d_date_sk#30)) -(312) Exchange +(313) Exchange Input [2]: [d_date_sk#30, d_year#31] Arguments: hashpartitioning(d_date_sk#30, 100), ENSURE_REQUIREMENTS, [plan_id=34] -(313) Sort +(314) Sort Input [2]: [d_date_sk#30, d_year#31] Arguments: [d_date_sk#30 ASC NULLS FIRST], false, 0 -(314) SortMergeJoin +(315) SortMergeJoin Left keys [1]: [ss_sold_date_sk#20] Right keys [1]: [d_date_sk#30] Join type: Inner Join condition: None -(315) Project +(316) Project Output [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, d_year#31] Input [11]: [ss_sold_date_sk#20, ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, d_date_sk#30, d_year#31] -(316) Exchange +(317) Exchange Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, d_year#31] Arguments: hashpartitioning(ss_ticket_number#22, ss_item_sk#21, 100), ENSURE_REQUIREMENTS, [plan_id=35] -(317) Sort +(318) Sort Input [9]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, d_year#31] Arguments: [ss_ticket_number#22 ASC NULLS FIRST, ss_item_sk#21 ASC NULLS FIRST], false, 0 -(318) Scan parquet +(319) Scan parquet Output [4]: [sr_item_sk#32, sr_ticket_number#33, sr_return_quantity#34, sr_return_amt#35] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(319) Filter +(320) Filter Input [4]: [sr_item_sk#32, sr_ticket_number#33, sr_return_quantity#34, sr_return_amt#35] Condition : (isnotnull(sr_ticket_number#33) AND isnotnull(sr_item_sk#32)) -(320) Exchange +(321) Exchange Input [4]: [sr_item_sk#32, sr_ticket_number#33, sr_return_quantity#34, sr_return_amt#35] Arguments: hashpartitioning(sr_ticket_number#33, sr_item_sk#32, 100), ENSURE_REQUIREMENTS, [plan_id=36] -(321) Sort +(322) Sort Input [4]: [sr_item_sk#32, sr_ticket_number#33, sr_return_quantity#34, sr_return_amt#35] Arguments: [sr_ticket_number#33 ASC NULLS FIRST, sr_item_sk#32 ASC NULLS FIRST], false, 0 -(322) SortMergeJoin +(323) SortMergeJoin Left keys [2]: [ss_ticket_number#22, ss_item_sk#21] Right keys [2]: [sr_ticket_number#33, sr_item_sk#32] Join type: LeftOuter Join condition: None -(323) Project +(324) Project Output [7]: [d_year#31, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, (ss_quantity#23 - coalesce(sr_return_quantity#34, 0)) AS sales_cnt#36, (ss_ext_sales_price#24 - coalesce(sr_return_amt#35, 0.00)) AS sales_amt#37] Input [13]: [ss_item_sk#21, ss_ticket_number#22, ss_quantity#23, ss_ext_sales_price#24, i_brand_id#26, i_class_id#27, i_category_id#28, i_manufact_id#29, d_year#31, sr_item_sk#32, sr_ticket_number#33, sr_return_quantity#34, sr_return_amt#35] -(324) Scan parquet +(325) Scan parquet Output [5]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(325) Filter +(326) Filter Input [5]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42] Condition : (isnotnull(ws_item_sk#39) AND isnotnull(ws_sold_date_sk#38)) -(326) Exchange +(327) Exchange Input [5]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42] Arguments: hashpartitioning(ws_item_sk#39, 100), ENSURE_REQUIREMENTS, [plan_id=37] -(327) Sort +(328) Sort Input [5]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42] Arguments: [ws_item_sk#39 ASC NULLS FIRST], false, 0 -(328) Scan parquet +(329) Scan parquet Output [6]: [i_item_sk#43, i_brand_id#44, i_class_id#45, i_category_id#46, i_category#122, i_manufact_id#47] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(329) Filter +(330) Filter Input [6]: [i_item_sk#43, i_brand_id#44, i_class_id#45, i_category_id#46, i_category#122, i_manufact_id#47] Condition : ((((((isnotnull(i_category#122) AND (i_category#122 = Books)) AND isnotnull(i_item_sk#43)) AND isnotnull(i_brand_id#44)) AND isnotnull(i_class_id#45)) AND isnotnull(i_category_id#46)) AND isnotnull(i_manufact_id#47)) -(330) Project +(331) Project Output [5]: [i_item_sk#43, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47] Input [6]: [i_item_sk#43, i_brand_id#44, i_class_id#45, i_category_id#46, i_category#122, i_manufact_id#47] -(331) Exchange +(332) Exchange Input [5]: [i_item_sk#43, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47] Arguments: hashpartitioning(i_item_sk#43, 100), ENSURE_REQUIREMENTS, [plan_id=38] -(332) Sort +(333) Sort Input [5]: [i_item_sk#43, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47] Arguments: [i_item_sk#43 ASC NULLS FIRST], false, 0 -(333) SortMergeJoin +(334) SortMergeJoin Left keys [1]: [ws_item_sk#39] Right keys [1]: [i_item_sk#43] Join type: Inner Join condition: None -(334) Project +(335) Project Output [9]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47] Input [10]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_item_sk#43, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47] -(335) Exchange +(336) Exchange Input [9]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47] Arguments: hashpartitioning(ws_sold_date_sk#38, 100), ENSURE_REQUIREMENTS, [plan_id=39] -(336) Sort +(337) Sort Input [9]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47] Arguments: [ws_sold_date_sk#38 ASC NULLS FIRST], false, 0 -(337) Scan parquet +(338) Scan parquet Output [2]: [d_date_sk#48, d_year#49] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(338) Filter +(339) Filter Input [2]: [d_date_sk#48, d_year#49] Condition : ((isnotnull(d_year#49) AND (d_year#49 = 2002)) AND isnotnull(d_date_sk#48)) -(339) Exchange +(340) Exchange Input [2]: [d_date_sk#48, d_year#49] Arguments: hashpartitioning(d_date_sk#48, 100), ENSURE_REQUIREMENTS, [plan_id=40] -(340) Sort +(341) Sort Input [2]: [d_date_sk#48, d_year#49] Arguments: [d_date_sk#48 ASC NULLS FIRST], false, 0 -(341) SortMergeJoin +(342) SortMergeJoin Left keys [1]: [ws_sold_date_sk#38] Right keys [1]: [d_date_sk#48] Join type: Inner Join condition: None -(342) Project +(343) Project Output [9]: [ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47, d_year#49] Input [11]: [ws_sold_date_sk#38, ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47, d_date_sk#48, d_year#49] -(343) Exchange +(344) Exchange Input [9]: [ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47, d_year#49] Arguments: hashpartitioning(ws_order_number#40, ws_item_sk#39, 100), ENSURE_REQUIREMENTS, [plan_id=41] -(344) Sort +(345) Sort Input [9]: [ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47, d_year#49] Arguments: [ws_order_number#40 ASC NULLS FIRST, ws_item_sk#39 ASC NULLS FIRST], false, 0 -(345) Scan parquet +(346) Scan parquet Output [4]: [wr_item_sk#50, wr_order_number#51, wr_return_quantity#52, wr_return_amt#53] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(346) Filter +(347) Filter Input [4]: [wr_item_sk#50, wr_order_number#51, wr_return_quantity#52, wr_return_amt#53] Condition : (isnotnull(wr_order_number#51) AND isnotnull(wr_item_sk#50)) -(347) Exchange +(348) Exchange Input [4]: [wr_item_sk#50, wr_order_number#51, wr_return_quantity#52, wr_return_amt#53] Arguments: hashpartitioning(wr_order_number#51, wr_item_sk#50, 100), ENSURE_REQUIREMENTS, [plan_id=42] -(348) Sort +(349) Sort Input [4]: [wr_item_sk#50, wr_order_number#51, wr_return_quantity#52, wr_return_amt#53] Arguments: [wr_order_number#51 ASC NULLS FIRST, wr_item_sk#50 ASC NULLS FIRST], false, 0 -(349) SortMergeJoin +(350) SortMergeJoin Left keys [2]: [ws_order_number#40, ws_item_sk#39] Right keys [2]: [wr_order_number#51, wr_item_sk#50] Join type: LeftOuter Join condition: None -(350) Project +(351) Project Output [7]: [d_year#49, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47, (ws_quantity#41 - coalesce(wr_return_quantity#52, 0)) AS sales_cnt#54, (ws_ext_sales_price#42 - coalesce(wr_return_amt#53, 0.00)) AS sales_amt#55] Input [13]: [ws_item_sk#39, ws_order_number#40, ws_quantity#41, ws_ext_sales_price#42, i_brand_id#44, i_class_id#45, i_category_id#46, i_manufact_id#47, d_year#49, wr_item_sk#50, wr_order_number#51, wr_return_quantity#52, wr_return_amt#53] -(351) Union +(352) Union -(352) HashAggregate +(353) HashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19] -(353) Exchange +(354) Exchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19] Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19, 100), ENSURE_REQUIREMENTS, [plan_id=43] -(354) HashAggregate +(355) HashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19] Keys [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19] -(355) HashAggregate +(356) HashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#18, sales_amt#19] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [partial_sum(sales_cnt#18), partial_sum(UnscaledValue(sales_amt#19))] Aggregate Attributes [2]: [sum#58, sum#59] Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#123, sum#124] -(356) Exchange +(357) Exchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#123, sum#124] Arguments: hashpartitioning(d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 100), ENSURE_REQUIREMENTS, [plan_id=44] -(357) HashAggregate +(358) HashAggregate Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum#123, sum#124] Keys [5]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Functions [2]: [sum(sales_cnt#18), sum(UnscaledValue(sales_amt#19))] Aggregate Attributes [2]: [sum(sales_cnt#18)#61, sum(UnscaledValue(sales_amt#19))#62] Results [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sum(sales_cnt#18)#61 AS sales_cnt#63, MakeDecimal(sum(UnscaledValue(sales_amt#19))#62,18,2) AS sales_amt#64] -(358) Filter +(359) Filter Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] Condition : isnotnull(sales_cnt#63) -(359) Exchange +(360) Exchange Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] Arguments: hashpartitioning(i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, 100), ENSURE_REQUIREMENTS, [plan_id=45] -(360) Sort +(361) Sort Input [7]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64] Arguments: [i_brand_id#7 ASC NULLS FIRST, i_class_id#8 ASC NULLS FIRST, i_category_id#9 ASC NULLS FIRST, i_manufact_id#11 ASC NULLS FIRST], false, 0 -(361) Scan parquet +(362) Scan parquet Output [5]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_sold_date_sk)] ReadSchema: struct -(362) Filter +(363) Filter Input [5]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69] Condition : (isnotnull(cs_item_sk#66) AND isnotnull(cs_sold_date_sk#65)) -(363) Exchange +(364) Exchange Input [5]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69] Arguments: hashpartitioning(cs_item_sk#66, 100), ENSURE_REQUIREMENTS, [plan_id=46] -(364) Sort +(365) Sort Input [5]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69] Arguments: [cs_item_sk#66 ASC NULLS FIRST], false, 0 -(365) Scan parquet +(366) Scan parquet Output [6]: [i_item_sk#125, i_brand_id#70, i_class_id#71, i_category_id#72, i_category#126, i_manufact_id#73] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(366) Filter +(367) Filter Input [6]: [i_item_sk#125, i_brand_id#70, i_class_id#71, i_category_id#72, i_category#126, i_manufact_id#73] Condition : ((((((isnotnull(i_category#126) AND (i_category#126 = Books)) AND isnotnull(i_item_sk#125)) AND isnotnull(i_brand_id#70)) AND isnotnull(i_class_id#71)) AND isnotnull(i_category_id#72)) AND isnotnull(i_manufact_id#73)) -(367) Project +(368) Project Output [5]: [i_item_sk#125, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Input [6]: [i_item_sk#125, i_brand_id#70, i_class_id#71, i_category_id#72, i_category#126, i_manufact_id#73] -(368) Exchange +(369) Exchange Input [5]: [i_item_sk#125, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Arguments: hashpartitioning(i_item_sk#125, 100), ENSURE_REQUIREMENTS, [plan_id=47] -(369) Sort +(370) Sort Input [5]: [i_item_sk#125, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Arguments: [i_item_sk#125 ASC NULLS FIRST], false, 0 -(370) SortMergeJoin +(371) SortMergeJoin Left keys [1]: [cs_item_sk#66] Right keys [1]: [i_item_sk#125] Join type: Inner Join condition: None -(371) Project +(372) Project Output [9]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Input [10]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_item_sk#125, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] -(372) Exchange +(373) Exchange Input [9]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Arguments: hashpartitioning(cs_sold_date_sk#65, 100), ENSURE_REQUIREMENTS, [plan_id=48] -(373) Sort +(374) Sort Input [9]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Arguments: [cs_sold_date_sk#65 ASC NULLS FIRST], false, 0 -(374) Scan parquet +(375) Scan parquet Output [2]: [d_date_sk#74, d_year#75] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(375) Filter +(376) Filter Input [2]: [d_date_sk#74, d_year#75] Condition : ((isnotnull(d_year#75) AND (d_year#75 = 2001)) AND isnotnull(d_date_sk#74)) -(376) Exchange +(377) Exchange Input [2]: [d_date_sk#74, d_year#75] Arguments: hashpartitioning(d_date_sk#74, 100), ENSURE_REQUIREMENTS, [plan_id=49] -(377) Sort +(378) Sort Input [2]: [d_date_sk#74, d_year#75] Arguments: [d_date_sk#74 ASC NULLS FIRST], false, 0 -(378) SortMergeJoin +(379) SortMergeJoin Left keys [1]: [cs_sold_date_sk#65] Right keys [1]: [d_date_sk#74] Join type: Inner Join condition: None -(379) Project +(380) Project Output [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, d_year#75] Input [11]: [cs_sold_date_sk#65, cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, d_date_sk#74, d_year#75] -(380) Exchange +(381) Exchange Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, d_year#75] Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 100), ENSURE_REQUIREMENTS, [plan_id=50] -(381) Sort +(382) Sort Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, d_year#75] Arguments: [cs_order_number#67 ASC NULLS FIRST, cs_item_sk#66 ASC NULLS FIRST], false, 0 -(382) Scan parquet +(383) Scan parquet Output [4]: [cr_item_sk#76, cr_order_number#77, cr_return_quantity#78, cr_return_amount#79] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(383) Filter +(384) Filter Input [4]: [cr_item_sk#76, cr_order_number#77, cr_return_quantity#78, cr_return_amount#79] Condition : (isnotnull(cr_order_number#77) AND isnotnull(cr_item_sk#76)) -(384) Exchange +(385) Exchange Input [4]: [cr_item_sk#76, cr_order_number#77, cr_return_quantity#78, cr_return_amount#79] Arguments: hashpartitioning(cr_order_number#77, cr_item_sk#76, 100), ENSURE_REQUIREMENTS, [plan_id=51] -(385) Sort +(386) Sort Input [4]: [cr_item_sk#76, cr_order_number#77, cr_return_quantity#78, cr_return_amount#79] Arguments: [cr_order_number#77 ASC NULLS FIRST, cr_item_sk#76 ASC NULLS FIRST], false, 0 -(386) SortMergeJoin +(387) SortMergeJoin Left keys [2]: [cs_order_number#67, cs_item_sk#66] Right keys [2]: [cr_order_number#77, cr_item_sk#76] Join type: LeftOuter Join condition: None -(387) Project +(388) Project Output [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, (cs_quantity#68 - coalesce(cr_return_quantity#78, 0)) AS sales_cnt#18, (cs_ext_sales_price#69 - coalesce(cr_return_amount#79, 0.00)) AS sales_amt#19] Input [13]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, d_year#75, cr_item_sk#76, cr_order_number#77, cr_return_quantity#78, cr_return_amount#79] -(388) Scan parquet +(389) Scan parquet Output [5]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_sold_date_sk)] ReadSchema: struct -(389) Filter +(390) Filter Input [5]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84] Condition : (isnotnull(ss_item_sk#81) AND isnotnull(ss_sold_date_sk#80)) -(390) Exchange +(391) Exchange Input [5]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84] Arguments: hashpartitioning(ss_item_sk#81, 100), ENSURE_REQUIREMENTS, [plan_id=52] -(391) Sort +(392) Sort Input [5]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84] Arguments: [ss_item_sk#81 ASC NULLS FIRST], false, 0 -(392) Scan parquet +(393) Scan parquet Output [6]: [i_item_sk#127, i_brand_id#85, i_class_id#86, i_category_id#87, i_category#128, i_manufact_id#88] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(393) Filter +(394) Filter Input [6]: [i_item_sk#127, i_brand_id#85, i_class_id#86, i_category_id#87, i_category#128, i_manufact_id#88] Condition : ((((((isnotnull(i_category#128) AND (i_category#128 = Books)) AND isnotnull(i_item_sk#127)) AND isnotnull(i_brand_id#85)) AND isnotnull(i_class_id#86)) AND isnotnull(i_category_id#87)) AND isnotnull(i_manufact_id#88)) -(394) Project +(395) Project Output [5]: [i_item_sk#127, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] Input [6]: [i_item_sk#127, i_brand_id#85, i_class_id#86, i_category_id#87, i_category#128, i_manufact_id#88] -(395) Exchange +(396) Exchange Input [5]: [i_item_sk#127, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] Arguments: hashpartitioning(i_item_sk#127, 100), ENSURE_REQUIREMENTS, [plan_id=53] -(396) Sort +(397) Sort Input [5]: [i_item_sk#127, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] Arguments: [i_item_sk#127 ASC NULLS FIRST], false, 0 -(397) SortMergeJoin +(398) SortMergeJoin Left keys [1]: [ss_item_sk#81] Right keys [1]: [i_item_sk#127] Join type: Inner Join condition: None -(398) Project +(399) Project Output [9]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] Input [10]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_item_sk#127, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] -(399) Exchange +(400) Exchange Input [9]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] Arguments: hashpartitioning(ss_sold_date_sk#80, 100), ENSURE_REQUIREMENTS, [plan_id=54] -(400) Sort +(401) Sort Input [9]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88] Arguments: [ss_sold_date_sk#80 ASC NULLS FIRST], false, 0 -(401) Scan parquet +(402) Scan parquet Output [2]: [d_date_sk#89, d_year#90] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(402) Filter +(403) Filter Input [2]: [d_date_sk#89, d_year#90] Condition : ((isnotnull(d_year#90) AND (d_year#90 = 2001)) AND isnotnull(d_date_sk#89)) -(403) Exchange +(404) Exchange Input [2]: [d_date_sk#89, d_year#90] Arguments: hashpartitioning(d_date_sk#89, 100), ENSURE_REQUIREMENTS, [plan_id=55] -(404) Sort +(405) Sort Input [2]: [d_date_sk#89, d_year#90] Arguments: [d_date_sk#89 ASC NULLS FIRST], false, 0 -(405) SortMergeJoin +(406) SortMergeJoin Left keys [1]: [ss_sold_date_sk#80] Right keys [1]: [d_date_sk#89] Join type: Inner Join condition: None -(406) Project +(407) Project Output [9]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] Input [11]: [ss_sold_date_sk#80, ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_date_sk#89, d_year#90] -(407) Exchange +(408) Exchange Input [9]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] Arguments: hashpartitioning(ss_ticket_number#82, ss_item_sk#81, 100), ENSURE_REQUIREMENTS, [plan_id=56] -(408) Sort +(409) Sort Input [9]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90] Arguments: [ss_ticket_number#82 ASC NULLS FIRST, ss_item_sk#81 ASC NULLS FIRST], false, 0 -(409) Scan parquet +(410) Scan parquet Output [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(410) Filter +(411) Filter Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] Condition : (isnotnull(sr_ticket_number#92) AND isnotnull(sr_item_sk#91)) -(411) Exchange +(412) Exchange Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] Arguments: hashpartitioning(sr_ticket_number#92, sr_item_sk#91, 100), ENSURE_REQUIREMENTS, [plan_id=57] -(412) Sort +(413) Sort Input [4]: [sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] Arguments: [sr_ticket_number#92 ASC NULLS FIRST, sr_item_sk#91 ASC NULLS FIRST], false, 0 -(413) SortMergeJoin +(414) SortMergeJoin Left keys [2]: [ss_ticket_number#82, ss_item_sk#81] Right keys [2]: [sr_ticket_number#92, sr_item_sk#91] Join type: LeftOuter Join condition: None -(414) Project +(415) Project Output [7]: [d_year#90, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, (ss_quantity#83 - coalesce(sr_return_quantity#93, 0)) AS sales_cnt#36, (ss_ext_sales_price#84 - coalesce(sr_return_amt#94, 0.00)) AS sales_amt#37] Input [13]: [ss_item_sk#81, ss_ticket_number#82, ss_quantity#83, ss_ext_sales_price#84, i_brand_id#85, i_class_id#86, i_category_id#87, i_manufact_id#88, d_year#90, sr_item_sk#91, sr_ticket_number#92, sr_return_quantity#93, sr_return_amt#94] -(415) Scan parquet +(416) Scan parquet Output [5]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(416) Filter +(417) Filter Input [5]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99] Condition : (isnotnull(ws_item_sk#96) AND isnotnull(ws_sold_date_sk#95)) -(417) Exchange +(418) Exchange Input [5]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99] Arguments: hashpartitioning(ws_item_sk#96, 100), ENSURE_REQUIREMENTS, [plan_id=58] -(418) Sort +(419) Sort Input [5]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99] Arguments: [ws_item_sk#96 ASC NULLS FIRST], false, 0 -(419) Scan parquet +(420) Scan parquet Output [6]: [i_item_sk#129, i_brand_id#100, i_class_id#101, i_category_id#102, i_category#130, i_manufact_id#103] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(i_category), EqualTo(i_category,Books), IsNotNull(i_item_sk), IsNotNull(i_brand_id), IsNotNull(i_class_id), IsNotNull(i_category_id), IsNotNull(i_manufact_id)] ReadSchema: struct -(420) Filter +(421) Filter Input [6]: [i_item_sk#129, i_brand_id#100, i_class_id#101, i_category_id#102, i_category#130, i_manufact_id#103] Condition : ((((((isnotnull(i_category#130) AND (i_category#130 = Books)) AND isnotnull(i_item_sk#129)) AND isnotnull(i_brand_id#100)) AND isnotnull(i_class_id#101)) AND isnotnull(i_category_id#102)) AND isnotnull(i_manufact_id#103)) -(421) Project +(422) Project Output [5]: [i_item_sk#129, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103] Input [6]: [i_item_sk#129, i_brand_id#100, i_class_id#101, i_category_id#102, i_category#130, i_manufact_id#103] -(422) Exchange +(423) Exchange Input [5]: [i_item_sk#129, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103] Arguments: hashpartitioning(i_item_sk#129, 100), ENSURE_REQUIREMENTS, [plan_id=59] -(423) Sort +(424) Sort Input [5]: [i_item_sk#129, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103] Arguments: [i_item_sk#129 ASC NULLS FIRST], false, 0 -(424) SortMergeJoin +(425) SortMergeJoin Left keys [1]: [ws_item_sk#96] Right keys [1]: [i_item_sk#129] Join type: Inner Join condition: None -(425) Project +(426) Project Output [9]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103] Input [10]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_item_sk#129, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103] -(426) Exchange +(427) Exchange Input [9]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103] Arguments: hashpartitioning(ws_sold_date_sk#95, 100), ENSURE_REQUIREMENTS, [plan_id=60] -(427) Sort +(428) Sort Input [9]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103] Arguments: [ws_sold_date_sk#95 ASC NULLS FIRST], false, 0 -(428) Scan parquet +(429) Scan parquet Output [2]: [d_date_sk#104, d_year#105] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(429) Filter +(430) Filter Input [2]: [d_date_sk#104, d_year#105] Condition : ((isnotnull(d_year#105) AND (d_year#105 = 2001)) AND isnotnull(d_date_sk#104)) -(430) Exchange +(431) Exchange Input [2]: [d_date_sk#104, d_year#105] Arguments: hashpartitioning(d_date_sk#104, 100), ENSURE_REQUIREMENTS, [plan_id=61] -(431) Sort +(432) Sort Input [2]: [d_date_sk#104, d_year#105] Arguments: [d_date_sk#104 ASC NULLS FIRST], false, 0 -(432) SortMergeJoin +(433) SortMergeJoin Left keys [1]: [ws_sold_date_sk#95] Right keys [1]: [d_date_sk#104] Join type: Inner Join condition: None -(433) Project +(434) Project Output [9]: [ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103, d_year#105] Input [11]: [ws_sold_date_sk#95, ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103, d_date_sk#104, d_year#105] -(434) Exchange +(435) Exchange Input [9]: [ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103, d_year#105] Arguments: hashpartitioning(ws_order_number#97, ws_item_sk#96, 100), ENSURE_REQUIREMENTS, [plan_id=62] -(435) Sort +(436) Sort Input [9]: [ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103, d_year#105] Arguments: [ws_order_number#97 ASC NULLS FIRST, ws_item_sk#96 ASC NULLS FIRST], false, 0 -(436) Scan parquet +(437) Scan parquet Output [4]: [wr_item_sk#106, wr_order_number#107, wr_return_quantity#108, wr_return_amt#109] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(437) Filter +(438) Filter Input [4]: [wr_item_sk#106, wr_order_number#107, wr_return_quantity#108, wr_return_amt#109] Condition : (isnotnull(wr_order_number#107) AND isnotnull(wr_item_sk#106)) -(438) Exchange +(439) Exchange Input [4]: [wr_item_sk#106, wr_order_number#107, wr_return_quantity#108, wr_return_amt#109] Arguments: hashpartitioning(wr_order_number#107, wr_item_sk#106, 100), ENSURE_REQUIREMENTS, [plan_id=63] -(439) Sort +(440) Sort Input [4]: [wr_item_sk#106, wr_order_number#107, wr_return_quantity#108, wr_return_amt#109] Arguments: [wr_order_number#107 ASC NULLS FIRST, wr_item_sk#106 ASC NULLS FIRST], false, 0 -(440) SortMergeJoin +(441) SortMergeJoin Left keys [2]: [ws_order_number#97, ws_item_sk#96] Right keys [2]: [wr_order_number#107, wr_item_sk#106] Join type: LeftOuter Join condition: None -(441) Project +(442) Project Output [7]: [d_year#105, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103, (ws_quantity#98 - coalesce(wr_return_quantity#108, 0)) AS sales_cnt#54, (ws_ext_sales_price#99 - coalesce(wr_return_amt#109, 0.00)) AS sales_amt#55] Input [13]: [ws_item_sk#96, ws_order_number#97, ws_quantity#98, ws_ext_sales_price#99, i_brand_id#100, i_class_id#101, i_category_id#102, i_manufact_id#103, d_year#105, wr_item_sk#106, wr_order_number#107, wr_return_quantity#108, wr_return_amt#109] -(442) Union +(443) Union -(443) HashAggregate +(444) HashAggregate Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19] Keys [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19] -(444) Exchange +(445) Exchange Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19] Arguments: hashpartitioning(d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19, 100), ENSURE_REQUIREMENTS, [plan_id=64] -(445) HashAggregate +(446) HashAggregate Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19] Keys [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19] Functions: [] Aggregate Attributes: [] Results [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19] -(446) HashAggregate +(447) HashAggregate Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#18, sales_amt#19] Keys [5]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Functions [2]: [partial_sum(sales_cnt#18), partial_sum(UnscaledValue(sales_amt#19))] Aggregate Attributes [2]: [sum#58, sum#112] Results [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sum#123, sum#131] -(447) Exchange +(448) Exchange Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sum#123, sum#131] Arguments: hashpartitioning(d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, 100), ENSURE_REQUIREMENTS, [plan_id=65] -(448) HashAggregate +(449) HashAggregate Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sum#123, sum#131] Keys [5]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Functions [2]: [sum(sales_cnt#18), sum(UnscaledValue(sales_amt#19))] Aggregate Attributes [2]: [sum(sales_cnt#18)#61, sum(UnscaledValue(sales_amt#19))#62] Results [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sum(sales_cnt#18)#61 AS sales_cnt#113, MakeDecimal(sum(UnscaledValue(sales_amt#19))#62,18,2) AS sales_amt#114] -(449) Filter +(450) Filter Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#113, sales_amt#114] Condition : isnotnull(sales_cnt#113) -(450) Exchange +(451) Exchange Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#113, sales_amt#114] Arguments: hashpartitioning(i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, 100), ENSURE_REQUIREMENTS, [plan_id=66] -(451) Sort +(452) Sort Input [7]: [d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#113, sales_amt#114] Arguments: [i_brand_id#70 ASC NULLS FIRST, i_class_id#71 ASC NULLS FIRST, i_category_id#72 ASC NULLS FIRST, i_manufact_id#73 ASC NULLS FIRST], false, 0 -(452) SortMergeJoin +(453) SortMergeJoin Left keys [4]: [i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11] Right keys [4]: [i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73] Join type: Inner Join condition: ((cast(sales_cnt#63 as decimal(17,2)) / cast(sales_cnt#113 as decimal(17,2))) < 0.90000000000000000000) -(453) Project +(454) Project Output [10]: [d_year#75 AS prev_year#115, d_year#13 AS year#116, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#113 AS prev_yr_cnt#117, sales_cnt#63 AS curr_yr_cnt#118, (sales_cnt#63 - sales_cnt#113) AS sales_cnt_diff#119, (sales_amt#64 - sales_amt#114) AS sales_amt_diff#120] Input [14]: [d_year#13, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, sales_cnt#63, sales_amt#64, d_year#75, i_brand_id#70, i_class_id#71, i_category_id#72, i_manufact_id#73, sales_cnt#113, sales_amt#114] -(454) TakeOrderedAndProject +(455) TakeOrderedAndProject Input [10]: [prev_year#115, year#116, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#117, curr_yr_cnt#118, sales_cnt_diff#119, sales_amt_diff#120] Arguments: X, [sales_cnt_diff#119 ASC NULLS FIRST, sales_amt_diff#120 ASC NULLS FIRST], [prev_year#115, year#116, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#117, curr_yr_cnt#118, sales_cnt_diff#119, sales_amt_diff#120] -(455) AdaptiveSparkPlan +(456) AdaptiveSparkPlan Output [10]: [prev_year#115, year#116, i_brand_id#7, i_class_id#8, i_category_id#9, i_manufact_id#11, prev_yr_cnt#117, curr_yr_cnt#118, sales_cnt_diff#119, sales_amt_diff#120] Arguments: isFinalPlan=true diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q81.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q81.txt index 34ee7ca5a..9442fa765 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q81.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q81.txt @@ -16,9 +16,9 @@ AdaptiveSparkPlan (200) : : +- AQEShuffleRead (100) : : +- ShuffleQueryStage (99), Statistics(X) : : +- NativeShuffleExchange (98) - : : +- ConvertToNative (97) - : : +- * Project (96) - : : +- * SortMergeJoin Inner (95) + : : +- NativeProject (97) + : : +- NativeFilter (96) + : : +- NativeSortMergeJoin Inner (95) : : :- NativeSort (48) : : : +- InputAdapter (47) : : : +- AQEShuffleRead (46) @@ -603,18 +603,19 @@ Condition : isnotnull((avg(ctr_total_return) * 1.2)#31) Input [2]: [(avg(ctr_total_return) * 1.2)#31, ctr_state#25] Arguments: [ctr_state#25 ASC NULLS FIRST], false -(95) SortMergeJoin [codegen id : X] +(95) NativeSortMergeJoin Left keys [1]: [ctr_state#14] Right keys [1]: [ctr_state#25] Join type: Inner -Join condition: (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31) +Join condition: None -(96) Project [codegen id : X] -Output [2]: [ctr_customer_sk#13, ctr_total_return#15] +(96) NativeFilter Input [5]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#31, ctr_state#25] +Condition : (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#31) -(97) ConvertToNative -Input [2]: [ctr_customer_sk#13, ctr_total_return#15] +(97) NativeProject +Output [2]: [ctr_customer_sk#13, ctr_total_return#15] +Input [5]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#31, ctr_state#25] (98) NativeShuffleExchange Input [2]: [ctr_customer_sk#13, ctr_total_return#15] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q85.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q85.txt index 8deea7807..28c98f7db 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q85.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q85.txt @@ -24,9 +24,9 @@ AdaptiveSparkPlan (186) : : +- AQEShuffleRead (84) : : +- ShuffleQueryStage (83), Statistics(X) : : +- NativeShuffleExchange (82) - : : +- ConvertToNative (81) - : : +- * Project (80) - : : +- * SortMergeJoin Inner (79) + : : +- NativeProject (81) + : : +- NativeFilter (80) + : : +- NativeSortMergeJoin Inner (79) : : :- NativeSort (69) : : : +- InputAdapter (68) : : : +- AQEShuffleRead (67) @@ -39,9 +39,9 @@ AdaptiveSparkPlan (186) : : : : +- AQEShuffleRead (52) : : : : +- ShuffleQueryStage (51), Statistics(X) : : : : +- NativeShuffleExchange (50) - : : : : +- ConvertToNative (49) - : : : : +- * Project (48) - : : : : +- * SortMergeJoin Inner (47) + : : : : +- NativeProject (49) + : : : : +- NativeFilter (48) + : : : : +- NativeSortMergeJoin Inner (47) : : : : :- NativeSort (38) : : : : : +- InputAdapter (37) : : : : : +- AQEShuffleRead (36) @@ -383,18 +383,19 @@ Input [3]: [#17#17, #18#18, #19#19] Input [3]: [#17#17, #18#18, #19#19] Arguments: [cd_demo_sk#17 ASC NULLS FIRST], false -(47) SortMergeJoin [codegen id : X] +(47) NativeSortMergeJoin Left keys [1]: [wr_refunded_cdemo_sk#9] Right keys [1]: [cd_demo_sk#17] Join type: Inner -Join condition: ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) AND (ws_sales_price#6 >= 100.00)) AND (ws_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College)) AND (ws_sales_price#6 >= 50.00)) AND (ws_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)) AND (ws_sales_price#6 >= 150.00)) AND (ws_sales_price#6 <= 200.00))) +Join condition: None -(48) Project [codegen id : X] -Output [10]: [ws_sold_date_sk#1, ws_quantity#5, ws_net_profit#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#18, cd_education_status#19] +(48) NativeFilter Input [13]: [ws_sold_date_sk#1, ws_quantity#5, ws_sales_price#6, ws_net_profit#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, #17#17, #18#18, #19#19] +Condition : ((((((cd_marital_status#18 = M) AND (cd_education_status#19 = Advanced Degree)) AND (ws_sales_price#6 >= 100.00)) AND (ws_sales_price#6 <= 150.00)) OR ((((cd_marital_status#18 = S) AND (cd_education_status#19 = College)) AND (ws_sales_price#6 >= 50.00)) AND (ws_sales_price#6 <= 100.00))) OR ((((cd_marital_status#18 = W) AND (cd_education_status#19 = 2 yr Degree)) AND (ws_sales_price#6 >= 150.00)) AND (ws_sales_price#6 <= 200.00))) -(49) ConvertToNative -Input [10]: [ws_sold_date_sk#1, ws_quantity#5, ws_net_profit#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#18, cd_education_status#19] +(49) NativeProject +Output [10]: [ws_sold_date_sk#1, ws_quantity#5, ws_net_profit#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#18, cd_education_status#19] +Input [13]: [ws_sold_date_sk#1, ws_quantity#5, ws_sales_price#6, ws_net_profit#7, wr_refunded_cdemo_sk#9, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, #17#17, #18#18, #19#19] (50) NativeShuffleExchange Input [10]: [ws_sold_date_sk#1, ws_quantity#5, ws_net_profit#7, wr_refunded_addr_sk#10, wr_returning_cdemo_sk#11, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, cd_marital_status#18, cd_education_status#19] @@ -516,18 +517,19 @@ Input [2]: [ca_address_sk#23, ca_state#24] Input [2]: [ca_address_sk#23, ca_state#24] Arguments: [ca_address_sk#23 ASC NULLS FIRST], false -(79) SortMergeJoin [codegen id : X] +(79) NativeSortMergeJoin Left keys [1]: [wr_refunded_addr_sk#10] Right keys [1]: [ca_address_sk#23] Join type: Inner -Join condition: ((((ca_state#24 IN (IN,OH,NJ) AND (ws_net_profit#7 >= 100.00)) AND (ws_net_profit#7 <= 200.00)) OR ((ca_state#24 IN (WI,CT,KY) AND (ws_net_profit#7 >= 150.00)) AND (ws_net_profit#7 <= 300.00))) OR ((ca_state#24 IN (LA,IA,AR) AND (ws_net_profit#7 >= 50.00)) AND (ws_net_profit#7 <= 250.00))) +Join condition: None -(80) Project [codegen id : X] -Output [5]: [ws_sold_date_sk#1, ws_quantity#5, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +(80) NativeFilter Input [9]: [ws_sold_date_sk#1, ws_quantity#5, ws_net_profit#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#23, ca_state#24] +Condition : ((((ca_state#24 IN (IN,OH,NJ) AND (ws_net_profit#7 >= 100.00)) AND (ws_net_profit#7 <= 200.00)) OR ((ca_state#24 IN (WI,CT,KY) AND (ws_net_profit#7 >= 150.00)) AND (ws_net_profit#7 <= 300.00))) OR ((ca_state#24 IN (LA,IA,AR) AND (ws_net_profit#7 >= 50.00)) AND (ws_net_profit#7 <= 250.00))) -(81) ConvertToNative -Input [5]: [ws_sold_date_sk#1, ws_quantity#5, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +(81) NativeProject +Output [5]: [ws_sold_date_sk#1, ws_quantity#5, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] +Input [9]: [ws_sold_date_sk#1, ws_quantity#5, ws_net_profit#7, wr_refunded_addr_sk#10, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15, ca_address_sk#23, ca_state#24] (82) NativeShuffleExchange Input [5]: [ws_sold_date_sk#1, ws_quantity#5, wr_reason_sk#12, wr_fee#14, wr_refunded_cash#15] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q92.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q92.txt index 2e0bf8dd6..32d0df2b4 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q92.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q92.txt @@ -14,9 +14,9 @@ AdaptiveSparkPlan (112) : +- AQEShuffleRead (54) : +- ShuffleQueryStage (53), Statistics(X) : +- NativeShuffleExchange (52) - : +- ConvertToNative (51) - : +- * Project (50) - : +- * SortMergeJoin Inner (49) + : +- NativeProject (51) + : +- NativeFilter (50) + : +- NativeSortMergeJoin Inner (49) : :- NativeProject (19) : : +- NativeSortMergeJoin Inner (18) : : :- NativeSort (8) @@ -324,18 +324,19 @@ Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#17) Input [2]: [(1.3 * avg(ws_ext_discount_amt))#17, ws_item_sk#7] Arguments: [ws_item_sk#7 ASC NULLS FIRST], false -(49) SortMergeJoin [codegen id : X] +(49) NativeSortMergeJoin Left keys [1]: [i_item_sk#4] Right keys [1]: [ws_item_sk#7] Join type: Inner -Join condition: (cast(ws_ext_discount_amt#3 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#17) +Join condition: None -(50) Project [codegen id : X] -Output [2]: [ws_sold_date_sk#1, ws_ext_discount_amt#3] +(50) NativeFilter Input [5]: [ws_sold_date_sk#1, ws_ext_discount_amt#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#17, ws_item_sk#7] +Condition : (cast(ws_ext_discount_amt#3 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#17) -(51) ConvertToNative -Input [2]: [ws_sold_date_sk#1, ws_ext_discount_amt#3] +(51) NativeProject +Output [2]: [ws_sold_date_sk#1, ws_ext_discount_amt#3] +Input [5]: [ws_sold_date_sk#1, ws_ext_discount_amt#3, i_item_sk#4, (1.3 * avg(ws_ext_discount_amt))#17, ws_item_sk#7] (52) NativeShuffleExchange Input [2]: [ws_sold_date_sk#1, ws_ext_discount_amt#3] diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q95.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q95.txt index cab7d1c24..10b5e2013 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q95.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q95.txt @@ -44,9 +44,9 @@ AdaptiveSparkPlan (178) : : : : : +- NativeFilter (3) : : : : : +- InputAdapter (2) : : : : : +- NativeParquetScan (1) - : : : : +- ConvertToNative (25) - : : : : +- * Project (24) - : : : : +- * SortMergeJoin Inner (23) + : : : : +- NativeProject (25) + : : : : +- NativeFilter (24) + : : : : +- NativeSortMergeJoin Inner (23) : : : : :- NativeSort (16) : : : : : +- InputAdapter (15) : : : : : +- AQEShuffleRead (14) @@ -71,9 +71,9 @@ AdaptiveSparkPlan (178) : : : : +- NativeFilter (29) : : : : +- InputAdapter (28) : : : : +- NativeParquetScan (27) - : : : +- ConvertToNative (49) - : : : +- * Project (48) - : : : +- * SortMergeJoin Inner (47) + : : : +- NativeProject (49) + : : : +- NativeFilter (48) + : : : +- NativeSortMergeJoin Inner (47) : : : :- NativeSort (40) : : : : +- InputAdapter (39) : : : : +- InputAdapter (38) @@ -271,18 +271,19 @@ Input [2]: [#9#9, #10#10] Input [2]: [#9#9, #10#10] Arguments: [ws_order_number#10 ASC NULLS FIRST], false -(23) SortMergeJoin [codegen id : X] +(23) NativeSortMergeJoin Left keys [1]: [ws_order_number#8] Right keys [1]: [ws_order_number#10] Join type: Inner -Join condition: NOT (ws_warehouse_sk#7 = ws_warehouse_sk#9) +Join condition: None -(24) Project [codegen id : X] -Output [1]: [ws_order_number#8] +(24) NativeFilter Input [4]: [#7#7, #8#8, #9#9, #10#10] +Condition : NOT (ws_warehouse_sk#7 = ws_warehouse_sk#9) -(25) ConvertToNative -Input [1]: [ws_order_number#8] +(25) NativeProject +Output [1]: [ws_order_number#8] +Input [4]: [#7#7, #8#8, #9#9, #10#10] (26) NativeSortMergeJoin Left keys [1]: [ws_order_number#4] @@ -368,18 +369,19 @@ Input [2]: [#14#14, #15#15] Input [2]: [#14#14, #15#15] Arguments: [ws_order_number#15 ASC NULLS FIRST], false -(47) SortMergeJoin [codegen id : X] +(47) NativeSortMergeJoin Left keys [1]: [ws_order_number#13] Right keys [1]: [ws_order_number#15] Join type: Inner -Join condition: NOT (ws_warehouse_sk#12 = ws_warehouse_sk#14) +Join condition: None -(48) Project [codegen id : X] -Output [1]: [ws_order_number#13] +(48) NativeFilter Input [4]: [#12#12, #13#13, #14#14, #15#15] +Condition : NOT (ws_warehouse_sk#12 = ws_warehouse_sk#14) -(49) ConvertToNative -Input [1]: [ws_order_number#13] +(49) NativeProject +Output [1]: [ws_order_number#13] +Input [4]: [#12#12, #13#13, #14#14, #15#15] (50) NativeSortMergeJoin Left keys [1]: [wr_order_number#11] diff --git a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala index 40eecda4e..0369c0491 100644 --- a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala +++ b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala @@ -17,6 +17,8 @@ package org.apache.auron import org.apache.spark.sql.{AuronQueryTest, Row} +import org.apache.spark.sql.execution.auron.plan.NativeShuffledHashJoinBase +import org.apache.spark.sql.execution.auron.plan.NativeSortMergeJoinBase import org.apache.spark.sql.execution.joins.auron.plan.NativeBroadcastJoinExec import org.apache.auron.spark.configuration.SparkAuronConfiguration @@ -612,6 +614,91 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ } } + test("native sort merge join supports inner residual condition") { + withSparkConf("spark.auron.forceShuffledHashJoin" -> "false") { + withSQLConf( + "spark.sql.adaptive.enabled" -> "false", + "spark.sql.autoBroadcastJoinThreshold" -> "-1", + "spark.sql.join.preferSortMergeJoin" -> "true") { + withTable("smj_left", "smj_right") { + sql(""" + |CREATE TABLE smj_left USING parquet AS + |SELECT * FROM VALUES + | (1, 1), + | (2, 5), + | (3, 7), + | (4, 9) + |AS t(id, lv) + |""".stripMargin) + + sql(""" + |CREATE TABLE smj_right USING parquet AS + |SELECT * FROM VALUES + | (1, 2), + | (2, 4), + | (3, 8), + | (4, 9) + |AS t(id, rv) + |""".stripMargin) + + val df = checkSparkAnswerAndOperator(""" + |SELECT /*+ MERGE(l, r) */ l.id, l.lv, r.rv + |FROM smj_left l + |JOIN smj_right r + | ON l.id = r.id AND l.lv < r.rv + |ORDER BY l.id + |""".stripMargin) + + val plan = stripAQEPlan(df.queryExecution.executedPlan) + assert( + plan.collectFirst { case _: NativeSortMergeJoinBase => true }.isDefined, + s"expected NativeSortMergeJoinBase in executed plan, but got:\n$plan") + } + } + } + } + + test( + "native shuffled hash join supports inner residual condition in forceShuffledHashJoin mode") { + withSparkConf("spark.auron.forceShuffledHashJoin" -> "true") { + withSQLConf( + "spark.sql.adaptive.enabled" -> "false", + "spark.sql.autoBroadcastJoinThreshold" -> "-1", + "spark.sql.join.preferSortMergeJoin" -> "true") { + withTable("shj_left", "shj_right") { + sql(""" + |CREATE TABLE shj_left USING parquet AS + |SELECT id, cast(id % 4 as int) AS lv + |FROM range(0, 1000) + |""".stripMargin) + + sql(""" + |CREATE TABLE shj_right USING parquet AS + |SELECT * FROM VALUES + | (1L, 2), + | (2L, 1), + | (3L, 5), + | (10L, 4) + |AS t(id, rv) + |""".stripMargin) + + val df = checkSparkAnswerAndOperator(""" + |SELECT /*+ MERGE(l, r) */ l.id, l.lv, r.rv + |FROM shj_left l + |JOIN shj_right r + | ON l.id = r.id AND l.lv < r.rv + |ORDER BY l.id + |""".stripMargin) + + val plan = stripAQEPlan(df.queryExecution.executedPlan) + assert( + plan.collectFirst { case _: NativeShuffledHashJoinBase => true }.isDefined, + s"expected NativeShuffledHashJoinBase in executed plan, but got:\n$plan") + } + } + } + } + test("left join with NOT IN subquery should filter NULL values") { // This test verifies the fix for the NULL handling issue in Anti join. withSQLConf("spark.sql.autoBroadcastJoinThreshold" -> "-1") { diff --git a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronSQLTestHelper.scala b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronSQLTestHelper.scala index cef12dbd6..b80e2d4ad 100644 --- a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronSQLTestHelper.scala +++ b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronSQLTestHelper.scala @@ -16,9 +16,12 @@ */ package org.apache.auron +import org.apache.spark.SparkEnv import org.apache.spark.sql.internal.SQLConf trait AuronSQLTestHelper { + self: BaseAuronSQLSuite => + def withEnvConf(pairs: (String, String)*)(f: => Unit): Unit = { val conf = SQLConf.get val (keys, values) = pairs.unzip @@ -29,7 +32,7 @@ trait AuronSQLTestHelper { None } } - (keys, values).zipped.foreach { (k, v) => + keys.zip(values).foreach { case (k, v) => conf.setConfString(k, v) } try f @@ -40,4 +43,26 @@ trait AuronSQLTestHelper { } } } + + def withSparkConf(pairs: (String, String)*)(f: => Unit): Unit = { + val confs = Seq(spark.sparkContext.getConf, SparkEnv.get.conf).distinct + val (keys, values) = pairs.unzip + val currentValuesByConf = confs.map(conf => conf -> keys.map(conf.getOption)) + + confs.foreach { conf => + keys.zip(values).foreach { case (k, v) => + conf.set(k, v) + } + } + + try f + finally { + currentValuesByConf.foreach { case (conf, currentValues) => + keys.zip(currentValues).foreach { + case (key, Some(value)) => conf.set(key, value) + case (key, None) => conf.remove(key) + } + } + } + } } diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala index cc12a176a..b0e2f6a9a 100644 --- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala +++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala @@ -46,6 +46,8 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction import org.apache.spark.sql.catalyst.expressions.aggregate.Final import org.apache.spark.sql.catalyst.expressions.aggregate.Partial +import org.apache.spark.sql.catalyst.plans.InnerLike +import org.apache.spark.sql.catalyst.plans.JoinType import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning import org.apache.spark.sql.catalyst.plans.physical.Partitioning import org.apache.spark.sql.catalyst.plans.physical.RangePartitioning @@ -506,7 +508,6 @@ object AuronConverters extends Logging { "rightKeys" -> rightKeys, "joinType" -> joinType, "condition" -> condition)) - assert(condition.isEmpty, "join condition is not supported") val buildSide = exec .getTagValue(joinSmallerSideTag) @@ -516,14 +517,17 @@ object AuronConverters extends Logging { JoinBuildRight } - return Shims.get.createNativeShuffledHashJoinExec( - addRenameColumnsExec(convertToNative(left.children(0))), - addRenameColumnsExec(convertToNative(right.children(0))), - leftKeys, - rightKeys, + return wrapInnerResidualJoinCondition( joinType, - buildSide, - isSkewJoin) + condition, + Shims.get.createNativeShuffledHashJoinExec( + addRenameColumnsExec(convertToNative(left.children(0))), + addRenameColumnsExec(convertToNative(right.children(0))), + leftKeys, + rightKeys, + joinType, + buildSide, + isSkewJoin)) } val (leftKeys, rightKeys, joinType, condition, left, right, isSkewJoin) = @@ -542,15 +546,17 @@ object AuronConverters extends Logging { "rightKeys" -> rightKeys, "joinType" -> joinType, "condition" -> condition)) - assert(condition.isEmpty, "join condition is not supported") - Shims.get.createNativeSortMergeJoinExec( - addRenameColumnsExec(convertToNative(left)), - addRenameColumnsExec(convertToNative(right)), - leftKeys, - rightKeys, + wrapInnerResidualJoinCondition( joinType, - isSkewJoin) + condition, + Shims.get.createNativeSortMergeJoinExec( + addRenameColumnsExec(convertToNative(left)), + addRenameColumnsExec(convertToNative(right)), + leftKeys, + rightKeys, + joinType, + isSkewJoin)) } def convertShuffledHashJoinExec(exec: ShuffledHashJoinExec): SparkPlan = { @@ -566,15 +572,17 @@ object AuronConverters extends Logging { "condition" -> condition, "buildSide" -> buildSide)) try { - assert(condition.isEmpty, "join condition is not supported") - Shims.get.createNativeShuffledHashJoinExec( - addRenameColumnsExec(convertToNative(left)), - addRenameColumnsExec(convertToNative(right)), - leftKeys, - rightKeys, + wrapInnerResidualJoinCondition( joinType, - buildSide, - Shims.get.getIsSkewJoinFromSHJ(exec)) + condition, + Shims.get.createNativeShuffledHashJoinExec( + addRenameColumnsExec(convertToNative(left)), + addRenameColumnsExec(convertToNative(right)), + leftKeys, + rightKeys, + joinType, + buildSide, + Shims.get.getIsSkewJoinFromSHJ(exec))) } catch { case _ if sparkAuronConfig.getBoolean(SparkAuronConfiguration.FORCE_SHUFFLED_HASH_JOIN) => logWarning( @@ -613,6 +621,19 @@ object AuronConverters extends Logging { } } + private def wrapInnerResidualJoinCondition( + joinType: JoinType, + condition: Option[Expression], + joined: SparkPlan): SparkPlan = { + condition match { + case Some(residualCondition) => + assert(joinType.isInstanceOf[InnerLike], "join condition is not supported") + convertFilterExec(FilterExec(residualCondition, joined)) + case None => + joined + } + } + def convertBroadcastHashJoinExec(exec: BroadcastHashJoinExec): SparkPlan = { val buildSide = Shims.get.getJoinBuildSide(exec) try {