From f2ec8014398b3f076b1120f10231004cf74faf95 Mon Sep 17 00:00:00 2001 From: Soumyakanti Das Date: Wed, 13 May 2026 16:33:48 -0700 Subject: [PATCH 1/5] HIVE-29612: LATERAL VIEW OUTER doesn't work with CBO enabled --- .../reloperators/HiveTableFunctionScan.java | 36 +++- .../calcite/translator/ASTConverter.java | 15 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 13 +- .../ql/parse/relnodegen/LateralViewPlan.java | 13 +- .../clientpositive/lateral_view_outer_cbo.q | 16 ++ .../llap/lateral_view_outer_cbo.q.out | 160 ++++++++++++++++++ 6 files changed, 235 insertions(+), 18 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q create mode 100644 ql/src/test/results/clientpositive/llap/lateral_view_outer_cbo.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableFunctionScan.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableFunctionScan.java index 176d27e07d91..eb148dda25fe 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableFunctionScan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveTableFunctionScan.java @@ -25,6 +25,7 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelInput; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.core.TableFunctionScan; import org.apache.calcite.rel.metadata.RelColumnMapping; import org.apache.calcite.rel.type.RelDataType; @@ -34,6 +35,9 @@ public class HiveTableFunctionScan extends TableFunctionScan implements HiveRelNode { + // Whether this is a LATERAL VIEW OUTER + private final boolean outer; + /** * @param cluster * cluster - Cluster that this relational expression belongs to @@ -49,28 +53,54 @@ public class HiveTableFunctionScan extends TableFunctionScan implements HiveRelN * rowType - Row type produced by function * @param columnMappings * columnMappings - Column mappings associated with this function + * @param outer + * outer - true if this is a LATERAL VIEW OUTER */ protected HiveTableFunctionScan(RelOptCluster cluster, RelTraitSet traitSet, List inputs, - RexNode rexCall, Type elementType, RelDataType rowType, Set columnMappings) { + RexNode rexCall, Type elementType, RelDataType rowType, Set columnMappings, + boolean outer) { super(cluster, traitSet, inputs, rexCall, elementType, rowType, columnMappings); + this.outer = outer; } public HiveTableFunctionScan(RelInput input) { super(input); + this.outer = input.getBoolean("outer", false); } public static HiveTableFunctionScan create(RelOptCluster cluster, RelTraitSet traitSet, List inputs, RexNode rexCall, Type elementType, RelDataType rowType, Set columnMappings) throws CalciteSemanticException { return new HiveTableFunctionScan(cluster, traitSet, inputs, rexCall, elementType, rowType, - columnMappings); + columnMappings, false); + } + + public static HiveTableFunctionScan create(RelOptCluster cluster, RelTraitSet traitSet, + List inputs, RexNode rexCall, Type elementType, RelDataType rowType, + Set columnMappings, boolean outer) throws CalciteSemanticException { + return new HiveTableFunctionScan(cluster, traitSet, inputs, rexCall, elementType, rowType, + columnMappings, outer); + } + + /** Returns true if this represents a LATERAL VIEW OUTER. */ + public boolean isOuter() { + return outer; + } + + @Override + public RelWriter explainTerms(RelWriter pw) { + super.explainTerms(pw); + if (outer) { + pw.item("outer", true); + } + return pw; } @Override public TableFunctionScan copy(RelTraitSet traitSet, List inputs, RexNode rexCall, Type elementType, RelDataType rowType, Set columnMappings) { return new HiveTableFunctionScan(getCluster(), traitSet, inputs, rexCall, - elementType, rowType, columnMappings); + elementType, rowType, columnMappings, outer); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java index 695196502ed1..f3e47f010646 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java @@ -703,14 +703,21 @@ private static QueryBlockInfo createASTLateralView(TableFunctionScan tfs, QueryB sel.add(selexpr.node()); // place the SELECT clause under the LATERAL VIEW clause - ASTBuilder lateralview = ASTBuilder.construct(HiveParser.TOK_LATERAL_VIEW, "TOK_LATERAL_VIEW"); - lateralview.add(sel.node()); + final boolean isOuterLateralView = tfs instanceof HiveTableFunctionScan htfs && htfs.isOuter(); + final int lateralViewToken = isOuterLateralView + ? HiveParser.TOK_LATERAL_VIEW_OUTER + : HiveParser.TOK_LATERAL_VIEW; + final String lateralViewText = isOuterLateralView + ? "TOK_LATERAL_VIEW_OUTER" + : "TOK_LATERAL_VIEW"; + ASTBuilder lateralView = ASTBuilder.construct(lateralViewToken, lateralViewText); + lateralView.add(sel.node()); // finally, add the LATERAL VIEW clause under the left side source which is the base table. - lateralview.add(tableFunctionSource.ast); + lateralView.add(tableFunctionSource.ast); Schema outputSchema = new Schema(tableFunctionSource.schema, new Schema(alias, lvFields)); - return new QueryBlockInfo(outputSchema, lateralview.node()); + return new QueryBlockInfo(outputSchema, lateralView.node()); } private boolean isLateralView(RelNode relNode) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index d5c683daa303..15fe7a664078 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1042,8 +1042,8 @@ boolean isCBOExecuted() { @Override boolean isCBOSupportedLateralView(ASTNode lateralView) { - // LATERAL VIEW OUTER not supported in CBO - return lateralView.getToken().getType() != HiveParser.TOK_LATERAL_VIEW_OUTER; + // Both LATERAL VIEW and LATERAL VIEW OUTER are supported in CBO. + return true; } @Override @@ -2980,7 +2980,8 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map SELECT -> SELEXPR -> FUNCTION -> function info tree ASTNode selExprAST = (ASTNode) lateralView.getChild(0).getChild(0); ASTNode functionAST = (ASTNode) selExprAST.getChild(0); @@ -118,7 +118,7 @@ public LateralViewPlan(ASTNode lateralView, RelOptCluster cluster, RelNode input this.lateralViewRel = HiveTableFunctionScan.create(cluster, TraitsUtil.getDefaultTraitSet(cluster), ImmutableList.of(inputRel), udtfCall, - null, retType, createColumnMappings(inputRel)); + null, retType, createColumnMappings(inputRel), isOuter); } public static void validateLateralView(ASTNode lateralView) throws SemanticException { @@ -128,8 +128,9 @@ public static void validateLateralView(ASTNode lateralView) throws SemanticExcep } ASTNode next = (ASTNode) lateralView.getChild(1); if (!TABLE_ALIAS_TOKEN_TYPES.contains(next.getToken().getType()) && - HiveParser.TOK_LATERAL_VIEW != next.getToken().getType()) { - throw new SemanticException(ASTErrorUtils.getMsg( + HiveParser.TOK_LATERAL_VIEW != next.getToken().getType() && + HiveParser.TOK_LATERAL_VIEW_OUTER != next.getToken().getType()) { + throw new SemanticException(ASTErrorUtils.getMsg( ErrorMsg.LATERAL_VIEW_INVALID_CHILD.getMsg(), lateralView)); } } diff --git a/ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q b/ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q new file mode 100644 index 000000000000..497700def744 --- /dev/null +++ b/ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q @@ -0,0 +1,16 @@ +CREATE TABLE test (id string, items array); +INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array()); + +CREATE VIEW v AS +SELECT test.id AS id, item +FROM test +LATERAL VIEW OUTER explode(test.items) lv AS item; + +-- CBO plan should contain `outer=[true]` in HiveTableFunctionScan node. +EXPLAIN CBO +SELECT id, item FROM v ORDER BY id, item; +-- Explain plan should contain `outer lateral view: true` in the UDTF Operator +EXPLAIN +SELECT id, item FROM v ORDER BY id, item; +-- One of the output row should be ('D', NULL) since it's an outer lateral view. +SELECT id, item FROM v ORDER BY id, item; diff --git a/ql/src/test/results/clientpositive/llap/lateral_view_outer_cbo.q.out b/ql/src/test/results/clientpositive/llap/lateral_view_outer_cbo.q.out new file mode 100644 index 000000000000..b34312f3ca31 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/lateral_view_outer_cbo.q.out @@ -0,0 +1,160 @@ +PREHOOK: query: CREATE TABLE test (id string, items array) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test +POSTHOOK: query: CREATE TABLE test (id string, items array) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test +PREHOOK: query: INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array()) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test +POSTHOOK: query: INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array()) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test +POSTHOOK: Lineage: test.id SCRIPT [] +POSTHOOK: Lineage: test.items SCRIPT [] +PREHOOK: query: CREATE VIEW v AS +SELECT test.id AS id, item +FROM test +LATERAL VIEW OUTER explode(test.items) lv AS item +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@test +PREHOOK: Output: database:default +PREHOOK: Output: default@v +POSTHOOK: query: CREATE VIEW v AS +SELECT test.id AS id, item +FROM test +LATERAL VIEW OUTER explode(test.items) lv AS item +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@test +POSTHOOK: Output: database:default +POSTHOOK: Output: default@v +POSTHOOK: Lineage: v.id SIMPLE [(test)test.FieldSchema(name:id, type:string, comment:null), ] +POSTHOOK: Lineage: v.item SCRIPT [(test)test.FieldSchema(name:items, type:array, comment:null), ] +PREHOOK: query: EXPLAIN CBO +SELECT id, item FROM v ORDER BY id, item +PREHOOK: type: QUERY +PREHOOK: Input: default@test +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO +SELECT id, item FROM v ORDER BY id, item +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test +POSTHOOK: Input: default@v +#### A masked pattern was here #### +CBO PLAN: +HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + HiveProject(id=[$0], item=[$6]) + HiveTableFunctionScan(invocation=[LATERAL(explode($1), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) id, VARCHAR(2147483647) ARRAY items, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) lv.item)], outer=[true]) + HiveTableScan(table=[[default, test]], table:alias=[test]) + +PREHOOK: query: EXPLAIN +SELECT id, item FROM v ORDER BY id, item +PREHOOK: type: QUERY +PREHOOK: Input: default@test +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT id, item FROM v ORDER BY id, item +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test +POSTHOOK: Input: default@v +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: test + properties: + insideView TRUE + Statistics: Num rows: 3 Data size: 6015 Basic stats: COMPLETE Column stats: PARTIAL + Lateral View Forward + Statistics: Num rows: 3 Data size: 6015 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: id (type: string) + outputColumnNames: id + Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL + Lateral View Join Operator + outputColumnNames: _col0, _col6 + Statistics: Num rows: 3 Data size: 6015 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: items (type: array) + outputColumnNames: _col0 + Statistics: Num rows: 3 Data size: 5760 Basic stats: COMPLETE Column stats: PARTIAL + UDTF Operator + Statistics: Num rows: 3 Data size: 5760 Basic stats: COMPLETE Column stats: PARTIAL + function name: explode + outer lateral view: true + Lateral View Join Operator + outputColumnNames: _col0, _col6 + Statistics: Num rows: 3 Data size: 6015 Basic stats: COMPLETE Column stats: PARTIAL + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string), _col1 (type: string) + null sort order: zz + sort order: ++ + Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL + File Output Operator + compressed: false + Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT id, item FROM v ORDER BY id, item +PREHOOK: type: QUERY +PREHOOK: Input: default@test +PREHOOK: Input: default@v +#### A masked pattern was here #### +POSTHOOK: query: SELECT id, item FROM v ORDER BY id, item +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test +POSTHOOK: Input: default@v +#### A masked pattern was here #### +A a +A b +B c +D NULL From da242cd09f43a3bf4e29826805e8915d9e2c81ad Mon Sep 17 00:00:00 2001 From: Soumyakanti Das Date: Fri, 15 May 2026 10:50:54 -0700 Subject: [PATCH 2/5] Refactor to address review comments, and update isCBOSupportedLateralView --- .../hadoop/hive/ql/parse/CalcitePlanner.java | 23 ++++++++++++------- .../hive/ql/parse/SemanticAnalyzer.java | 6 ++--- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 15fe7a664078..20275b9772da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -1041,9 +1041,9 @@ boolean isCBOExecuted() { } @Override - boolean isCBOSupportedLateralView(ASTNode lateralView) { + boolean isCBOSupportedLateralView() { // Both LATERAL VIEW and LATERAL VIEW OUTER are supported in CBO. - return true; + return !this.conf.getBoolVar(HiveConf.ConfVars.HIVE_CBO_RETPATH_HIVEOP); } @Override @@ -2980,8 +2980,7 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map Date: Mon, 18 May 2026 10:10:49 -0700 Subject: [PATCH 3/5] Refactor again to capture missing calls to isASTNodeLateralViewOrOuter --- .../hadoop/hive/ql/parse/CalcitePlanner.java | 10 ---------- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 16 ++++++++++++---- .../ql/parse/relnodegen/LateralViewPlan.java | 5 ++--- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 20275b9772da..abd6b77eacb5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -5646,14 +5646,4 @@ private RelNode endGenOBLogicalPlan(RelNode sortRel) throws CalciteSemanticExcep return sortRel; } } - - /** - * Utility method to determine if an AST node represents a lateral view or lateral view outer. - * @param node AST node - * @return true if node is of lateral view or lateral view outer; false otherwise. - */ - private boolean isASTNodeLateralViewOrOuter(ASTNode node) { - return node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW - || node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER; - } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 749ad8acda32..96cfe0020fed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -1683,8 +1683,7 @@ private void processJoin(QB qb, ASTNode join) throws SemanticException { "PTF invocation in a Join must have an alias")); } - } else if (child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW || - child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) { + } else if (isASTNodeLateralViewOrOuter(child)) { // SELECT * FROM src1 LATERAL VIEW udtf() AS myTable JOIN src2 ... // is not supported. Instead, the lateral view must be in a subquery // SELECT * FROM (SELECT * FROM src1 LATERAL VIEW udtf() AS myTable) a @@ -1887,8 +1886,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, processTable(qb, frm); } else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) { processSubQuery(qb, frm); - } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW || - frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) { + } else if (isASTNodeLateralViewOrOuter(frm)) { queryProperties.setHasLateralViews(true); processLateralView(qb, frm); } else if (isJoinToken(frm)) { @@ -15524,4 +15522,14 @@ public void startAnalysis() { queryState.createHMSCache(); } } + + /** + * Utility method to determine if an AST node represents a lateral view or lateral view outer. + * @param node AST node + * @return true if node is of lateral view or lateral view outer; false otherwise. + */ + public static boolean isASTNodeLateralViewOrOuter(ASTNode node) { + return node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW + || node.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/relnodegen/LateralViewPlan.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/relnodegen/LateralViewPlan.java index da08f51d1f7c..ea072aafb274 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/relnodegen/LateralViewPlan.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/relnodegen/LateralViewPlan.java @@ -127,9 +127,8 @@ public static void validateLateralView(ASTNode lateralView) throws SemanticExcep " children."); } ASTNode next = (ASTNode) lateralView.getChild(1); - if (!TABLE_ALIAS_TOKEN_TYPES.contains(next.getToken().getType()) && - HiveParser.TOK_LATERAL_VIEW != next.getToken().getType() && - HiveParser.TOK_LATERAL_VIEW_OUTER != next.getToken().getType()) { + if (!TABLE_ALIAS_TOKEN_TYPES.contains(next.getToken().getType()) && + !SemanticAnalyzer.isASTNodeLateralViewOrOuter(next)) { throw new SemanticException(ASTErrorUtils.getMsg( ErrorMsg.LATERAL_VIEW_INVALID_CHILD.getMsg(), lateralView)); } From ac8eec2a61bbd0e1938a27d47b3144bce72c5a30 Mon Sep 17 00:00:00 2001 From: Soumyakanti Das Date: Tue, 26 May 2026 15:12:09 -0700 Subject: [PATCH 4/5] Refactor method name and tests --- .../hadoop/hive/ql/parse/CalcitePlanner.java | 6 +- .../hive/ql/parse/SemanticAnalyzer.java | 6 +- .../ql/parse/relnodegen/LateralViewPlan.java | 2 +- .../clientpositive/lateral_view_outer.q | 19 +++ .../clientpositive/lateral_view_outer_cbo.q | 16 -- .../llap/lateral_view_cbo.q.out | 113 ------------- .../llap/lateral_view_outer.q.out | 160 ++++++++++++++++++ 7 files changed, 186 insertions(+), 136 deletions(-) delete mode 100644 ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q delete mode 100644 ql/src/test/results/clientpositive/llap/lateral_view_cbo.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index abd6b77eacb5..f7dc75efe89e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -2980,7 +2980,7 @@ private RelNode genJoinLogicalPlan(QB qb, ASTNode joinParseTree, Map 300, array(value, value), nul explain select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10; +explain ast select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10; +explain cbo +select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10; +select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10; + +-- array_valued already has a nullable array column, which can be used for the view-based test +CREATE VIEW array_valued_view AS +SELECT array_valued.key AS key, a +FROM array_valued +LATERAL VIEW OUTER explode(value) lv AS a; + +-- CBO plan should contain `outer=[true]` in HiveTableFunctionScan node. +EXPLAIN CBO +SELECT key, a FROM array_valued_view limit 10; +-- Explain plan should contain `outer lateral view: true` in the UDTF Operator +EXPLAIN +SELECT key, a FROM array_valued_view limit 10; +-- Rows with null array value should still appear with a=NULL +SELECT key, a FROM array_valued_view limit 10; diff --git a/ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q b/ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q deleted file mode 100644 index 497700def744..000000000000 --- a/ql/src/test/queries/clientpositive/lateral_view_outer_cbo.q +++ /dev/null @@ -1,16 +0,0 @@ -CREATE TABLE test (id string, items array); -INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array()); - -CREATE VIEW v AS -SELECT test.id AS id, item -FROM test -LATERAL VIEW OUTER explode(test.items) lv AS item; - --- CBO plan should contain `outer=[true]` in HiveTableFunctionScan node. -EXPLAIN CBO -SELECT id, item FROM v ORDER BY id, item; --- Explain plan should contain `outer lateral view: true` in the UDTF Operator -EXPLAIN -SELECT id, item FROM v ORDER BY id, item; --- One of the output row should be ('D', NULL) since it's an outer lateral view. -SELECT id, item FROM v ORDER BY id, item; diff --git a/ql/src/test/results/clientpositive/llap/lateral_view_cbo.q.out b/ql/src/test/results/clientpositive/llap/lateral_view_cbo.q.out deleted file mode 100644 index 762801321efb..000000000000 --- a/ql/src/test/results/clientpositive/llap/lateral_view_cbo.q.out +++ /dev/null @@ -1,113 +0,0 @@ -PREHOOK: query: CREATE TABLE simple_table (col string, array_col array) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@simple_table -POSTHOOK: query: CREATE TABLE simple_table (col string, array_col array) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@simple_table -PREHOOK: query: EXPLAIN CBO SELECT myTable.myCol FROM simple_table -LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -PREHOOK: type: QUERY -PREHOOK: Input: default@simple_table -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN CBO SELECT myTable.myCol FROM simple_table -LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -POSTHOOK: type: QUERY -POSTHOOK: Input: default@simple_table -#### A masked pattern was here #### -CBO PLAN: -HiveProject(mytable.mycol=[$6]) - HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol)]) - HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) - -PREHOOK: query: EXPLAIN CBO SELECT myTable.myCol, myTable2.myCol2 FROM simple_table -LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2 -PREHOOK: type: QUERY -PREHOOK: Input: default@simple_table -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN CBO SELECT myTable.myCol, myTable2.myCol2 FROM simple_table -LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@simple_table -#### A masked pattern was here #### -CBO PLAN: -HiveProject(mytable.mycol=[$6], mytable2.mycol2=[$7]) - HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5, $6)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol, INTEGER mytable2.mycol2)]) - HiveProject(col=[$0], array_col=[$1], BLOCK__OFFSET__INSIDE__FILE=[$2], INPUT__FILE__NAME=[$3], ROW__ID=[$4], ROW__IS__DELETED=[$5], mytable.mycol=[$6]) - HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol)]) - HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) - -PREHOOK: query: EXPLAIN CBO SELECT tf.col1, tf.col2, tf.col3 -FROM simple_table - LATERAL TABLE(VALUES('A', 10, simple_table.col),('B', 20, simple_table.col)) AS tf(col1, col2, col3) -PREHOOK: type: QUERY -PREHOOK: Input: default@simple_table -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN CBO SELECT tf.col1, tf.col2, tf.col3 -FROM simple_table - LATERAL TABLE(VALUES('A', 10, simple_table.col),('B', 20, simple_table.col)) AS tf(col1, col2, col3) -POSTHOOK: type: QUERY -POSTHOOK: Input: default@simple_table -#### A masked pattern was here #### -CBO PLAN: -HiveProject(tf.col1=[$6], tf.col2=[$7], tf.col3=[$8]) - HiveTableFunctionScan(invocation=[inline(ARRAY(ROW($0, $1, $2, $3, $4, $5, _UTF-16LE'A':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", 10, $0), ROW($0, $1, $2, $3, $4, $5, _UTF-16LE'B':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", 20, $0)))], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) tf.col1, INTEGER tf.col2, VARCHAR(2147483647) tf.col3)]) - HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) - -PREHOOK: query: EXPLAIN CBO SELECT myTable.myCol FROM simple_table -LATERAL VIEW explode(simple_table.array_col) myTable AS myCol -PREHOOK: type: QUERY -PREHOOK: Input: default@simple_table -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN CBO SELECT myTable.myCol FROM simple_table -LATERAL VIEW explode(simple_table.array_col) myTable AS myCol -POSTHOOK: type: QUERY -POSTHOOK: Input: default@simple_table -#### A masked pattern was here #### -CBO PLAN: -HiveProject(mytable.mycol=[$6]) - HiveTableFunctionScan(invocation=[LATERAL(explode($1), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) mytable.mycol)]) - HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) - -PREHOOK: query: EXPLAIN CBO SELECT myCol FROM -(SELECT * FROM simple_table -LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2) a WHERE col='0' -PREHOOK: type: QUERY -PREHOOK: Input: default@simple_table -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN CBO SELECT myCol FROM -(SELECT * FROM simple_table -LATERAL VIEW explode(array(1,2,3)) myTable AS myCol -LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2) a WHERE col='0' -POSTHOOK: type: QUERY -POSTHOOK: Input: default@simple_table -#### A masked pattern was here #### -CBO PLAN: -HiveProject(mycol=[$6]) - HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5, $6)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol, INTEGER mytable2.mycol2)]) - HiveProject(col=[$0], array_col=[$1], BLOCK__OFFSET__INSIDE__FILE=[$2], INPUT__FILE__NAME=[$3], ROW__ID=[$4], ROW__IS__DELETED=[$5], mytable.mycol=[$6]) - HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol)]) - HiveProject(col=[$0], array_col=[$1], BLOCK__OFFSET__INSIDE__FILE=[$2], INPUT__FILE__NAME=[$3], ROW__ID=[$4], ROW__IS__DELETED=[$5]) - HiveFilter(condition=[=($0, _UTF-16LE'0')]) - HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) - -PREHOOK: query: EXPLAIN CBO SELECT myCol FROM simple_table -LATERAL VIEW explode(simple_table.array_col) myTable AS myCol where myCol = 1 -PREHOOK: type: QUERY -PREHOOK: Input: default@simple_table -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN CBO SELECT myCol FROM simple_table -LATERAL VIEW explode(simple_table.array_col) myTable AS myCol where myCol = 1 -POSTHOOK: type: QUERY -POSTHOOK: Input: default@simple_table -#### A masked pattern was here #### -CBO PLAN: -HiveProject(mycol=[$6]) - HiveFilter(condition=[=(CAST($6):DOUBLE, 1)]) - HiveTableFunctionScan(invocation=[LATERAL(explode($1), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) mytable.mycol)]) - HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) - diff --git a/ql/src/test/results/clientpositive/llap/lateral_view_outer.q.out b/ql/src/test/results/clientpositive/llap/lateral_view_outer.q.out index c5959dce36b5..a44dddfc09e5 100644 --- a/ql/src/test/results/clientpositive/llap/lateral_view_outer.q.out +++ b/ql/src/test/results/clientpositive/llap/lateral_view_outer.q.out @@ -187,6 +187,59 @@ STAGE PLANS: outputColumnNames: _col0, _col1, _col2 ListSink +PREHOOK: query: explain ast +select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@array_valued +#### A masked pattern was here #### +POSTHOOK: query: explain ast +select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@array_valued +#### A masked pattern was here #### +ABSTRACT SYNTAX TREE: + +TOK_QUERY + TOK_FROM + TOK_LATERAL_VIEW_OUTER + TOK_SELECT + TOK_SELEXPR + TOK_FUNCTION + explode + TOK_TABLE_OR_COL + value + a + TOK_TABALIAS + C + TOK_TABREF + TOK_TABNAME + array_valued + TOK_INSERT + TOK_DESTINATION + TOK_DIR + TOK_TMP_FILE + TOK_SELECT + TOK_SELEXPR + TOK_ALLCOLREF + TOK_LIMIT + 10 + +PREHOOK: query: explain cbo +select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@array_valued +#### A masked pattern was here #### +POSTHOOK: query: explain cbo +select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@array_valued +#### A masked pattern was here #### +CBO PLAN: +HiveSortLimit(fetch=[10]) + HiveProject(array_valued.key=[$0], array_valued.value=[$1], c.a=[$6]) + HiveTableFunctionScan(invocation=[LATERAL(explode($1), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) key, VARCHAR(2147483647) ARRAY value, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) c.a)], outer=[true]) + HiveTableScan(table=[[default, array_valued]], table:alias=[array_valued]) + PREHOOK: query: select * from array_valued LATERAL VIEW OUTER explode(value) C AS a limit 10 PREHOOK: type: QUERY PREHOOK: Input: default@array_valued @@ -205,3 +258,110 @@ POSTHOOK: Input: default@array_valued 409 ["val_409","val_409"] val_409 255 NULL NULL 278 NULL NULL +PREHOOK: query: CREATE VIEW array_valued_view AS +SELECT array_valued.key AS key, a +FROM array_valued +LATERAL VIEW OUTER explode(value) lv AS a +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@array_valued +PREHOOK: Output: database:default +PREHOOK: Output: default@array_valued_view +POSTHOOK: query: CREATE VIEW array_valued_view AS +SELECT array_valued.key AS key, a +FROM array_valued +LATERAL VIEW OUTER explode(value) lv AS a +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@array_valued +POSTHOOK: Output: database:default +POSTHOOK: Output: default@array_valued_view +POSTHOOK: Lineage: array_valued_view.a SCRIPT [(array_valued)array_valued.FieldSchema(name:value, type:array, comment:null), ] +POSTHOOK: Lineage: array_valued_view.key SIMPLE [(array_valued)array_valued.FieldSchema(name:key, type:string, comment:null), ] +PREHOOK: query: EXPLAIN CBO +SELECT key, a FROM array_valued_view limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@array_valued +PREHOOK: Input: default@array_valued_view +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO +SELECT key, a FROM array_valued_view limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@array_valued +POSTHOOK: Input: default@array_valued_view +#### A masked pattern was here #### +CBO PLAN: +HiveSortLimit(fetch=[10]) + HiveProject(key=[$0], a=[$6]) + HiveTableFunctionScan(invocation=[LATERAL(explode($1), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) key, VARCHAR(2147483647) ARRAY value, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) lv.a)], outer=[true]) + HiveTableScan(table=[[default, array_valued]], table:alias=[array_valued]) + +PREHOOK: query: EXPLAIN +SELECT key, a FROM array_valued_view limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@array_valued +PREHOOK: Input: default@array_valued_view +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN +SELECT key, a FROM array_valued_view limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@array_valued +POSTHOOK: Input: default@array_valued_view +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: 10 + Processor Tree: + TableScan + alias: array_valued + properties: + insideView TRUE + Lateral View Forward + Select Operator + expressions: key (type: string) + outputColumnNames: key + Lateral View Join Operator + outputColumnNames: _col0, _col6 + Limit + Number of rows: 10 + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + ListSink + Select Operator + expressions: value (type: array) + outputColumnNames: _col0 + UDTF Operator + function name: explode + outer lateral view: true + Lateral View Join Operator + outputColumnNames: _col0, _col6 + Limit + Number of rows: 10 + Select Operator + expressions: _col0 (type: string), _col6 (type: string) + outputColumnNames: _col0, _col1 + ListSink + +PREHOOK: query: SELECT key, a FROM array_valued_view limit 10 +PREHOOK: type: QUERY +PREHOOK: Input: default@array_valued +PREHOOK: Input: default@array_valued_view +#### A masked pattern was here #### +POSTHOOK: query: SELECT key, a FROM array_valued_view limit 10 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@array_valued +POSTHOOK: Input: default@array_valued_view +#### A masked pattern was here #### +238 NULL +86 NULL +311 val_311 +311 val_311 +27 NULL +165 NULL +409 val_409 +409 val_409 +255 NULL +278 NULL From fe5faf7d95783cebe55ef07b6d7ee2ebff61efd3 Mon Sep 17 00:00:00 2001 From: Soumyakanti Das Date: Wed, 27 May 2026 09:49:48 -0700 Subject: [PATCH 5/5] removed incorrect file, so fixing that; also removing dangling qouts --- .../llap/lateral_view_cbo.q.out | 113 +++++++++++++ .../llap/lateral_view_outer_cbo.q.out | 160 ------------------ 2 files changed, 113 insertions(+), 160 deletions(-) create mode 100644 ql/src/test/results/clientpositive/llap/lateral_view_cbo.q.out delete mode 100644 ql/src/test/results/clientpositive/llap/lateral_view_outer_cbo.q.out diff --git a/ql/src/test/results/clientpositive/llap/lateral_view_cbo.q.out b/ql/src/test/results/clientpositive/llap/lateral_view_cbo.q.out new file mode 100644 index 000000000000..762801321efb --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/lateral_view_cbo.q.out @@ -0,0 +1,113 @@ +PREHOOK: query: CREATE TABLE simple_table (col string, array_col array) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@simple_table +POSTHOOK: query: CREATE TABLE simple_table (col string, array_col array) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@simple_table +PREHOOK: query: EXPLAIN CBO SELECT myTable.myCol FROM simple_table +LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@simple_table +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO SELECT myTable.myCol FROM simple_table +LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@simple_table +#### A masked pattern was here #### +CBO PLAN: +HiveProject(mytable.mycol=[$6]) + HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol)]) + HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) + +PREHOOK: query: EXPLAIN CBO SELECT myTable.myCol, myTable2.myCol2 FROM simple_table +LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2 +PREHOOK: type: QUERY +PREHOOK: Input: default@simple_table +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO SELECT myTable.myCol, myTable2.myCol2 FROM simple_table +LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@simple_table +#### A masked pattern was here #### +CBO PLAN: +HiveProject(mytable.mycol=[$6], mytable2.mycol2=[$7]) + HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5, $6)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol, INTEGER mytable2.mycol2)]) + HiveProject(col=[$0], array_col=[$1], BLOCK__OFFSET__INSIDE__FILE=[$2], INPUT__FILE__NAME=[$3], ROW__ID=[$4], ROW__IS__DELETED=[$5], mytable.mycol=[$6]) + HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol)]) + HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) + +PREHOOK: query: EXPLAIN CBO SELECT tf.col1, tf.col2, tf.col3 +FROM simple_table + LATERAL TABLE(VALUES('A', 10, simple_table.col),('B', 20, simple_table.col)) AS tf(col1, col2, col3) +PREHOOK: type: QUERY +PREHOOK: Input: default@simple_table +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO SELECT tf.col1, tf.col2, tf.col3 +FROM simple_table + LATERAL TABLE(VALUES('A', 10, simple_table.col),('B', 20, simple_table.col)) AS tf(col1, col2, col3) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@simple_table +#### A masked pattern was here #### +CBO PLAN: +HiveProject(tf.col1=[$6], tf.col2=[$7], tf.col3=[$8]) + HiveTableFunctionScan(invocation=[inline(ARRAY(ROW($0, $1, $2, $3, $4, $5, _UTF-16LE'A':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", 10, $0), ROW($0, $1, $2, $3, $4, $5, _UTF-16LE'B':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", 20, $0)))], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) tf.col1, INTEGER tf.col2, VARCHAR(2147483647) tf.col3)]) + HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) + +PREHOOK: query: EXPLAIN CBO SELECT myTable.myCol FROM simple_table +LATERAL VIEW explode(simple_table.array_col) myTable AS myCol +PREHOOK: type: QUERY +PREHOOK: Input: default@simple_table +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO SELECT myTable.myCol FROM simple_table +LATERAL VIEW explode(simple_table.array_col) myTable AS myCol +POSTHOOK: type: QUERY +POSTHOOK: Input: default@simple_table +#### A masked pattern was here #### +CBO PLAN: +HiveProject(mytable.mycol=[$6]) + HiveTableFunctionScan(invocation=[LATERAL(explode($1), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) mytable.mycol)]) + HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) + +PREHOOK: query: EXPLAIN CBO SELECT myCol FROM +(SELECT * FROM simple_table +LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2) a WHERE col='0' +PREHOOK: type: QUERY +PREHOOK: Input: default@simple_table +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO SELECT myCol FROM +(SELECT * FROM simple_table +LATERAL VIEW explode(array(1,2,3)) myTable AS myCol +LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2) a WHERE col='0' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@simple_table +#### A masked pattern was here #### +CBO PLAN: +HiveProject(mycol=[$6]) + HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5, $6)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol, INTEGER mytable2.mycol2)]) + HiveProject(col=[$0], array_col=[$1], BLOCK__OFFSET__INSIDE__FILE=[$2], INPUT__FILE__NAME=[$3], ROW__ID=[$4], ROW__IS__DELETED=[$5], mytable.mycol=[$6]) + HiveTableFunctionScan(invocation=[LATERAL(explode(ARRAY(1, 2, 3)), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, INTEGER mytable.mycol)]) + HiveProject(col=[$0], array_col=[$1], BLOCK__OFFSET__INSIDE__FILE=[$2], INPUT__FILE__NAME=[$3], ROW__ID=[$4], ROW__IS__DELETED=[$5]) + HiveFilter(condition=[=($0, _UTF-16LE'0')]) + HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) + +PREHOOK: query: EXPLAIN CBO SELECT myCol FROM simple_table +LATERAL VIEW explode(simple_table.array_col) myTable AS myCol where myCol = 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@simple_table +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN CBO SELECT myCol FROM simple_table +LATERAL VIEW explode(simple_table.array_col) myTable AS myCol where myCol = 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@simple_table +#### A masked pattern was here #### +CBO PLAN: +HiveProject(mycol=[$6]) + HiveFilter(condition=[=(CAST($6):DOUBLE, 1)]) + HiveTableFunctionScan(invocation=[LATERAL(explode($1), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) col, VARCHAR(2147483647) ARRAY array_col, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) mytable.mycol)]) + HiveTableScan(table=[[default, simple_table]], table:alias=[simple_table]) + diff --git a/ql/src/test/results/clientpositive/llap/lateral_view_outer_cbo.q.out b/ql/src/test/results/clientpositive/llap/lateral_view_outer_cbo.q.out deleted file mode 100644 index b34312f3ca31..000000000000 --- a/ql/src/test/results/clientpositive/llap/lateral_view_outer_cbo.q.out +++ /dev/null @@ -1,160 +0,0 @@ -PREHOOK: query: CREATE TABLE test (id string, items array) -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@test -POSTHOOK: query: CREATE TABLE test (id string, items array) -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@test -PREHOOK: query: INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array()) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@test -POSTHOOK: query: INSERT INTO test VALUES ('A', array('a', 'b')), ('B', array('c')), ('D', array()) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@test -POSTHOOK: Lineage: test.id SCRIPT [] -POSTHOOK: Lineage: test.items SCRIPT [] -PREHOOK: query: CREATE VIEW v AS -SELECT test.id AS id, item -FROM test -LATERAL VIEW OUTER explode(test.items) lv AS item -PREHOOK: type: CREATEVIEW -PREHOOK: Input: default@test -PREHOOK: Output: database:default -PREHOOK: Output: default@v -POSTHOOK: query: CREATE VIEW v AS -SELECT test.id AS id, item -FROM test -LATERAL VIEW OUTER explode(test.items) lv AS item -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: default@test -POSTHOOK: Output: database:default -POSTHOOK: Output: default@v -POSTHOOK: Lineage: v.id SIMPLE [(test)test.FieldSchema(name:id, type:string, comment:null), ] -POSTHOOK: Lineage: v.item SCRIPT [(test)test.FieldSchema(name:items, type:array, comment:null), ] -PREHOOK: query: EXPLAIN CBO -SELECT id, item FROM v ORDER BY id, item -PREHOOK: type: QUERY -PREHOOK: Input: default@test -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN CBO -SELECT id, item FROM v ORDER BY id, item -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test -POSTHOOK: Input: default@v -#### A masked pattern was here #### -CBO PLAN: -HiveSortLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) - HiveProject(id=[$0], item=[$6]) - HiveTableFunctionScan(invocation=[LATERAL(explode($1), $0, $1, $2, $3, $4, $5)], rowType=[RecordType(VARCHAR(2147483647) id, VARCHAR(2147483647) ARRAY items, BIGINT BLOCK__OFFSET__INSIDE__FILE, VARCHAR(2147483647) INPUT__FILE__NAME, RecordType(BIGINT writeid, INTEGER bucketid, BIGINT rowid) ROW__ID, BOOLEAN ROW__IS__DELETED, VARCHAR(2147483647) lv.item)], outer=[true]) - HiveTableScan(table=[[default, test]], table:alias=[test]) - -PREHOOK: query: EXPLAIN -SELECT id, item FROM v ORDER BY id, item -PREHOOK: type: QUERY -PREHOOK: Input: default@test -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: EXPLAIN -SELECT id, item FROM v ORDER BY id, item -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test -POSTHOOK: Input: default@v -#### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: test - properties: - insideView TRUE - Statistics: Num rows: 3 Data size: 6015 Basic stats: COMPLETE Column stats: PARTIAL - Lateral View Forward - Statistics: Num rows: 3 Data size: 6015 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: id (type: string) - outputColumnNames: id - Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL - Lateral View Join Operator - outputColumnNames: _col0, _col6 - Statistics: Num rows: 3 Data size: 6015 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: items (type: array) - outputColumnNames: _col0 - Statistics: Num rows: 3 Data size: 5760 Basic stats: COMPLETE Column stats: PARTIAL - UDTF Operator - Statistics: Num rows: 3 Data size: 5760 Basic stats: COMPLETE Column stats: PARTIAL - function name: explode - outer lateral view: true - Lateral View Join Operator - outputColumnNames: _col0, _col6 - Statistics: Num rows: 3 Data size: 6015 Basic stats: COMPLETE Column stats: PARTIAL - Select Operator - expressions: _col0 (type: string), _col6 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL - Reduce Output Operator - key expressions: _col0 (type: string), _col1 (type: string) - null sort order: zz - sort order: ++ - Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL - File Output Operator - compressed: false - Statistics: Num rows: 3 Data size: 255 Basic stats: COMPLETE Column stats: PARTIAL - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - -PREHOOK: query: SELECT id, item FROM v ORDER BY id, item -PREHOOK: type: QUERY -PREHOOK: Input: default@test -PREHOOK: Input: default@v -#### A masked pattern was here #### -POSTHOOK: query: SELECT id, item FROM v ORDER BY id, item -POSTHOOK: type: QUERY -POSTHOOK: Input: default@test -POSTHOOK: Input: default@v -#### A masked pattern was here #### -A a -A b -B c -D NULL