diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 6007e488d396..48f9dc8f2c5d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -103,6 +103,7 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext.InConstantType; import org.apache.hadoop.hive.ql.exec.vector.VectorizationContextRegion; import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport.Support; +import org.apache.hadoop.hive.ql.exec.vector.expressions.ConvertDecimal64ToDecimal; import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression; import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression; @@ -4798,6 +4799,8 @@ public static Operator vectorizeSelectOperator( VectorExpression[] vectorSelectExprs = new VectorExpression[size]; int[] projectedOutputColumns = new int[size]; + int[] vectorSelectExprIndexForCol = new int[size]; + Arrays.fill(vectorSelectExprIndexForCol, -1); for (int i = 0; i < size; i++) { ExprNodeDesc expr = colList.get(i); VectorExpression ve = vContext.getVectorExpression(expr); @@ -4806,6 +4809,7 @@ public static Operator vectorizeSelectOperator( // Suppress useless evaluation. continue; } + vectorSelectExprIndexForCol[i] = index; vectorSelectExprs[index++] = ve; } if (index < size) { @@ -4818,6 +4822,12 @@ public static Operator vectorizeSelectOperator( // The following method introduces a cast if x or y is DECIMAL_64 and parent expression (x % y) is DECIMAL. try { fixDecimalDataTypePhysicalVariations(vContext, vectorSelectExprs); + for (int i = 0; i < size; i++) { + int exprIndex = vectorSelectExprIndexForCol[i]; + if (exprIndex >= 0) { + projectedOutputColumns[i] = vectorSelectExprs[exprIndex].getOutputColumnNum(); + } + } } finally { vContext.freeMarkedScratchColumns(); } @@ -4880,7 +4890,6 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto } } else { Object[] arguments; - int argumentCount = children.length + (parent.getOutputColumnNum() == -1 ? 0 : 1); // VectorCoalesce receives arguments as an array. // Need to handle it as a special case to avoid instantiation failure. if (parent instanceof VectorCoalesce) { @@ -4892,20 +4901,7 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto } arguments[1] = parent.getOutputColumnNum(); } else { - if (parent instanceof DecimalColDivideDecimalScalar) { - arguments = new Object[argumentCount + 1]; - arguments[children.length] = ((DecimalColDivideDecimalScalar) parent).getValue(); - } else { - arguments = new Object[argumentCount]; - } - for (int i = 0; i < children.length; i++) { - VectorExpression vce = children[i]; - arguments[i] = vce.getOutputColumnNum(); - } - } - // retain output column number from parent - if (parent.getOutputColumnNum() != -1) { - arguments[arguments.length - 1] = parent.getOutputColumnNum(); + arguments = rebuildArgumentsForDecimal64(parent, children); } // re-instantiate the parent expression with new arguments VectorExpression newParent = vContext.instantiateExpression(parent.getClass(), parent.getOutputTypeInfo(), @@ -4914,7 +4910,7 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto newParent.setOutputDataTypePhysicalVariation(parent.getOutputDataTypePhysicalVariation()); newParent.setInputTypeInfos(parent.getInputTypeInfos()); newParent.setInputDataTypePhysicalVariations(dataTypePhysicalVariations); - newParent.setChildExpressions(parent.getChildExpressions()); + newParent.setChildExpressions(children); return newParent; } } @@ -4922,6 +4918,65 @@ private static VectorExpression fixDecimalDataTypePhysicalVariations(final Vecto return parent; } + /** + * Rebuild constructor arguments for a vector expression after wrapping DECIMAL_64 child + * expressions with {@link ConvertDecimal64ToDecimal}. Column reference inputs live in + * {@link VectorExpression#inputColumnNum} and are not included in childExpressions, so they + * must be preserved when re-instantiating the parent. + */ + static Object[] rebuildArgumentsForDecimal64(VectorExpression parent, + VectorExpression[] children) { + int inputCount = 0; + for (int col : parent.inputColumnNum) { + if (col != -1) { + inputCount++; + } + } + + int[] updatedInputCols = new int[inputCount]; + int idx = 0; + for (int col : parent.inputColumnNum) { + if (col != -1) { + updatedInputCols[idx++] = col; + } + } + + for (VectorExpression child : children) { + int sourceCol = getSourceColumnForDecimal64Conversion(child); + int convertedCol = child.getOutputColumnNum(); + for (int i = 0; i < updatedInputCols.length; i++) { + if (updatedInputCols[i] == sourceCol) { + updatedInputCols[i] = convertedCol; + break; + } + } + } + + if (parent instanceof DecimalColDivideDecimalScalar) { + Object[] arguments = new Object[inputCount + 2]; + for (int i = 0; i < inputCount; i++) { + arguments[i] = updatedInputCols[i]; + } + arguments[inputCount] = ((DecimalColDivideDecimalScalar) parent).getValue(); + arguments[inputCount + 1] = parent.getOutputColumnNum(); + return arguments; + } + + Object[] arguments = new Object[inputCount + 1]; + for (int i = 0; i < inputCount; i++) { + arguments[i] = updatedInputCols[i]; + } + arguments[inputCount] = parent.getOutputColumnNum(); + return arguments; + } + + private static int getSourceColumnForDecimal64Conversion(VectorExpression child) { + if (child instanceof ConvertDecimal64ToDecimal) { + return child.inputColumnNum[0]; + } + return child.getOutputColumnNum(); + } + private static void fillInPTFEvaluators( List windowsFunctions, String[] evaluatorFunctionNames, diff --git a/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q b/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q new file mode 100644 index 000000000000..a5d7c74a3af0 --- /dev/null +++ b/ql/src/test/queries/clientpositive/vector_decimal64_col_multiply_case_subquery_join.q @@ -0,0 +1,111 @@ +SET hive.support.concurrency=TRUE; +SET hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.cbo.enable=true; + +-- HIVE-28329: subquery + LEFT JOIN + CAST(decimal) with col * CASE returns wrong result (0.00000000) + +-- Test tables +create EXTERNAL table test1( +col1 varchar(28), +col2 varchar(28), +col3 decimal(26,6), +col4 varchar(28) +); + +insert into test1 values('12345678','abc','1.00','DEF'); + +create EXTERNAL table test2( +col1 varchar(28), +col2 varchar(28), +col3 decimal(26,6), +col4 varchar(28), +col5 varchar(28) +); + +insert into test2 values('12345678','abc','1.00','DEF','22222222'); + +explain vectorization detail +SELECT + int_cost +FROM + ( + SELECT + a.col4, + CAST( + CASE when a.col1 = '12345678' then a.col3 * case when a.col2 = '1' then 1.77 else 0.72 end / 100 / 365 * 10 else a.col3 * 10 / 365 / 100 END AS DECIMAL(26, 9) + ) AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + WHERE + col5 = '22222222' + ) bb ON trim(aa.col4) = trim(bb.col4); + +SELECT + int_cost +FROM + ( + SELECT + a.col4, + CAST( + CASE when a.col1 = '12345678' then a.col3 * case when a.col2 = '1' then 1.77 else 0.72 end / 100 / 365 * 10 else a.col3 * 10 / 365 / 100 END AS DECIMAL(26, 9) + ) AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + WHERE + col5 = '22222222' + ) bb ON trim(aa.col4) = trim(bb.col4); + + +explain vectorization detail +SELECT + int_cost +FROM + ( + SELECT + a.col4, + a.col3 * case when a.col2 = 'bc' then 1.77 else 0.72 end + AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + ) bb ON trim(aa.col4) = trim(bb.col4); + + +SELECT + int_cost +FROM + ( + SELECT + a.col4, + a.col3 * case when a.col2 = 'bc' then 1.77 else 0.72 end + AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + ) bb ON trim(aa.col4) = trim(bb.col4); + +DROP TABLE test1; +DROP TABLE test2; + diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out new file mode 100644 index 000000000000..304e2a53f650 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/vector_decimal64_col_multiply_case_subquery_join.q.out @@ -0,0 +1,567 @@ +PREHOOK: query: create EXTERNAL table test1( +col1 varchar(28), +col2 varchar(28), +col3 decimal(26,6), +col4 varchar(28) +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test1 +POSTHOOK: query: create EXTERNAL table test1( +col1 varchar(28), +col2 varchar(28), +col3 decimal(26,6), +col4 varchar(28) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test1 +PREHOOK: query: insert into test1 values('12345678','abc','1.00','DEF') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test1 +POSTHOOK: query: insert into test1 values('12345678','abc','1.00','DEF') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test1 +POSTHOOK: Lineage: test1.col1 SCRIPT [] +POSTHOOK: Lineage: test1.col2 SCRIPT [] +POSTHOOK: Lineage: test1.col3 SCRIPT [] +POSTHOOK: Lineage: test1.col4 SCRIPT [] +PREHOOK: query: create EXTERNAL table test2( +col1 varchar(28), +col2 varchar(28), +col3 decimal(26,6), +col4 varchar(28), +col5 varchar(28) +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@test2 +POSTHOOK: query: create EXTERNAL table test2( +col1 varchar(28), +col2 varchar(28), +col3 decimal(26,6), +col4 varchar(28), +col5 varchar(28) +) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test2 +PREHOOK: query: insert into test2 values('12345678','abc','1.00','DEF','22222222') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@test2 +POSTHOOK: query: insert into test2 values('12345678','abc','1.00','DEF','22222222') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@test2 +POSTHOOK: Lineage: test2.col1 SCRIPT [] +POSTHOOK: Lineage: test2.col2 SCRIPT [] +POSTHOOK: Lineage: test2.col3 SCRIPT [] +POSTHOOK: Lineage: test2.col4 SCRIPT [] +POSTHOOK: Lineage: test2.col5 SCRIPT [] +PREHOOK: query: explain vectorization detail +SELECT + int_cost +FROM + ( + SELECT + a.col4, + CAST( + CASE when a.col1 = '12345678' then a.col3 * case when a.col2 = '1' then 1.77 else 0.72 end / 100 / 365 * 10 else a.col3 * 10 / 365 / 100 END AS DECIMAL(26, 9) + ) AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + WHERE + col5 = '22222222' + ) bb ON trim(aa.col4) = trim(bb.col4) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail +SELECT + int_cost +FROM + ( + SELECT + a.col4, + CAST( + CASE when a.col1 = '12345678' then a.col3 * case when a.col2 = '1' then 1.77 else 0.72 end / 100 / 365 * 10 else a.col3 * 10 / 365 / 100 END AS DECIMAL(26, 9) + ) AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + WHERE + col5 = '22222222' + ) bb ON trim(aa.col4) = trim(bb.col4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 378 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:col1:varchar(28), 1:col2:varchar(28), 2:col3:decimal(26,6), 3:col4:varchar(28), 4:ROW__ID:struct, 5:ROW__IS__DELETED:boolean] + Select Operator + expressions: col4 (type: varchar(28)), CAST( if((col1 = '12345678'), ((((col3 * if((col2 = '1'), 1.77, 0.72)) / 100) / 365) * 10), (((col3 * 10) / 365) / 100)) AS decimal(26,9)) (type: decimal(26,9)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [3, 18] + selectExpressions: CastDecimalToDecimal(col 17:decimal(38,13))(children: IfExprCondExprCondExpr(col 6:boolean, col 12:decimal(38,13), col 16:decimal(38,13))(children: StringGroupColEqualVarCharScalar(col 0:varchar(28), val 12345678) -> 6:boolean, DecimalColMultiplyDecimalScalar(col 11:decimal(38,16), val 10)(children: DecimalColDivideDecimalScalar(col 10:decimal(34,12), val 365)(children: DecimalColDivideDecimalScalar(col 9:decimal(30,8), val 100)(children: DecimalColMultiplyDecimalColumn(col 2:decimal(26,6), col 19:decimal(3,2)/DECIMAL_64)(children: ConvertDecimal64ToDecimal(col 8:decimal(3,2)/DECIMAL_64)(children: IfExprDecimal64ScalarDecimal64Scalar(col 7:boolean, decimal64Val1 177, decimalVal1 1.77, decimal64Val2 72, decimalVal2 0.72)(children: StringGroupColEqualVarCharScalar(col 1:varchar(28), val 1) -> 7:boolean) -> 8:decimal(3,2)/DECIMAL_64) -> 19:decimal(3,2)) -> 9:decimal(30,8)) -> 10:decimal(34,12)) -> 11:decimal(38,16)) -> 12:decimal(38,13), CastDecimalToDecimal(col 15:decimal(37,14))(children: DecimalColDivideDecimalScalar(col 14:decimal(33,10), val 100)(children: DecimalColDivideDecimalScalar(col 13:decimal(29,6), val 365)(children: DecimalColMultiplyDecimalScalar(col 2:decimal(26,6), val 10) -> 13:decimal(29,6)) -> 14:decimal(33,10)) -> 15:decimal(37,14)) -> 16:decimal(38,13)) -> 17:decimal(38,13)) -> 18:decimal(26,9) + Statistics: Num rows: 1 Data size: 199 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: trim(_col0) (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: trim(_col0) (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkStringOperator + keyColumns: 20:string + keyExpressions: StringTrimCol(col 3:varchar(28)) -> 20:string + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 18:decimal(26,9) + Statistics: Num rows: 1 Data size: 199 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(26,9)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [0, 1, 2, 3] + dataColumns: col1:varchar(28), col2:varchar(28), col3:decimal(26,6), col4:varchar(28) + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, bigint, decimal(3,2)/DECIMAL_64, decimal(30,8), decimal(34,12), decimal(38,16), decimal(38,13), decimal(29,6), decimal(33,10), decimal(37,14), decimal(38,13), decimal(38,13), decimal(26,9), decimal(3,2), string] + Map 3 + Map Operator Tree: + TableScan + alias: test2 + filterExpr: ((col5 = '22222222') and trim(col4) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 179 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:col1:varchar(28), 1:col2:varchar(28), 2:col3:decimal(26,6), 3:col4:varchar(28), 4:col5:varchar(28), 5:ROW__ID:struct, 6:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: FilterExprAndExpr(children: FilterStringGroupColEqualVarCharScalar(col 4:varchar(28), val 22222222), SelectColumnIsNotNull(col 7:string)(children: StringTrimCol(col 3:varchar(28)) -> 7:string)) + predicate: ((col5 = '22222222') and trim(col4) is not null) (type: boolean) + Statistics: Num rows: 1 Data size: 179 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col4 (type: varchar(28)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [3] + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: trim(_col0) (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: trim(_col0) (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkStringOperator + keyColumns: 7:string + keyExpressions: StringTrimCol(col 3:varchar(28)) -> 7:string + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 5 + includeColumns: [3, 4] + dataColumns: col1:varchar(28), col2:varchar(28), col3:decimal(26,6), col4:varchar(28), col5:varchar(28) + partitionColumnCount: 0 + scratchColumnTypeNames: [string] + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 trim(_col0) (type: string) + 1 trim(_col0) (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: decimal(26,9)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + MergeJoin Vectorization: + enabled: false + enableConditionsNotMet: Vectorizing MergeJoin Supported IS false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT + int_cost +FROM + ( + SELECT + a.col4, + CAST( + CASE when a.col1 = '12345678' then a.col3 * case when a.col2 = '1' then 1.77 else 0.72 end / 100 / 365 * 10 else a.col3 * 10 / 365 / 100 END AS DECIMAL(26, 9) + ) AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + WHERE + col5 = '22222222' + ) bb ON trim(aa.col4) = trim(bb.col4) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT + int_cost +FROM + ( + SELECT + a.col4, + CAST( + CASE when a.col1 = '12345678' then a.col3 * case when a.col2 = '1' then 1.77 else 0.72 end / 100 / 365 * 10 else a.col3 * 10 / 365 / 100 END AS DECIMAL(26, 9) + ) AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + WHERE + col5 = '22222222' + ) bb ON trim(aa.col4) = trim(bb.col4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +0.000197260 +PREHOOK: query: explain vectorization detail +SELECT + int_cost +FROM + ( + SELECT + a.col4, + a.col3 * case when a.col2 = 'bc' then 1.77 else 0.72 end + AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + ) bb ON trim(aa.col4) = trim(bb.col4) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: explain vectorization detail +SELECT + int_cost +FROM + ( + SELECT + a.col4, + a.col3 * case when a.col2 = 'bc' then 1.77 else 0.72 end + AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + ) bb ON trim(aa.col4) = trim(bb.col4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: a + Statistics: Num rows: 1 Data size: 286 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:col1:varchar(28), 1:col2:varchar(28), 2:col3:decimal(26,6), 3:col4:varchar(28), 4:ROW__ID:struct, 5:ROW__IS__DELETED:boolean] + Select Operator + expressions: col4 (type: varchar(28)), (col3 * if((col2 = 'bc'), 1.77, 0.72)) (type: decimal(30,8)) + outputColumnNames: _col0, _col1 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [3, 8] + selectExpressions: DecimalColMultiplyDecimalColumn(col 2:decimal(26,6), col 9:decimal(3,2)/DECIMAL_64)(children: ConvertDecimal64ToDecimal(col 7:decimal(3,2)/DECIMAL_64)(children: IfExprDecimal64ScalarDecimal64Scalar(col 6:boolean, decimal64Val1 177, decimalVal1 1.77, decimal64Val2 72, decimalVal2 0.72)(children: StringGroupColEqualVarCharScalar(col 1:varchar(28), val bc) -> 6:boolean) -> 7:decimal(3,2)/DECIMAL_64) -> 9:decimal(3,2)) -> 8:decimal(30,8) + Statistics: Num rows: 1 Data size: 199 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: trim(_col0) (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: trim(_col0) (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkStringOperator + keyColumns: 10:string + keyExpressions: StringTrimCol(col 3:varchar(28)) -> 10:string + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + valueColumns: 8:decimal(30,8) + Statistics: Num rows: 1 Data size: 199 Basic stats: COMPLETE Column stats: COMPLETE + value expressions: _col1 (type: decimal(30,8)) + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 4 + includeColumns: [1, 2, 3] + dataColumns: col1:varchar(28), col2:varchar(28), col3:decimal(26,6), col4:varchar(28) + partitionColumnCount: 0 + scratchColumnTypeNames: [bigint, decimal(3,2)/DECIMAL_64, decimal(30,8), decimal(3,2), string] + Map 3 + Map Operator Tree: + TableScan + alias: test2 + filterExpr: trim(col4) is not null (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + TableScan Vectorization: + native: true + vectorizationSchemaColumns: [0:col1:varchar(28), 1:col2:varchar(28), 2:col3:decimal(26,6), 3:col4:varchar(28), 4:col5:varchar(28), 5:ROW__ID:struct, 6:ROW__IS__DELETED:boolean] + Filter Operator + Filter Vectorization: + className: VectorFilterOperator + native: true + predicateExpression: SelectColumnIsNotNull(col 7:string)(children: StringTrimCol(col 3:varchar(28)) -> 7:string) + predicate: trim(col4) is not null (type: boolean) + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: col4 (type: varchar(28)) + outputColumnNames: _col0 + Select Vectorization: + className: VectorSelectOperator + native: true + projectedOutputColumnNums: [3] + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Reduce Output Operator + key expressions: trim(_col0) (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: trim(_col0) (type: string) + Reduce Sink Vectorization: + className: VectorReduceSinkStringOperator + keyColumns: 7:string + keyExpressions: StringTrimCol(col 3:varchar(28)) -> 7:string + native: true + nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true + Statistics: Num rows: 1 Data size: 87 Basic stats: COMPLETE Column stats: COMPLETE + Execution mode: vectorized, llap + LLAP IO: all inputs + Map Vectorization: + enabled: true + enabledConditionsMet: hive.vectorized.use.vector.serde.deserialize IS true + inputFormatFeatureSupport: [DECIMAL_64] + featureSupportInUse: [DECIMAL_64] + inputFileFormats: org.apache.hadoop.mapred.TextInputFormat + allNative: true + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 5 + includeColumns: [3] + dataColumns: col1:varchar(28), col2:varchar(28), col3:decimal(26,6), col4:varchar(28), col5:varchar(28) + partitionColumnCount: 0 + scratchColumnTypeNames: [string] + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Merge Join Operator + condition map: + Left Outer Join 0 to 1 + keys: + 0 trim(_col0) (type: string) + 1 trim(_col0) (type: string) + outputColumnNames: _col1 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: _col1 (type: decimal(30,8)) + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + MergeJoin Vectorization: + enabled: false + enableConditionsNotMet: Vectorizing MergeJoin Supported IS false + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: SELECT + int_cost +FROM + ( + SELECT + a.col4, + a.col3 * case when a.col2 = 'bc' then 1.77 else 0.72 end + AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + ) bb ON trim(aa.col4) = trim(bb.col4) +PREHOOK: type: QUERY +PREHOOK: Input: default@test1 +PREHOOK: Input: default@test2 +#### A masked pattern was here #### +POSTHOOK: query: SELECT + int_cost +FROM + ( + SELECT + a.col4, + a.col3 * case when a.col2 = 'bc' then 1.77 else 0.72 end + AS int_cost + FROM + test1 a + ) aa + LEFT JOIN ( + SELECT + col4 + FROM + test2 + ) bb ON trim(aa.col4) = trim(bb.col4) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test1 +POSTHOOK: Input: default@test2 +#### A masked pattern was here #### +0.72000000 +PREHOOK: query: DROP TABLE test1 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test1 +PREHOOK: Output: database:default +PREHOOK: Output: default@test1 +POSTHOOK: query: DROP TABLE test1 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test1 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test1 +PREHOOK: query: DROP TABLE test2 +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@test2 +PREHOOK: Output: database:default +PREHOOK: Output: default@test2 +POSTHOOK: query: DROP TABLE test2 +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@test2 +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test2