Skip to content

[WIP] HIVE-29262: Incorrect column ordering output in case of different ord… #6512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tanishq-chugh wants to merge 1 commit into
base: master
Choose a base branch
from
+14,490 −4
Open

[WIP] HIVE-29262: Incorrect column ordering output in case of different ord… #6512

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5113,7 +5113,7 @@
* Create the additional vectorization PTF information needed by the VectorPTFOperator during
* execution.
*/
private static VectorPTFInfo createVectorPTFInfo(Operator<? extends OperatorDesc> ptfOp,

Check failure on line 5116 in ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 19 to the 15 allowed. 


See more on https://sonarcloud.io/project/issues?id=apache_hive&issues=AZ5qZJevokFn1UQ2ggY6&open=AZ5qZJevokFn1UQ2ggY6&pullRequest=6512
PTFDesc ptfDesc, VectorizationContext vContext, VectorPTFDesc vectorPTFDesc)
throws HiveException {

Expand Down Expand Up @@ -5161,13 +5161,11 @@
partitionColumnVectorTypes = new Type[partitionKeyCount];
partitionExpressions = new VectorExpression[partitionKeyCount];

Map<Integer, VectorExpression> partitionExpressionByOutputColumn = new HashMap<>();
for (int i = 0; i < partitionKeyCount; i++) {
VectorExpression partitionExpression = vContext.getVectorExpression(partitionExprNodeDescs[i]);
TypeInfo typeInfo = partitionExpression.getOutputTypeInfo();
Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
partitionColumnVectorTypes[i] = columnVectorType;
partitionColumnMap[i] = partitionExpression.getOutputColumnNum();
partitionExpressions[i] = partitionExpression;
partitionExpressionByOutputColumn.put(partitionColumnMap[i], partitionExpression);
}

final int orderKeyCount = orderExprNodeDescs.length;
Expand All @@ -5190,6 +5188,23 @@
int[] keyInputColumnMap = ArrayUtils.toPrimitive(keyInputColumns.toArray(new Integer[0]));
int[] nonKeyInputColumnMap = ArrayUtils.toPrimitive(nonKeyInputColumns.toArray(new Integer[0]));

int partitionIndex = 0;
for (int keyCol : keyInputColumnMap) {
VectorExpression partitionExpression = partitionExpressionByOutputColumn.get(keyCol);
if (partitionExpression == null) {
continue;
}
TypeInfo typeInfo = partitionExpression.getOutputTypeInfo();
Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
partitionColumnVectorTypes[partitionIndex] = columnVectorType;
partitionColumnMap[partitionIndex] = keyCol;
partitionExpressions[partitionIndex] = partitionExpression;
partitionIndex++;
}
if (partitionIndex != partitionKeyCount) {
throw new HiveException("Failed to map partition columns to key input column order");
}

VectorExpression[][] evaluatorInputExpressions = new VectorExpression[evaluatorCount][];
Type[][] evaluatorInputColumnVectorTypes = new Type[evaluatorCount][];
for (int i = 0; i < evaluatorCount; i++) {
Expand Down
149 changes: 149 additions & 0 deletions ql/src/test/queries/clientpositive/lead_vec.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
SET hive.vectorized.execution.enabled=true;
create table web_sales_txt
(
ws_sold_date_sk int,
ws_sold_time_sk int,
ws_ship_date_sk int,
ws_item_sk int,
ws_bill_customer_sk int,
ws_bill_cdemo_sk int,
ws_bill_hdemo_sk int,
ws_bill_addr_sk int,
ws_ship_customer_sk int,
ws_ship_cdemo_sk int,
ws_ship_hdemo_sk int,
ws_ship_addr_sk int,
ws_web_page_sk int,
ws_web_site_sk int,
ws_ship_mode_sk int,
ws_warehouse_sk int,
ws_promo_sk int,
ws_order_number int,
ws_quantity int,
ws_wholesale_cost decimal(7,2),
ws_list_price decimal(7,2),
ws_sales_price decimal(7,2),
ws_ext_discount_amt decimal(7,2),
ws_ext_sales_price decimal(7,2),
ws_ext_wholesale_cost decimal(7,2),
ws_ext_list_price decimal(7,2),
ws_ext_tax decimal(7,2),
ws_coupon_amt decimal(7,2),
ws_ext_ship_cost decimal(7,2),
ws_net_paid decimal(7,2),
ws_net_paid_inc_tax decimal(7,2),
ws_net_paid_inc_ship decimal(7,2),
ws_net_paid_inc_ship_tax decimal(7,2),
ws_net_profit decimal(7,2)
)
row format delimited fields terminated by '|'
stored as textfile;

LOAD DATA LOCAL INPATH '../../data/files/web_sales_2k' OVERWRITE INTO TABLE web_sales_txt;
select ws_bill_customer_sk,ws_item_sk from web_sales_txt;

SET hive.vectorized.execution.enabled;
SELECT
ws_bill_customer_sk,
ws_item_sk,
ws_sold_date_sk,
ws_sales_price,
LAG(ws_sales_price) OVER (
PARTITION BY ws_item_sk,ws_bill_customer_sk
ORDER BY ws_sold_date_sk
) AS prev_sales_price,
ws_sales_price - LAG(ws_sales_price) OVER (
PARTITION BY ws_bill_customer_sk, ws_item_sk
ORDER BY ws_sold_date_sk
) AS sales_price_diff
FROM
web_sales_txt;

SELECT
ws_bill_customer_sk,
ws_item_sk,
ws_sold_date_sk,
ws_sales_price,
LEAD(ws_sales_price) OVER (
PARTITION BY ws_item_sk, ws_bill_customer_sk
ORDER BY ws_sold_date_sk
) AS next_sales_price,
LEAD(ws_sales_price) OVER (
PARTITION BY ws_bill_customer_sk, ws_item_sk
ORDER BY ws_sold_date_sk
) - ws_sales_price AS sales_price_diff
FROM
web_sales_txt;



SELECT
ws_bill_customer_sk,
ws_item_sk,
ws_sold_date_sk,
ws_sales_price,
FIRST_VALUE(ws_sales_price) OVER (
PARTITION BY ws_item_sk,ws_bill_customer_sk
ORDER BY ws_sold_date_sk
) AS first_price,
LAST_VALUE(ws_sales_price) OVER (
PARTITION BY ws_bill_customer_sk, ws_item_sk
ORDER BY ws_sold_date_sk
ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
) AS last_price
FROM
web_sales_txt;

SET hive.vectorized.execution.enabled=false;

SET hive.vectorized.execution.enabled;
SELECT
ws_bill_customer_sk,
ws_item_sk,
ws_sold_date_sk,
ws_sales_price,
LAG(ws_sales_price) OVER (
PARTITION BY ws_item_sk,ws_bill_customer_sk
ORDER BY ws_sold_date_sk
) AS prev_sales_price,
ws_sales_price - LAG(ws_sales_price) OVER (
PARTITION BY ws_bill_customer_sk, ws_item_sk
ORDER BY ws_sold_date_sk
) AS sales_price_diff
FROM
web_sales_txt;

SELECT
ws_bill_customer_sk,
ws_item_sk,
ws_sold_date_sk,
ws_sales_price,
LEAD(ws_sales_price) OVER (
PARTITION BY ws_item_sk, ws_bill_customer_sk
ORDER BY ws_sold_date_sk
) AS next_sales_price,
LEAD(ws_sales_price) OVER (
PARTITION BY ws_bill_customer_sk, ws_item_sk
ORDER BY ws_sold_date_sk
) - ws_sales_price AS sales_price_diff
FROM
web_sales_txt;



SELECT
ws_bill_customer_sk,
ws_item_sk,
ws_sold_date_sk,
ws_sales_price,
FIRST_VALUE(ws_sales_price) OVER (
PARTITION BY ws_item_sk,ws_bill_customer_sk
ORDER BY ws_sold_date_sk
) AS first_price,
LAST_VALUE(ws_sales_price) OVER (
PARTITION BY ws_bill_customer_sk, ws_item_sk
ORDER BY ws_sold_date_sk
ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
) AS last_price
FROM
web_sales_txt;
Loading
Loading