diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java index cd1b90fb923316..7843a7ac55b10e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/CloudSchemaChangeJobV2.java @@ -34,21 +34,27 @@ import org.apache.doris.cloud.qe.ComputeGroupException; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.proto.OlapFile; import org.apache.doris.qe.ConnectContext; import org.apache.doris.service.FrontendOptions; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TTaskType; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Table; import com.google.common.collect.Table.Cell; +import com.google.gson.annotations.SerializedName; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -58,6 +64,9 @@ public class CloudSchemaChangeJobV2 extends SchemaChangeJobV2 { private static final Logger LOG = LogManager.getLogger(SchemaChangeJobV2.class); + @SerializedName("fsv") + private Map baseShadowSchemaVersionByFormat = new HashMap<>(); + public CloudSchemaChangeJobV2(String rawSql, long jobId, long dbId, long tableId, String tableName, long timeoutMs) { super(rawSql, jobId, dbId, tableId, tableName, timeoutMs); @@ -79,6 +88,28 @@ public CloudSchemaChangeJobV2(String rawSql, long jobId, long dbId, long tableId private CloudSchemaChangeJobV2() {} + public void configureBaseShadowSchemaVersion(OlapTable table) throws DdlException { + long baseShadowIndexId = getBaseShadowIndexId(table.getBaseIndexId()); + if (baseShadowIndexId == -1) { + return; + } + int tableBaseSchemaVersion = table.getIndexMetaByIndexId(table.getBaseIndexId()).getSchemaVersion(); + baseShadowSchemaVersionByFormat = getBaseShadowSchemaVersions(table, tableBaseSchemaVersion); + Integer currentSchemaVersion = baseShadowSchemaVersionByFormat.get( + table.getPartitionInvertedIndexFileStorageFormat()); + Preconditions.checkState(currentSchemaVersion != null); + indexSchemaVersionAndHashMap.get(baseShadowIndexId).schemaVersion = currentSchemaVersion; + } + + public Map getBaseShadowSchemaVersionByFormat() { + return baseShadowSchemaVersionByFormat; + } + + public void setBaseShadowSchemaVersionByFormat( + Map baseShadowSchemaVersionByFormat) { + this.baseShadowSchemaVersionByFormat = baseShadowSchemaVersionByFormat; + } + @Override protected void commitShadowIndex() throws AlterCancelException { List shadowIdxList = @@ -227,16 +258,18 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio for (Map.Entry entry : shadowIndexMap.entrySet()) { long shadowIdxId = entry.getKey(); MaterializedIndex shadowIdx = entry.getValue(); - + boolean isBaseShadowIndex = isShadowIndexOfBase(shadowIdxId, tbl); short shadowShortKeyColumnCount = indexShortKeyMap.get(shadowIdxId); List shadowSchema = indexSchemaMap.get(shadowIdxId); - List clusterKeyUids = null; - if (shadowIdxId == tbl.getBaseIndexId() || isShadowIndexOfBase(shadowIdxId, tbl)) { - clusterKeyUids = OlapTable.getClusterKeyUids(shadowSchema); - } int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash; int shadowSchemaVersion = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaVersion; long originIndexId = indexIdMap.get(shadowIdxId); + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat = isBaseShadowIndex + ? tbl.getInvertedIndexFileStorageFormatForPartition(partitionId) : null; + int schemaVersion = isBaseShadowIndex + ? getBaseShadowSchemaVersion(invertedIndexFileStorageFormat) : shadowSchemaVersion; + List clusterKeyUids = isBaseShadowIndex + ? OlapTable.getClusterKeyUids(shadowSchema) : null; KeysType originKeysType = tbl.getKeysTypeByIndexId(originIndexId); List tabletIndexes = originIndexId == tbl.getBaseIndexId() ? indexes : null; @@ -255,7 +288,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.getStoragePolicy(), tbl.isInMemory(), true, tbl.getName(), tbl.getTTLSeconds(), tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), - shadowSchemaVersion, tbl.getCompactionPolicy(), + schemaVersion, tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), tbl.getTimeSeriesCompactionTimeThresholdSeconds(), @@ -263,7 +296,7 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), - tbl.getInvertedIndexFileStorageFormat(), + invertedIndexFileStorageFormat, tbl.rowStorePageSize(), tbl.variantEnableFlattenNested(), clusterKeyUids, tbl.storagePageSize(), tbl.getTDEAlgorithmPB(), @@ -279,6 +312,58 @@ private void createShadowIndexReplicaForPartition(OlapTable tbl) throws Exceptio } } + private long getBaseShadowIndexId(long baseIndexId) { + for (Map.Entry entry : indexIdMap.entrySet()) { + if (entry.getValue() == baseIndexId) { + return entry.getKey(); + } + } + return -1; + } + + private int getBaseShadowSchemaVersion(TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) { + Preconditions.checkState(invertedIndexFileStorageFormat != null, + "missing inverted index storage format for base partition"); + Integer schemaVersion = baseShadowSchemaVersionByFormat.get(invertedIndexFileStorageFormat); + Preconditions.checkState(schemaVersion != null, + "missing shadow schema version for format " + invertedIndexFileStorageFormat); + return schemaVersion; + } + + private Map getBaseShadowSchemaVersions(OlapTable table, + int tableBaseSchemaVersion) throws DdlException { + TInvertedIndexFileStorageFormat currentFormat = table.getPartitionInvertedIndexFileStorageFormat(); + Set formats = new HashSet<>(); + for (long partitionId : partitionIndexMap.rowKeySet()) { + TInvertedIndexFileStorageFormat format = table.getInvertedIndexFileStorageFormatForPartition(partitionId); + if (format == null) { + throw new DdlException("missing inverted index storage format for partition " + partitionId); + } + formats.add(format); + } + formats.add(currentFormat); + + List orderedFormats = new ArrayList<>(formats); + orderedFormats.sort((left, right) -> { + if (left == right) { + return 0; + } + if (left == currentFormat) { + return 1; + } + if (right == currentFormat) { + return -1; + } + return left.name().compareTo(right.name()); + }); + + Map shadowSchemaVersions = new HashMap<>(); + for (int i = 0; i < orderedFormats.size(); ++i) { + shadowSchemaVersions.put(orderedFormats.get(i), tableBaseSchemaVersion + i + 1); + } + return shadowSchemaVersions; + } + @Override protected void ensureCloudClusterExist(List tasks) throws AlterCancelException { if (((CloudSystemInfoService) Env.getCurrentSystemInfo()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java index 9ad1594bc4c0a2..7b4a687a4c607d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java @@ -32,6 +32,7 @@ import org.apache.doris.persist.RecoverInfo; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TStorageMedium; import com.google.common.base.Preconditions; @@ -234,9 +235,16 @@ public boolean recycleTable(long dbId, Table table, boolean isReplay, } public boolean recyclePartition(long dbId, long tableId, String tableName, Partition partition, - Range range, PartitionItem listPartitionItem, - DataProperty dataProperty, ReplicaAllocation replicaAlloc, - boolean isInMemory, boolean isMutable) { + Range range, PartitionItem listPartitionItem, DataProperty dataProperty, + ReplicaAllocation replicaAlloc, boolean isInMemory, boolean isMutable) { + return recyclePartition(dbId, tableId, tableName, partition, range, listPartitionItem, dataProperty, + replicaAlloc, isInMemory, isMutable, null); + } + + public boolean recyclePartition(long dbId, long tableId, String tableName, Partition partition, + Range range, PartitionItem listPartitionItem, DataProperty dataProperty, + ReplicaAllocation replicaAlloc, boolean isInMemory, boolean isMutable, + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) { writeLock(); try { if (idToPartition.containsKey(partition.getId())) { @@ -246,7 +254,8 @@ public boolean recyclePartition(long dbId, long tableId, String tableName, Parti // recycle partition RecyclePartitionInfo partitionInfo = new RecyclePartitionInfo(dbId, tableId, partition, - range, listPartitionItem, dataProperty, replicaAlloc, isInMemory, isMutable); + range, listPartitionItem, dataProperty, replicaAlloc, isInMemory, isMutable, + invertedIndexFileStorageFormat); idToRecycleTime.put(partition.getId(), System.currentTimeMillis()); idToPartition.put(partition.getId(), partitionInfo); dbTblIdPartitionNameToIds.computeIfAbsent(Pair.of(dbId, tableId), k -> new ConcurrentHashMap<>()) @@ -1051,7 +1060,10 @@ public void recoverPartition(long dbId, OlapTable table, String partitionName, partitionInfo.setReplicaAllocation(partitionId, recoverPartitionInfo.getReplicaAlloc()); partitionInfo.setIsInMemory(partitionId, recoverPartitionInfo.isInMemory()); partitionInfo.setIsMutable(partitionId, recoverPartitionInfo.isMutable()); - + if (recoverPartitionInfo.getInvertedIndexFileStorageFormat() != null) { + partitionInfo.setInvertedIndexFileStorageFormat(partitionId, + recoverPartitionInfo.getInvertedIndexFileStorageFormat()); + } // remove from recycle bin idToPartition.remove(partitionId); idToRecycleTime.remove(partitionId); @@ -1117,6 +1129,10 @@ public void replayRecoverPartition(OlapTable table, long partitionId, partitionInfo.setReplicaAllocation(partitionId, recyclePartitionInfo.getReplicaAlloc()); partitionInfo.setIsInMemory(partitionId, recyclePartitionInfo.isInMemory()); partitionInfo.setIsMutable(partitionId, recyclePartitionInfo.isMutable()); + if (recyclePartitionInfo.getInvertedIndexFileStorageFormat() != null) { + partitionInfo.setInvertedIndexFileStorageFormat(partitionId, + recyclePartitionInfo.getInvertedIndexFileStorageFormat()); + } iterator.remove(); idToRecycleTime.remove(partitionId); @@ -1767,6 +1783,8 @@ public class RecyclePartitionInfo { private boolean isInMemory; @SerializedName("mu") private boolean isMutable = true; + @SerializedName("iifsf") + private TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat; public RecyclePartitionInfo() { // for persist @@ -1775,7 +1793,8 @@ public RecyclePartitionInfo() { public RecyclePartitionInfo(long dbId, long tableId, Partition partition, Range range, PartitionItem listPartitionItem, DataProperty dataProperty, ReplicaAllocation replicaAlloc, - boolean isInMemory, boolean isMutable) { + boolean isInMemory, boolean isMutable, + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) { this.dbId = dbId; this.tableId = tableId; this.partition = partition; @@ -1785,6 +1804,7 @@ public RecyclePartitionInfo(long dbId, long tableId, Partition partition, this.replicaAlloc = replicaAlloc; this.isInMemory = isInMemory; this.isMutable = isMutable; + this.invertedIndexFileStorageFormat = invertedIndexFileStorageFormat; } public long getDbId() { @@ -1823,6 +1843,10 @@ public boolean isMutable() { return isMutable; } + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { + return invertedIndexFileStorageFormat; + } + public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index dc8cac76500b15..b7edfab3684e0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -295,6 +295,7 @@ import org.apache.doris.thrift.TGetMetaResult; import org.apache.doris.thrift.TGetMetaTableMeta; import org.apache.doris.thrift.TGetMetaTabletMeta; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TStatus; import org.apache.doris.thrift.TStatusCode; @@ -3812,6 +3813,12 @@ public void addPartitionLike(Database db, String tableName, AddPartitionLikeOp a getInternalCatalog().addPartitionLike(db, tableName, addPartitionLikeOp); } + public void addPartitionLikeForInsertOverwrite(Database db, String tableName, + AddPartitionLikeOp addPartitionLikeOp) + throws DdlException { + getInternalCatalog().addPartitionLikeForInsertOverwrite(db, tableName, addPartitionLikeOp); + } + public void replayAddPartition(PartitionPersistInfo info) throws MetaNotFoundException { getInternalCatalog().replayAddPartition(info); } @@ -4021,6 +4028,13 @@ private static void addOlapTablePropertyInfo(OlapTable olapTable, StringBuilder // inverted index storage type sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT).append("\" = \""); sb.append(olapTable.getInvertedIndexFileStorageFormat()).append("\""); + if (olapTable.getTableProperty() != null + && olapTable.getTableProperty().getPartitionInvertedIndexFileStorageFormat() != null) { + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT) + .append("\" = \"") + .append(olapTable.getTableProperty().getPartitionInvertedIndexFileStorageFormat()) + .append("\""); + } // compression type if (olapTable.getCompressionType() != TCompressionType.valueOf(Config.default_compression_type)) { @@ -6346,6 +6360,19 @@ public void modifyTableDefaultReplicaAllocation(Database db, OlapTable table, // The caller need to hold the table write lock public void modifyTableProperties(Database db, OlapTable table, Map properties) { + modifyTableProperties(db, table, properties, new HashMap<>()); + } + + // The caller need to hold the table write lock + public void modifyTableProperties(Database db, OlapTable table, Map properties, + Map indexIdToSchemaVersion) { + modifyTableProperties(db, table, properties, indexIdToSchemaVersion, new HashMap<>()); + } + + // The caller need to hold the table write lock + public void modifyTableProperties(Database db, OlapTable table, Map properties, + Map indexIdToSchemaVersion, + Map partitionIdToInvertedIndexFileStorageFormat) { Preconditions.checkArgument(table.isWriteLockHeldByCurrentThread()); TableProperty tableProperty = table.getTableProperty(); if (tableProperty == null) { @@ -6377,9 +6404,17 @@ public void modifyTableProperties(Database db, OlapTable table, Map entry : indexIdToSchemaVersion.entrySet()) { + table.getIndexMetaByIndexId(entry.getKey()).setSchemaVersion(entry.getValue()); + } + for (Map.Entry entry + : partitionIdToInvertedIndexFileStorageFormat.entrySet()) { + table.getPartitionInfo().setInvertedIndexFileStorageFormat(entry.getKey(), entry.getValue()); + } + ModifyTablePropertyOperationLog info = new ModifyTablePropertyOperationLog(db.getId(), table.getId(), table.getName(), - properties); + properties, indexIdToSchemaVersion, partitionIdToInvertedIndexFileStorageFormat); editLog.logModifyTableProperties(info); } @@ -6418,6 +6453,21 @@ public void replayModifyTableProperty(short opCode, ModifyTablePropertyOperation tableProperty.buildProperty(opCode); } + Map indexIdToSchemaVersion = info.getIndexIdToSchemaVersion(); + if (indexIdToSchemaVersion != null) { + for (Map.Entry entry : indexIdToSchemaVersion.entrySet()) { + olapTable.getIndexMetaByIndexId(entry.getKey()).setSchemaVersion(entry.getValue()); + } + } + Map partitionIdToInvertedIndexFileStorageFormat = + info.getPartitionIdToInvertedIndexFileStorageFormat(); + if (partitionIdToInvertedIndexFileStorageFormat != null) { + for (Map.Entry entry + : partitionIdToInvertedIndexFileStorageFormat.entrySet()) { + olapTable.getPartitionInfo().setInvertedIndexFileStorageFormat(entry.getKey(), entry.getValue()); + } + } + // need to replay partition info meta switch (opCode) { case OperationType.OP_MODIFY_TABLE_PROPERTIES: diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index bcd9e76e7c434c..938b9155e0da92 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1366,7 +1366,8 @@ private void dropPartitionCommon(long dbId, boolean isForceDrop, recyclePartitionParam.dataProperty, recyclePartitionParam.replicaAlloc, recyclePartitionParam.isInMemory, - recyclePartitionParam.isMutable); + recyclePartitionParam.isMutable, + recyclePartitionParam.invertedIndexFileStorageFormat); } else if (partitionInfo.getType() == PartitionType.LIST) { // construct a dummy range @@ -1386,7 +1387,8 @@ private void dropPartitionCommon(long dbId, boolean isForceDrop, recyclePartitionParam.dataProperty, recyclePartitionParam.replicaAlloc, recyclePartitionParam.isInMemory, - recyclePartitionParam.isMutable); + recyclePartitionParam.isMutable, + recyclePartitionParam.invertedIndexFileStorageFormat); } else { // unpartition // construct a dummy range and dummy list. @@ -1405,7 +1407,8 @@ private void dropPartitionCommon(long dbId, boolean isForceDrop, recyclePartitionParam.dataProperty, recyclePartitionParam.replicaAlloc, recyclePartitionParam.isInMemory, - recyclePartitionParam.isMutable); + recyclePartitionParam.isMutable, + recyclePartitionParam.invertedIndexFileStorageFormat); } } else if (!reserveTablets) { Env.getCurrentEnv().onErasePartition(partition); @@ -2208,6 +2211,8 @@ public void fillInfo(Partition partition, RecyclePartitionParam recyclePartition recyclePartitionParam.replicaAlloc = partitionInfo.getReplicaAllocation(partition.getId()); recyclePartitionParam.isInMemory = partitionInfo.getIsInMemory(partition.getId()); recyclePartitionParam.isMutable = partitionInfo.getIsMutable(partition.getId()); + recyclePartitionParam.invertedIndexFileStorageFormat = + partitionInfo.getInvertedIndexFileStorageFormat(partition.getId()); recyclePartitionParam.partitionItem = partitionInfo.getItem(partition.getId()); recyclePartitionParam.partition = partition; } @@ -2230,10 +2235,13 @@ public Partition replacePartition(Partition newPartition, ReplicaAllocation replicaAlloc = partitionInfo.getReplicaAllocation(oldPartition.getId()); boolean isInMemory = partitionInfo.getIsInMemory(oldPartition.getId()); boolean isMutable = partitionInfo.getIsMutable(oldPartition.getId()); + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat = + partitionInfo.getInvertedIndexFileStorageFormat(oldPartition.getId()); recyclePartitionParam.dataProperty = dataProperty; recyclePartitionParam.replicaAlloc = replicaAlloc; recyclePartitionParam.isInMemory = isInMemory; recyclePartitionParam.isMutable = isMutable; + recyclePartitionParam.invertedIndexFileStorageFormat = invertedIndexFileStorageFormat; recyclePartitionParam.partitionItem = partitionInfo.getItem(oldPartition.getId()); recyclePartitionParam.partition = oldPartition; @@ -2247,7 +2255,10 @@ public Partition replacePartition(Partition newPartition, partitionInfo.dropPartition(oldPartition.getId()); partitionInfo.addPartition(newPartition.getId(), dataProperty, replicaAlloc, isInMemory, isMutable); } - + if (invertedIndexFileStorageFormat != null) { + partitionInfo.setInvertedIndexFileStorageFormat(newPartition.getId(), + invertedIndexFileStorageFormat); + } return oldPartition; } @@ -3176,6 +3187,13 @@ public void setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat in tableProperty.buildInvertedIndexFileStorageFormat(); } + public void setPartitionInvertedIndexFileStorageFormat( + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) { + getOrCreatTableProperty().modifyTableProperties( + PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, + invertedIndexFileStorageFormat.name()); + } + public TStorageFormat getStorageFormat() { if (tableProperty == null) { return TStorageFormat.DEFAULT; @@ -3190,6 +3208,28 @@ public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { return tableProperty.getInvertedIndexFileStorageFormat(); } + public TInvertedIndexFileStorageFormat getPartitionInvertedIndexFileStorageFormat() { + TInvertedIndexFileStorageFormat format = tableProperty == null + ? getInvertedIndexFileStorageFormat() + : tableProperty.getPartitionInvertedIndexFileStorageFormat(); + if (format == null) { + format = getInvertedIndexFileStorageFormat(); + } + return format == TInvertedIndexFileStorageFormat.DEFAULT + ? PropertyAnalyzer.getDefaultInvertedIndexFileStorageFormat() : format; + } + + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormatForPartition(long partitionId) { + TInvertedIndexFileStorageFormat format = partitionInfo.getInvertedIndexFileStorageFormat(partitionId); + if (format != null) { + return format; + } + if (tableProperty == null || tableProperty.getPartitionInvertedIndexFileStorageFormat() == null) { + return getInvertedIndexFileStorageFormat(); + } + return null; + } + public TCompressionType getCompressionType() { if (tableProperty == null) { return TCompressionType.LZ4F; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index 5cec453baac077..1883223238d647 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -27,6 +27,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TTabletType; @@ -69,6 +70,9 @@ public class PartitionInfo { // partition id -> storage policy @SerializedName("IdToStoragePolicy") protected Map idToStoragePolicy; + // partition id -> resolved base inverted-index storage format + @SerializedName("IdToInvertedIndexFileStorageFormat") + protected Map idToInvertedIndexFileStorageFormat; // partition id -> replication allocation @SerializedName("IdToReplicaAllocation") protected Map idToReplicaAllocation; @@ -98,6 +102,7 @@ public PartitionInfo() { this.idToInMemory = new HashMap<>(); this.idToTabletType = new HashMap<>(); this.idToStoragePolicy = new HashMap<>(); + this.idToInvertedIndexFileStorageFormat = new HashMap<>(); this.partitionExprs = new ArrayList<>(); } @@ -108,6 +113,7 @@ public PartitionInfo(PartitionType type) { this.idToInMemory = new HashMap<>(); this.idToTabletType = new HashMap<>(); this.idToStoragePolicy = new HashMap<>(); + this.idToInvertedIndexFileStorageFormat = new HashMap<>(); this.partitionExprs = new ArrayList<>(); } @@ -312,6 +318,19 @@ public void setStoragePolicy(long partitionId, String storagePolicy) { idToStoragePolicy.put(partitionId, storagePolicy); } + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat(long partitionId) { + return idToInvertedIndexFileStorageFormat == null + ? null : idToInvertedIndexFileStorageFormat.get(partitionId); + } + + public void setInvertedIndexFileStorageFormat(long partitionId, + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) { + if (idToInvertedIndexFileStorageFormat == null) { + idToInvertedIndexFileStorageFormat = new HashMap<>(); + } + idToInvertedIndexFileStorageFormat.put(partitionId, invertedIndexFileStorageFormat); + } + public Map getPartitionReplicaAllocations() { return idToReplicaAllocation; } @@ -361,6 +380,9 @@ public void dropPartition(long partitionId) { idToDataProperty.remove(partitionId); idToReplicaAllocation.remove(partitionId); idToInMemory.remove(partitionId); + if (idToInvertedIndexFileStorageFormat != null) { + idToInvertedIndexFileStorageFormat.remove(partitionId); + } idToItem.remove(partitionId); idToTempItem.remove(partitionId); } @@ -421,11 +443,14 @@ public void resetPartitionIdForRestore( Map origIdToItem = idToItem; Map origIdToInMemory = idToInMemory; Map origIdToStoragePolicy = idToStoragePolicy; + Map origIdToInvertedIndexFileStorageFormat = + idToInvertedIndexFileStorageFormat; idToDataProperty = Maps.newHashMap(); idToReplicaAllocation = Maps.newHashMap(); idToItem = Maps.newHashMap(); idToInMemory = Maps.newHashMap(); idToStoragePolicy = Maps.newHashMap(); + idToInvertedIndexFileStorageFormat = Maps.newHashMap(); for (Map.Entry entry : partitionIdMap.entrySet()) { long newPartId = entry.getKey(); @@ -455,6 +480,13 @@ public void resetPartitionIdForRestore( if (!isSinglePartitioned) { idToItem.put(newPartId, origIdToItem.get(origPartId)); } + if (origIdToInvertedIndexFileStorageFormat != null) { + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat = + origIdToInvertedIndexFileStorageFormat.get(origPartId); + if (invertedIndexFileStorageFormat != null) { + idToInvertedIndexFileStorageFormat.put(newPartId, invertedIndexFileStorageFormat); + } + } } } @@ -493,6 +525,7 @@ public boolean equals(Object o) { partitionColumns, that.partitionColumns) && Objects.equals(idToItem, that.idToItem) && Objects.equals(idToTempItem, that.idToTempItem) && Objects.equals(idToDataProperty, that.idToDataProperty) && Objects.equals(idToStoragePolicy, that.idToStoragePolicy) + && Objects.equals(idToInvertedIndexFileStorageFormat, that.idToInvertedIndexFileStorageFormat) && Objects.equals(idToReplicaAllocation, that.idToReplicaAllocation) && Objects.equals( idToInMemory, that.idToInMemory) && Objects.equals(idToTabletType, that.idToTabletType) && Objects.equals(partitionExprs, that.partitionExprs); @@ -501,6 +534,7 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash(type, partitionColumns, idToItem, idToTempItem, idToDataProperty, idToStoragePolicy, - idToReplicaAllocation, isMultiColumnPartition, idToInMemory, idToTabletType, partitionExprs); + idToInvertedIndexFileStorageFormat, idToReplicaAllocation, isMultiColumnPartition, idToInMemory, + idToTabletType, partitionExprs); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java index 51f38638938db5..3a35b4ecd6fc26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java @@ -17,6 +17,8 @@ package org.apache.doris.catalog; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; + public class RecyclePartitionParam { public Partition partition; public PartitionItem partitionItem; @@ -24,6 +26,7 @@ public class RecyclePartitionParam { public ReplicaAllocation replicaAlloc; public boolean isInMemory; public boolean isMutable = true; + public TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat; public RecyclePartitionParam() { // do nothing. diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java index 5a8455726b9cff..df3aa3376dc9a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java @@ -761,6 +761,11 @@ public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { return invertedIndexFileStorageFormat; } + public TInvertedIndexFileStorageFormat getPartitionInvertedIndexFileStorageFormat() { + String format = properties.get(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT); + return format == null ? null : TInvertedIndexFileStorageFormat.valueOf(format); + } + public DataSortInfo getDataSortInfo() { return dataSortInfo; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java index 47e6fbf0cbeab3..60a1b9e7d8730f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/alter/CloudSchemaChangeHandler.java @@ -17,6 +17,8 @@ package org.apache.doris.cloud.alter; +import org.apache.doris.alter.AlterJobV2; +import org.apache.doris.alter.CloudSchemaChangeJobV2; import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; @@ -28,6 +30,7 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.Tablet; +import org.apache.doris.cloud.datasource.CloudInternalCatalog; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.AnalysisException; @@ -38,6 +41,7 @@ import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.service.FrontendOptions; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -45,6 +49,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -54,6 +59,22 @@ public class CloudSchemaChangeHandler extends SchemaChangeHandler { private static final Logger LOG = LogManager.getLogger(CloudSchemaChangeHandler.class); + @Override + public void addAlterJobV2(AlterJobV2 alterJob) throws AnalysisException { + if (alterJob instanceof CloudSchemaChangeJobV2) { + Database db = Env.getCurrentInternalCatalog().getDbNullable(alterJob.getDbId()); + Preconditions.checkNotNull(db, alterJob.getDbId()); + Table table = db.getTableNullable(alterJob.getTableId()); + Preconditions.checkState(table instanceof OlapTable, alterJob.getTableId()); + try { + ((CloudSchemaChangeJobV2) alterJob).configureBaseShadowSchemaVersion((OlapTable) table); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage()); + } + } + super.addAlterJobV2(alterJob); + } + @Override public void updatePartitionsProperties(Database db, String tableName, List partitionNames, Map properties) throws DdlException, MetaNotFoundException { @@ -121,6 +142,7 @@ public void updateTableProperties(Database db, String tableName, Map partitions = Lists.newArrayList(); OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableName, Table.TableType.OLAP); UpdatePartitionMetaParam param = new UpdatePartitionMetaParam(); + boolean needSchemaVersionUpdate = + properties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT); + + if (properties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT)) { + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat = + PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat(new HashMap<>(properties)); + properties.put(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, + invertedIndexFileStorageFormat.name()); + } if (properties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_RETENTION_COUNT) && !(olapTable.getPartitionInfo().enableAutomaticPartition() @@ -370,6 +401,8 @@ public void updateTableProperties(Database db, String tableName, Map indexIdToSchemaVersion = new HashMap<>(); + Map partitionIdToInvertedIndexFileStorageFormat = + new HashMap<>(); + if (needSchemaVersionUpdate) { + long baseIndexId = olapTable.getBaseIndexId(); + indexIdToSchemaVersion.put(baseIndexId, + olapTable.getIndexMetaByIndexId(baseIndexId).getSchemaVersion() + 1); + partitionIdToInvertedIndexFileStorageFormat = + collectPartitionInvertedIndexFileStorageFormats(olapTable); + } + Env.getCurrentEnv().modifyTableProperties(db, olapTable, properties, indexIdToSchemaVersion, + partitionIdToInvertedIndexFileStorageFormat); } finally { olapTable.writeUnlock(); } @@ -402,6 +446,33 @@ public void updateTableProperties(Database db, String tableName, Map collectPartitionInvertedIndexFileStorageFormats( + OlapTable table) throws DdlException { + CloudInternalCatalog cloudCatalog = (CloudInternalCatalog) Env.getCurrentInternalCatalog(); + Map formats = new HashMap<>(); + for (Partition partition : table.getAllPartitions()) { + List tablets = partition.getBaseIndex().getTablets(); + if (tablets.isEmpty()) { + throw new DdlException("partition " + partition.getId() + " has no base tablet"); + } + Tablet tablet = tablets.get(0); + TInvertedIndexFileStorageFormat format = cloudCatalog.getPartitionInvertedIndexStorageFormat( + table.getId(), partition.getId(), table.getBaseIndexId(), tablet.getId()); + if (format == null) { + throw new DdlException("base tablet " + tablet.getId() + + " has no inverted index storage format"); + } + TInvertedIndexFileStorageFormat existingFormat = table.getPartitionInfo() + .getInvertedIndexFileStorageFormat(partition.getId()); + if (existingFormat != null && existingFormat != format) { + throw new DdlException("partition " + partition.getId() + + " has inconsistent FE and Meta Service inverted index storage formats"); + } + formats.put(partition.getId(), format); + } + return formats; + } + private static class UpdatePartitionMetaParam { public enum TabletMetaType { INMEMORY, diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index ee3091ec8e1751..da512d928a7ea2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -84,6 +84,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -110,7 +111,10 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa String storagePolicy, IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, - boolean isStorageMediumSpecified) + boolean isStorageMediumSpecified, + TInvertedIndexFileStorageFormat + partitionInvertedIndexFileStorageFormat, + Partition sourcePartition) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -141,6 +145,12 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa indexMap.put(indexId, rollup); } + // A replacement partition must preserve each source index's actual schema. The FE table + // metadata only describes the current table schema and is not authoritative once + // partitions created before a rolling format upgrade still exist. + Map sourceTabletMetas = sourcePartition == null + ? Collections.emptyMap() : getSourceTabletMetas(tbl.getId(), sourcePartition, indexMap.keySet()); + long version = partition.getVisibleVersion(); // short totalReplicaNum = replicaAlloc.getTotalReplicaNum(); @@ -148,9 +158,13 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa long indexId = entry.getKey(); MaterializedIndex index = entry.getValue(); MaterializedIndexMeta indexMeta = indexIdToMeta.get(indexId); + OlapFile.TabletMetaCloudPB sourceTabletMeta = sourceTabletMetas.get(indexId); + int schemaVersion = sourceTabletMeta == null + ? indexMeta.getSchemaVersion() : getTabletSchemaVersion(sourceTabletMeta); // create tablets - int schemaHash = indexMeta.getSchemaHash(); + int schemaHash = sourceTabletMeta == null + ? indexMeta.getSchemaHash() : sourceTabletMeta.getSchemaHash(); TabletMeta tabletMeta = new TabletMeta(dbId, tbl.getId(), partitionId, indexId, schemaHash, dataProperty.getStorageMedium()); createCloudTablets(index, ReplicaState.NORMAL, distributionInfo, version, replicaAlloc, @@ -167,6 +181,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa } else { indexes = Lists.newArrayList(); } + TInvertedIndexFileStorageFormat indexInvertedIndexFileStorageFormat = + indexId == tbl.getBaseIndexId() ? partitionInvertedIndexFileStorageFormat : null; List clusterKeyUids = null; if (indexId == tbl.getBaseIndexId()) { // only base and shadow index need cluster key unique column ids @@ -182,7 +198,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(), tbl.getCompressionType(), tbl.getStorageFormat(), storagePolicy, isInMemory, false, tbl.getName(), tbl.getTTLSeconds(), - tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), indexMeta.getSchemaVersion(), + tbl.getEnableUniqueKeyMergeOnWrite(), tbl.storeRowColumn(), schemaVersion, tbl.getCompactionPolicy(), tbl.getTimeSeriesCompactionGoalSizeMbytes(), tbl.getTimeSeriesCompactionFileCountThreshold(), tbl.getTimeSeriesCompactionTimeThresholdSeconds(), @@ -190,13 +206,16 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.getTimeSeriesCompactionLevelThreshold(), tbl.disableAutoCompaction(), tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), - tbl.getInvertedIndexFileStorageFormat(), + indexInvertedIndexFileStorageFormat, tbl.rowStorePageSize(), tbl.variantEnableFlattenNested(), clusterKeyUids, tbl.storagePageSize(), tbl.getTDEAlgorithmPB(), tbl.storageDictPageSize(), true, tbl.getColumnSeqMapping(), - tbl.getVerticalCompactionNumColumnsPerGroup()); + tbl.getVerticalCompactionNumColumnsPerGroup()); + if (sourceTabletMeta != null) { + applySourceSchema(builder, sourceTabletMeta); + } requestBuilder.addTabletMetas(builder); } requestBuilder.setDbId(dbId); @@ -216,6 +235,92 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa return partition; } + public OlapFile.TabletMetaCloudPB getTabletMeta(long tabletId) throws DdlException { + Cloud.GetTabletRequest request = Cloud.GetTabletRequest.newBuilder() + .setCloudUniqueId(Config.cloud_unique_id) + .setTabletId(tabletId) + .setRequestIp(FrontendOptions.getLocalHostAddressCached()) + .build(); + try { + Cloud.GetTabletResponse response = MetaServiceProxy.getInstance().getTablet(request); + if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK || !response.hasTabletMeta()) { + throw new DdlException("failed to get tablet " + tabletId + " from meta service: " + + response.getStatus().getMsg()); + } + return response.getTabletMeta(); + } catch (RpcException e) { + throw new DdlException("failed to get tablet " + tabletId + " from meta service: " + e.getMessage()); + } + } + + public TInvertedIndexFileStorageFormat getPartitionInvertedIndexStorageFormat(long tableId, + long partitionId, long baseIndexId, long tabletId) throws DdlException { + OlapFile.TabletMetaCloudPB tabletMeta = getPartitionIndexTabletMeta(tableId, partitionId, + baseIndexId, tabletId); + if (!tabletMeta.getSchema().hasInvertedIndexStorageFormat()) { + return null; + } + return TInvertedIndexFileStorageFormat.valueOf( + tabletMeta.getSchema().getInvertedIndexStorageFormat().name()); + } + + public int getTabletSchemaVersion(OlapFile.TabletMetaCloudPB tabletMeta) throws DdlException { + if (tabletMeta.hasSchemaVersion()) { + if (tabletMeta.hasSchema() && tabletMeta.getSchema().hasSchemaVersion() + && tabletMeta.getSchemaVersion() != tabletMeta.getSchema().getSchemaVersion()) { + throw new DdlException("tablet " + tabletMeta.getTabletId() + + " has inconsistent schema versions in tablet meta and schema"); + } + return tabletMeta.getSchemaVersion(); + } + if (tabletMeta.hasSchema() && tabletMeta.getSchema().hasSchemaVersion()) { + return tabletMeta.getSchema().getSchemaVersion(); + } + throw new DdlException("tablet " + tabletMeta.getTabletId() + " has no schema version"); + } + + public OlapFile.TabletMetaCloudPB getPartitionIndexTabletMeta(long tableId, long partitionId, long indexId, + long tabletId) throws DdlException { + OlapFile.TabletMetaCloudPB tabletMeta = getTabletMeta(tabletId); + if (tabletMeta.getTabletId() != tabletId || tabletMeta.getTableId() != tableId + || tabletMeta.getPartitionId() != partitionId || tabletMeta.getIndexId() != indexId + || !tabletMeta.hasSchema()) { + throw new DdlException("invalid source tablet meta for tablet " + tabletId); + } + getTabletSchemaVersion(tabletMeta); + return tabletMeta; + } + + private Map getSourceTabletMetas(long tableId, Partition sourcePartition, + Set indexIds) throws DdlException { + Map sourceTabletMetas = Maps.newHashMap(); + for (long indexId : indexIds) { + MaterializedIndex sourceIndex = sourcePartition.getIndex(indexId); + if (sourceIndex == null || sourceIndex.getTablets().isEmpty()) { + throw new DdlException("source partition " + sourcePartition.getId() + " has no tablet for index " + + indexId); + } + OlapFile.TabletMetaCloudPB sourceTabletMeta = getPartitionIndexTabletMeta(tableId, + sourcePartition.getId(), indexId, sourceIndex.getTablets().get(0).getId()); + sourceTabletMetas.put(indexId, sourceTabletMeta); + } + return sourceTabletMetas; + } + + private void applySourceSchema(OlapFile.TabletMetaCloudPB.Builder builder, + OlapFile.TabletMetaCloudPB sourceTabletMeta) throws DdlException { + int schemaVersion = getTabletSchemaVersion(sourceTabletMeta); + builder.setSchemaVersion(schemaVersion); + builder.setSchemaHash(sourceTabletMeta.getSchemaHash()); + builder.setSchema(sourceTabletMeta.getSchema()); + for (int i = 0; i < builder.getRsMetasCount(); ++i) { + builder.getRsMetasBuilder(i) + .setTabletSchemaHash(sourceTabletMeta.getSchemaHash()) + .setTabletSchema(sourceTabletMeta.getSchema()) + .setSchemaVersion(schemaVersion); + } + } + public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, long indexId, long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType, short shortKeyColumnCount, Set bfColumns, double bfFpp, List indexes, diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java index 7e02b530a060f1..cc4daf81f70769 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java @@ -165,6 +165,17 @@ public Cloud.UpdateTabletResponse updateTablet(Cloud.UpdateTabletRequest request .updateTablet(request); } + public Cloud.GetTabletResponse getTablet(Cloud.GetTabletRequest request) { + if (!request.hasCloudUniqueId()) { + Cloud.GetTabletRequest.Builder builder = Cloud.GetTabletRequest.newBuilder(); + builder.mergeFrom(request); + return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) + .getTablet(builder.setCloudUniqueId(Config.cloud_unique_id).build()); + } + return blockingStub.withDeadlineAfter(Config.meta_service_brpc_timeout_ms, TimeUnit.MILLISECONDS) + .getTablet(request); + } + public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest request) { if (!request.hasCloudUniqueId()) { Cloud.BeginTxnRequest.Builder builder = Cloud.BeginTxnRequest.newBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index 6b81f084717afe..d38e8bc647162d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -410,6 +410,11 @@ public Cloud.UpdateTabletResponse updateTablet(Cloud.UpdateTabletRequest request Cloud.UpdateTabletResponse::getStatus); } + public Cloud.GetTabletResponse getTablet(Cloud.GetTabletRequest request) throws RpcException { + return executeWithMetrics("getTablet", (client) -> client.getTablet(request), + Cloud.GetTabletResponse::getStatus); + } + public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest request) throws RpcException { return executeWithMetrics("beginTxn", (client) -> client.beginTxn(request), Cloud.BeginTxnResponse::getStatus); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java index 4bf00c0be67332..7e2d76f86fbdc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java @@ -78,6 +78,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.rpc.RpcException; import org.apache.doris.thrift.TCell; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TRow; import com.google.common.base.Joiner; @@ -114,7 +115,7 @@ public class PartitionsProcDir implements ProcDirInterface { .add("Buckets").add("ReplicationNum").add("StorageMedium").add("CooldownTime").add("RemoteStoragePolicy") .add("LastConsistencyCheckTime").add("DataSize").add("IsInMemory").add("ReplicaAllocation") .add("IsMutable").add("SyncWithBaseTables").add("UnsyncTables").add("CommittedVersion") - .add("RowCount") + .add("RowCount").add("InvertedIndexStorageFormat") .build(); private Database db; @@ -651,6 +652,17 @@ private List, TRow>> getPartitionInfosInrernal() throws An partitionInfo.add(partition.getRowCount()); trow.addToColumnValue(new TCell().setLongVal(partition.getRowCount())); + String invertedIndexStorageFormat = ""; + if (Config.isCloudMode()) { + TInvertedIndexFileStorageFormat format = + tblPartitionInfo.getInvertedIndexFileStorageFormat(partitionId); + invertedIndexStorageFormat = format == null ? "" : format.name(); + } else { + invertedIndexStorageFormat = olapTable.getInvertedIndexFileStorageFormat().name(); + } + partitionInfo.add(invertedIndexStorageFormat); + trow.addToColumnValue(new TCell().setStringVal(invertedIndexStorageFormat)); + partitionInfos.add(Pair.of(partitionInfo, trow)); } } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 305fcf19064603..bb738229e13b64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -127,6 +127,9 @@ public class PropertyAnalyzer { public static final String PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT = "inverted_index_storage_format"; + public static final String PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT = + "partition.inverted_index_storage_format"; + public static final String PROPERTIES_INMEMORY = "in_memory"; public static final String PROPERTIES_FILE_CACHE_TTL_SECONDS = "file_cache_ttl_seconds"; @@ -1222,39 +1225,58 @@ public static TStorageFormat analyzeStorageFormat(Map properties public static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFormat(Map properties) throws AnalysisException { - String invertedIndexFileStorageFormat = ""; if (properties != null && properties.containsKey(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT)) { - invertedIndexFileStorageFormat = properties.get(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT); - properties.remove(PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT); - } else { - if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { - return TInvertedIndexFileStorageFormat.V1; - } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { - return TInvertedIndexFileStorageFormat.V2; - } else { - return TInvertedIndexFileStorageFormat.V3; - } + return analyzeInvertedIndexFileStorageFormat(properties, + PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT, true); + } + + return getDefaultInvertedIndexFileStorageFormat(); + } + + public static TInvertedIndexFileStorageFormat analyzePartitionInvertedIndexFileStorageFormat( + Map properties) throws AnalysisException { + if (properties == null || !properties.containsKey(PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT)) { + return null; + } + if (!Config.isCloudMode()) { + throw new AnalysisException("partition.inverted_index_storage_format is only supported in cloud mode"); } - if (invertedIndexFileStorageFormat.equalsIgnoreCase("v1")) { + TInvertedIndexFileStorageFormat format = analyzeInvertedIndexFileStorageFormat(properties, + PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, false); + if (format == TInvertedIndexFileStorageFormat.V1) { + throw new AnalysisException("partition inverted index storage format only supports V2 and V3"); + } + return format; + } + + private static TInvertedIndexFileStorageFormat analyzeInvertedIndexFileStorageFormat( + Map properties, String propertyName, boolean allowV1) throws AnalysisException { + String invertedIndexFileStorageFormat = properties.remove(propertyName); + + if (allowV1 && invertedIndexFileStorageFormat.equalsIgnoreCase("v1")) { return TInvertedIndexFileStorageFormat.V1; } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v2")) { return TInvertedIndexFileStorageFormat.V2; } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("v3")) { return TInvertedIndexFileStorageFormat.V3; } else if (invertedIndexFileStorageFormat.equalsIgnoreCase("default")) { - if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { - return TInvertedIndexFileStorageFormat.V1; - } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { - return TInvertedIndexFileStorageFormat.V2; - } else { - return TInvertedIndexFileStorageFormat.V3; - } + return getDefaultInvertedIndexFileStorageFormat(); } else { throw new AnalysisException("unknown inverted index storage format: " + invertedIndexFileStorageFormat); } } + public static TInvertedIndexFileStorageFormat getDefaultInvertedIndexFileStorageFormat() { + if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { + return TInvertedIndexFileStorageFormat.V1; + } else if (Config.inverted_index_storage_format.equalsIgnoreCase("V2")) { + return TInvertedIndexFileStorageFormat.V2; + } else { + return TInvertedIndexFileStorageFormat.V3; + } + } + // analyze common boolean properties, such as "in_memory" = "false" public static boolean analyzeBooleanProp(Map properties, String propKey, boolean defaultVal) { if (properties != null && properties.containsKey(propKey)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 4b44421e583cc8..e2954d7b640aae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -1347,6 +1347,18 @@ private void replayCreateTableInternal(Database db, Table table) throws MetaNotF public void addPartitionLike(Database db, String tableName, AddPartitionLikeOp addPartitionLikeOp) throws DdlException { + addPartitionLike(db, tableName, addPartitionLikeOp, false); + } + + public void addPartitionLikeForInsertOverwrite(Database db, String tableName, + AddPartitionLikeOp addPartitionLikeOp) + throws DdlException { + addPartitionLike(db, tableName, addPartitionLikeOp, true); + } + + private void addPartitionLike(Database db, String tableName, AddPartitionLikeOp addPartitionLikeOp, + boolean preserveSourcePartitionSchema) + throws DdlException { try { Table table = db.getTableOrDdlException(tableName); @@ -1356,6 +1368,7 @@ public void addPartitionLike(Database db, String tableName, AddPartitionLikeOp a // Lock the table to prevent other SQL from performing write operation during table structure modification AddPartitionOp addPartitionOp = null; + Partition sourcePartition = null; table.readLock(); try { String partitionName = addPartitionLikeOp.getPartitionName(); @@ -1366,6 +1379,7 @@ public void addPartitionLike(Database db, String tableName, AddPartitionLikeOp a throw new DdlException("Failed to ADD PARTITION" + partitionName + " LIKE " + existedName + ". Reason: " + "partition " + existedName + "not exist"); } + sourcePartition = part; PartitionInfo partitionInfo = olapTable.getPartitionInfo(); SinglePartitionDesc newPartitionDesc; if (partitionInfo instanceof SinglePartitionInfo) { @@ -1389,7 +1403,8 @@ public void addPartitionLike(Database db, String tableName, AddPartitionLikeOp a } finally { table.readUnlock(); } - addPartition(db, tableName, addPartitionOp, false, 0, true, null); + addPartition(db, tableName, addPartitionOp, false, 0, true, null, + preserveSourcePartitionSchema ? sourcePartition : null); } catch (UserException e) { throw new DdlException("Failed to ADD PARTITION " + addPartitionLikeOp.getPartitionName() @@ -1479,9 +1494,19 @@ private ResultOr, DdlException> getCurrentPartitionFutur } public void addPartition(Database db, String tableName, AddPartitionOp addPartitionOp, - boolean isCreateTable, long generatedPartitionId, - boolean writeEditLog, - List> batchPartitions) + boolean isCreateTable, long generatedPartitionId, + boolean writeEditLog, + List> batchPartitions) + throws DdlException { + addPartition(db, tableName, addPartitionOp, isCreateTable, generatedPartitionId, + writeEditLog, batchPartitions, null); + } + + private void addPartition(Database db, String tableName, AddPartitionOp addPartitionOp, + boolean isCreateTable, long generatedPartitionId, + boolean writeEditLog, + List> batchPartitions, + Partition sourcePartition) throws DdlException { // in cloud mode, isCreateTable == true, create dynamic partition use, so partitionId must have been generated. // isCreateTable == false, other case, partitionId generate in below, must be set 0 @@ -1498,6 +1523,7 @@ public void addPartition(Database db, String tableName, AddPartitionOp addPartit Set bfColumns; String partitionName = singlePartitionDesc.getPartitionName(); BinlogConfig binlogConfig; + TInvertedIndexFileStorageFormat partitionInvertedIndexFileStorageFormat; // check OlapTable olapTable = db.getOlapTableOrDdlException(tableName); @@ -1680,6 +1706,17 @@ public void addPartition(Database db, String tableName, AddPartitionOp addPartit // get BinlogConfig binlogConfig = new BinlogConfig(olapTable.getBinlogConfig()); + if (Config.isCloudMode()) { + partitionInvertedIndexFileStorageFormat = sourcePartition == null + ? olapTable.getPartitionInvertedIndexFileStorageFormat() + : olapTable.getInvertedIndexFileStorageFormatForPartition(sourcePartition.getId()); + if (partitionInvertedIndexFileStorageFormat == null) { + throw new DdlException("missing inverted index storage format for source partition " + + sourcePartition.getId()); + } + } else { + partitionInvertedIndexFileStorageFormat = null; + } } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } finally { @@ -1738,7 +1775,8 @@ public void addPartition(Database db, String tableName, AddPartitionOp addPartit singlePartitionDesc.isInMemory(), singlePartitionDesc.getTabletType(), storagePolicy, idGeneratorBuffer, - binlogConfig, dataProperty.isStorageMediumSpecified()); + binlogConfig, dataProperty.isStorageMediumSpecified(), + partitionInvertedIndexFileStorageFormat, sourcePartition); // check again olapTable = db.getOlapTableOrDdlException(tableName); @@ -1807,6 +1845,10 @@ public void addPartition(Database db, String tableName, AddPartitionOp addPartit // update partition info partitionInfo.handleNewSinglePartitionDesc(singlePartitionDesc, partitionId, isTempPartition); + if (Config.isCloudMode()) { + partitionInfo.setInvertedIndexFileStorageFormat(partitionId, + partitionInvertedIndexFileStorageFormat); + } // log PartitionPersistInfo info = null; @@ -1814,17 +1856,20 @@ public void addPartition(Database db, String tableName, AddPartitionOp addPartit info = new PartitionPersistInfo(db.getId(), olapTable.getId(), partition, partitionInfo.getItem(partitionId).getItems(), ListPartitionItem.DUMMY_ITEM, dataProperty, partitionInfo.getReplicaAllocation(partitionId), partitionInfo.getIsInMemory(partitionId), - isTempPartition, partitionInfo.getIsMutable(partitionId)); + isTempPartition, partitionInfo.getIsMutable(partitionId), + partitionInvertedIndexFileStorageFormat); } else if (partitionInfo.getType() == PartitionType.LIST) { info = new PartitionPersistInfo(db.getId(), olapTable.getId(), partition, RangePartitionItem.DUMMY_RANGE, partitionInfo.getItem(partitionId), dataProperty, partitionInfo.getReplicaAllocation(partitionId), partitionInfo.getIsInMemory(partitionId), - isTempPartition, partitionInfo.getIsMutable(partitionId)); + isTempPartition, partitionInfo.getIsMutable(partitionId), + partitionInvertedIndexFileStorageFormat); } else { info = new PartitionPersistInfo(db.getId(), olapTable.getId(), partition, RangePartitionItem.DUMMY_RANGE, ListPartitionItem.DUMMY_ITEM, dataProperty, partitionInfo.getReplicaAllocation(partitionId), partitionInfo.getIsInMemory(partitionId), - isTempPartition, partitionInfo.getIsMutable(partitionId)); + isTempPartition, partitionInfo.getIsMutable(partitionId), + partitionInvertedIndexFileStorageFormat); } if (!isCreateTable) { @@ -1916,7 +1961,10 @@ public void replayAddPartition(PartitionPersistInfo info) throws MetaNotFoundExc partitionInfo.unprotectHandleNewSinglePartitionDesc(partition.getId(), info.isTempPartition(), partitionItem, info.getDataProperty(), info.getReplicaAlloc(), info.isInMemory(), info.isMutable()); - + if (info.getInvertedIndexFileStorageFormat() != null) { + partitionInfo.setInvertedIndexFileStorageFormat(partition.getId(), + info.getInvertedIndexFileStorageFormat()); + } // add to inverted index TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex(); for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.ALL)) { @@ -2084,7 +2132,10 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa String storagePolicy, IdGeneratorBuffer idGeneratorBuffer, BinlogConfig binlogConfig, - boolean isStorageMediumSpecified) + boolean isStorageMediumSpecified, + TInvertedIndexFileStorageFormat + partitionInvertedIndexFileStorageFormat, + Partition sourcePartition) throws DdlException { // create base index first. Preconditions.checkArgument(tbl.getBaseIndexId() != -1); @@ -2149,6 +2200,8 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa } KeysType keysType = indexMeta.getKeysType(); List indexes = indexId == tbl.getBaseIndexId() ? tbl.getCopiedIndexes() : null; + TInvertedIndexFileStorageFormat indexInvertedIndexFileStorageFormat = + indexId == tbl.getBaseIndexId() ? tbl.getInvertedIndexFileStorageFormat() : null; int totalTaskNum = index.getTablets().size() * totalReplicaNum; MarkedCountDownLatch countDownLatch = new MarkedCountDownLatch(totalTaskNum); AgentBatchTask batchTask = new AgentBatchTask(); @@ -2186,7 +2239,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa rowBinlogIndexMeta); task.setStorageFormat(tbl.getStorageFormat()); - task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); + task.setInvertedIndexFileStorageFormat(indexInvertedIndexFileStorageFormat); if (!CollectionUtils.isEmpty(clusterKeyUids)) { task.setClusterKeyUids(clusterKeyUids); LOG.info("table: {}, partition: {}, index: {}, tablet: {}, cluster key uids: {}", @@ -2587,6 +2640,17 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th } olapTable.setInvertedIndexFileStorageFormat(invertedIndexFileStorageFormat); + TInvertedIndexFileStorageFormat partitionInvertedIndexFileStorageFormat; + try { + partitionInvertedIndexFileStorageFormat = + PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + if (partitionInvertedIndexFileStorageFormat != null) { + olapTable.setPartitionInvertedIndexFileStorageFormat(partitionInvertedIndexFileStorageFormat); + } + // get compression type TCompressionType compressionType = TCompressionType.LZ4; try { @@ -3101,8 +3165,13 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th storagePolicy, idGeneratorBuffer, binlogConfigForTask, - partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified()); + partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(), + olapTable.getPartitionInvertedIndexFileStorageFormat(), null); olapTable.addPartition(partition); + if (Config.isCloudMode()) { + partitionInfo.setInvertedIndexFileStorageFormat(partitionId, + olapTable.getPartitionInvertedIndexFileStorageFormat()); + } afterCreatePartitions(db.getId(), olapTable.getId(), olapTable.getPartitionIds(), olapTable.getIndexIdList(), true /* isCreateTable */, true /* isBatchCommit */, olapTable); } else if (partitionInfo.getType() == PartitionType.RANGE @@ -3191,8 +3260,13 @@ private boolean createOlapTable(Database db, CreateTableInfo createTableInfo) th partitionInfo.getTabletType(entry.getValue()), partionStoragePolicy, idGeneratorBuffer, binlogConfigForTask, - dataProperty.isStorageMediumSpecified()); + dataProperty.isStorageMediumSpecified(), + olapTable.getPartitionInvertedIndexFileStorageFormat(), null); olapTable.addPartition(partition); + if (Config.isCloudMode()) { + partitionInfo.setInvertedIndexFileStorageFormat(partition.getId(), + olapTable.getPartitionInvertedIndexFileStorageFormat()); + } olapTable.getPartitionInfo().getDataProperty(partition.getId()) .setStoragePolicy(partionStoragePolicy); } @@ -3628,7 +3702,9 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa copiedTbl.getPartitionInfo().getTabletType(oldPartitionId), olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(), idGeneratorBuffer, binlogConfig, - copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified()); + copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(), + copiedTbl.getPartitionInfo().getInvertedIndexFileStorageFormat(oldPartitionId), + copiedTbl.getPartition(oldPartitionId)); newPartitions.add(newPartition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/FeServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/FeServiceClient.java index 8fa391fb77629c..6b6e5c1bee0f22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/FeServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/FeServiceClient.java @@ -447,8 +447,9 @@ public TNetworkAddress getMasterAddress() throws RuntimeException { } } - public void addPartitions(String catalogName, String dbName, String tableName, List partitionNames, - List tempPartitionNames, boolean isTemp) throws DdlException { + public void addPartitionsForInsertOverwrite(String catalogName, String dbName, String tableName, + List partitionNames, List tempPartitionNames, + boolean isTemp) throws DdlException { TAddOrDropPartitionsRequest request = new TAddOrDropPartitionsRequest(); request.setUser(user); request.setPasswd(password); @@ -462,7 +463,7 @@ public void addPartitions(String catalogName, String dbName, String tableName, L long startTime = System.currentTimeMillis(); TAddOrDropPartitionsResult result; try { - result = masterCallWithRetry(client -> client.addOrDropPartitions(request), + result = masterCallWithRetry(client -> client.addPartitionsForInsertOverwrite(request), "failed to add partitions to remote doris:" + name, timeoutMs); } catch (Exception e) { long costMs = System.currentTimeMillis() - startTime; diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteUtil.java index edbeea727edcb8..b1f6a348b701f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteUtil.java @@ -53,13 +53,14 @@ public class InsertOverwriteUtil { public static void addTempPartitions(TableIf tableIf, List partitionNames, List tempPartitionNames) throws DdlException { if (tableIf instanceof RemoteOlapTable) { - ((RemoteOlapTable) tableIf).getCatalog().getFeServiceClient().addPartitions( + ((RemoteOlapTable) tableIf).getCatalog().getFeServiceClient().addPartitionsForInsertOverwrite( InternalCatalog.INTERNAL_CATALOG_NAME, ((RemoteOlapTable) tableIf).getDBName(), tableIf.getName(), partitionNames, tempPartitionNames, true); } else if (tableIf instanceof OlapTable) { for (int i = 0; i < partitionNames.size(); i++) { - Env.getCurrentEnv().addPartitionLike((Database) tableIf.getDatabase(), tableIf.getName(), - new AddPartitionLikeOp(tempPartitionNames.get(i), partitionNames.get(i), true)); + Env.getCurrentEnv().addPartitionLikeForInsertOverwrite( + (Database) tableIf.getDatabase(), tableIf.getName(), + new AddPartitionLikeOp(tempPartitionNames.get(i), partitionNames.get(i), true)); LOG.info("successfully add temp partition [{}] for [{}]", tempPartitionNames.get(i), tableIf.getName()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommand.java index 458b597d64f122..1d7996c6a5afd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommand.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Table; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.util.Util; @@ -34,6 +35,7 @@ import org.apache.doris.qe.ShowResultSet; import org.apache.doris.qe.ShowResultSetMetaData; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; @@ -69,6 +71,7 @@ public ShowResultSetMetaData getMetaData() { builder.addColumn(new Column("PartitionName", ScalarType.createVarchar(30))); builder.addColumn(new Column("DbId", ScalarType.createVarchar(30))); builder.addColumn(new Column("TableId", ScalarType.createVarchar(30))); + builder.addColumn(new Column("InvertedIndexStorageFormat", ScalarType.createVarchar(30))); return builder.build(); } @@ -84,11 +87,14 @@ private ShowResultSet handleShowPartitionId(ConnectContext ctx, StmtExecutor exe List tables = database.getTables(); for (Table tbl : tables) { if (tbl instanceof OlapTable) { + List row = null; + TInvertedIndexFileStorageFormat format = null; tbl.readLock(); try { - Partition partition = ((OlapTable) tbl).getPartition(partitionId); + OlapTable olapTable = (OlapTable) tbl; + Partition partition = olapTable.getPartition(partitionId); if (partition != null) { - List row = new ArrayList<>(); + row = new ArrayList<>(); row.add(database.getFullName()); if (tbl.isTemporary()) { if (!Util.isTempTableInCurrentSession(tbl.getName())) { @@ -101,14 +107,23 @@ private ShowResultSet handleShowPartitionId(ConnectContext ctx, StmtExecutor exe row.add(partition.getName()); row.add(String.valueOf(database.getId())); row.add(String.valueOf(tbl.getId())); - rows.add(row); - break; + if (Config.isCloudMode()) { + format = olapTable.getPartitionInfo() + .getInvertedIndexFileStorageFormat(partitionId); + } else { + format = olapTable.getInvertedIndexFileStorageFormat(); + } } } catch (Exception e) { LOG.error("failed to get partition info for {}", partitionId, e); } finally { tbl.readUnlock(); } + if (row != null) { + row.add(format == null ? "" : format.name()); + rows.add(row); + break; + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index ad4a526cb6a134..3a672893605965 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -849,14 +849,23 @@ public void validate(ConnectContext ctx) { columns.forEach(c -> c.validate(engineName.equals(ENGINE_OLAP), keysSet, orderKeySet, finalEnableMergeOnWrite, keysType)); + try { + invertedIndexFileStorageFormat = + PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat(new HashMap<>(properties)); + } catch (Exception e) { + throw new AnalysisException(e.getMessage(), e.getCause()); + } + // validate index if (!indexes.isEmpty()) { Set distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - try { - invertedIndexFileStorageFormat = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( - new HashMap<>(properties)); - } catch (Exception e) { - throw new AnalysisException(e.getMessage(), e.getCause()); + if (invertedIndexFileStorageFormat == null) { + try { + invertedIndexFileStorageFormat = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat( + new HashMap<>(properties)); + } catch (Exception e) { + throw new AnalysisException(e.getMessage(), e.getCause()); + } } for (IndexDefinition indexDef : indexes) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java index 6e24f648490a24..6d191549725568 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ModifyTablePropertiesOp.java @@ -281,6 +281,10 @@ public void validate(ConnectContext ctx) throws UserException { + " should be set to true or false"); } this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; + } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT)) { + PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat( + new java.util.HashMap<>(properties)); + this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_INVERTED_INDEX_STORAGE_FORMAT)) { throw new AnalysisException( "Property " diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyTablePropertyOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyTablePropertyOperationLog.java index 1d64029bcfdb00..e02d3ca1765318 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyTablePropertyOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyTablePropertyOperationLog.java @@ -20,6 +20,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import com.google.gson.annotations.SerializedName; @@ -43,14 +44,31 @@ public class ModifyTablePropertyOperationLog implements Writable { private String tableName; @SerializedName(value = "properties") private Map properties = new HashMap<>(); + @SerializedName(value = "indexIdToSchemaVersion") + private Map indexIdToSchemaVersion = new HashMap<>(); + @SerializedName(value = "partitionIdToInvertedIndexFileStorageFormat") + private Map partitionIdToInvertedIndexFileStorageFormat = new HashMap<>(); @SerializedName(value = "sql") private String sql; public ModifyTablePropertyOperationLog(long dbId, long tableId, String tableName, Map properties) { + this(dbId, tableId, tableName, properties, new HashMap<>()); + } + + public ModifyTablePropertyOperationLog(long dbId, long tableId, String tableName, Map properties, + Map indexIdToSchemaVersion) { + this(dbId, tableId, tableName, properties, indexIdToSchemaVersion, new HashMap<>()); + } + + public ModifyTablePropertyOperationLog(long dbId, long tableId, String tableName, Map properties, + Map indexIdToSchemaVersion, + Map partitionIdToInvertedIndexFileStorageFormat) { this.dbId = dbId; this.tableId = tableId; this.tableName = tableName; this.properties = properties; + this.indexIdToSchemaVersion = indexIdToSchemaVersion; + this.partitionIdToInvertedIndexFileStorageFormat = partitionIdToInvertedIndexFileStorageFormat; StringBuilder sb = new StringBuilder(); sb.append("SET ("); @@ -69,6 +87,7 @@ public ModifyTablePropertyOperationLog(String ctlName, String dbName, String tab this.dbName = dbName; this.tableName = tableName; this.properties = properties; + this.indexIdToSchemaVersion = new HashMap<>(); } public long getDbId() { @@ -95,6 +114,14 @@ public Map getProperties() { return properties; } + public Map getIndexIdToSchemaVersion() { + return indexIdToSchemaVersion; + } + + public Map getPartitionIdToInvertedIndexFileStorageFormat() { + return partitionIdToInvertedIndexFileStorageFormat; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java index 8dad8a90733c4f..0880c8fce75df6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/PartitionPersistInfo.java @@ -25,6 +25,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import com.google.common.collect.Range; import com.google.gson.annotations.SerializedName; @@ -55,13 +56,16 @@ public class PartitionPersistInfo implements Writable { private boolean isTempPartition = false; @SerializedName(value = "isMutable") private boolean isMutable = true; + @SerializedName(value = "invertedIndexFileStorageFormat") + private TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat; public PartitionPersistInfo() { } public PartitionPersistInfo(long dbId, long tableId, Partition partition, Range range, PartitionItem listPartitionItem, DataProperty dataProperty, ReplicaAllocation replicaAlloc, - boolean isInMemory, boolean isTempPartition, boolean isMutable) { + boolean isInMemory, boolean isTempPartition, boolean isMutable, + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat) { this.dbId = dbId; this.tableId = tableId; this.partition = partition; @@ -74,6 +78,7 @@ public PartitionPersistInfo(long dbId, long tableId, Partition partition, Range< this.isInMemory = isInMemory; this.isTempPartition = isTempPartition; this.isMutable = isMutable; + this.invertedIndexFileStorageFormat = invertedIndexFileStorageFormat; } public Long getDbId() { @@ -116,6 +121,10 @@ public boolean isTempPartition() { return isTempPartition; } + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { + return invertedIndexFileStorageFormat; + } + public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 6a5695cebb040f..63626e7e9a69c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -2499,6 +2499,17 @@ public TMasterAddressResult getMasterAddress(TMasterAddressRequest request) thro @Override public TAddOrDropPartitionsResult addOrDropPartitions(TAddOrDropPartitionsRequest request) throws TException { + return addOrDropPartitions(request, false); + } + + @Override + public TAddOrDropPartitionsResult addPartitionsForInsertOverwrite(TAddOrDropPartitionsRequest request) + throws TException { + return addOrDropPartitions(request, true); + } + + private TAddOrDropPartitionsResult addOrDropPartitions(TAddOrDropPartitionsRequest request, + boolean isInsertOverwrite) throws TException { String clientAddr = getClientAddrAsString(); if (LOG.isDebugEnabled()) { LOG.debug("receive get master address request: {}, client: {}", request, clientAddr); @@ -2520,7 +2531,7 @@ public TAddOrDropPartitionsResult addOrDropPartitions(TAddOrDropPartitionsReques if (status.getStatusCode() != TStatusCode.OK) { return result; } - addOrDropPartitionsImpl(request); + addOrDropPartitionsImpl(request, isInsertOverwrite); } catch (AuthenticationException e) { status.setStatusCode(TStatusCode.NOT_AUTHORIZED); status.addToErrorMsgs(e.getMessage()); @@ -2536,7 +2547,8 @@ public TAddOrDropPartitionsResult addOrDropPartitions(TAddOrDropPartitionsReques return result; } - private void addOrDropPartitionsImpl(TAddOrDropPartitionsRequest request) throws DdlException { + private void addOrDropPartitionsImpl(TAddOrDropPartitionsRequest request, boolean isInsertOverwrite) + throws DdlException { String dbName = request.getDb(); Database db = Env.getCurrentInternalCatalog().getDbNullable(dbName); if (db == null) { @@ -2568,8 +2580,14 @@ private void addOrDropPartitionsImpl(TAddOrDropPartitionsRequest request) throws List partitionNames = request.getPartitionNames(); List tempPartitionNames = request.getTempPartitionNames(); for (int i = 0; i < partitionNames.size(); i++) { - Env.getCurrentEnv().addPartitionLike(db, request.getTbl(), - new AddPartitionLikeOp(tempPartitionNames.get(i), partitionNames.get(i), request.is_temp)); + AddPartitionLikeOp addPartitionLikeOp = new AddPartitionLikeOp( + tempPartitionNames.get(i), partitionNames.get(i), request.is_temp); + if (isInsertOverwrite) { + Env.getCurrentEnv().addPartitionLikeForInsertOverwrite(db, request.getTbl(), + addPartitionLikeOp); + } else { + Env.getCurrentEnv().addPartitionLike(db, request.getTbl(), addPartitionLikeOp); + } LOG.info("successfully add {}partition [{}] for [{}]", request.is_temp ? "temp " : "", partitionNames.get(i), olapTable.getName()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java index 160399bfd000b3..675fde3e6e738e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/PartitionsTableValuedFunction.java @@ -88,7 +88,10 @@ public class PartitionsTableValuedFunction extends MetadataTableValuedFunction { new Column("ReplicaAllocation", ScalarType.createStringType()), new Column("IsMutable", ScalarType.createType(PrimitiveType.BOOLEAN)), new Column("SyncWithBaseTables", ScalarType.createType(PrimitiveType.BOOLEAN)), - new Column("UnsyncTables", ScalarType.createStringType())); + new Column("UnsyncTables", ScalarType.createStringType()), + new Column("CommittedVersion", ScalarType.createType(PrimitiveType.BIGINT)), + new Column("RowCount", ScalarType.createType(PrimitiveType.BIGINT)), + new Column("InvertedIndexStorageFormat", ScalarType.createStringType())); private static final ImmutableList SCHEMA_FOR_EXTERNAL_TABLE = ImmutableList.of( new Column("Partition", ScalarType.createStringType())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudSchemaChangeJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudSchemaChangeJobV2Test.java new file mode 100644 index 00000000000000..13ed83e6ab29bc --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudSchemaChangeJobV2Test.java @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndexMeta; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.common.DdlException; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.util.HashMap; +import java.util.Map; + +public class CloudSchemaChangeJobV2Test { + private static final long BASE_INDEX_ID = 100L; + private static final long SHADOW_INDEX_ID = 101L; + private static final long V2_PARTITION_ID = 200L; + private static final long V3_PARTITION_ID = 201L; + + @Test + public void testBaseShadowVersionsFollowPartitionFormats() throws Exception { + CloudSchemaChangeJobV2 job = newJob(V2_PARTITION_ID, V3_PARTITION_ID); + OlapTable table = tableWithBaseSchemaVersion(2, TInvertedIndexFileStorageFormat.V3, + Map.of(V2_PARTITION_ID, TInvertedIndexFileStorageFormat.V2, + V3_PARTITION_ID, TInvertedIndexFileStorageFormat.V3)); + + job.configureBaseShadowSchemaVersion(table); + + Assert.assertEquals(Map.of(TInvertedIndexFileStorageFormat.V2, 3, + TInvertedIndexFileStorageFormat.V3, 4), + job.getBaseShadowSchemaVersionByFormat()); + Assert.assertEquals(4, + job.indexSchemaVersionAndHashMap.get(SHADOW_INDEX_ID).schemaVersion); + } + + @Test + public void testCurrentFormatGetsSchemaVersionWithoutLivePartition() throws Exception { + CloudSchemaChangeJobV2 job = newJob(V2_PARTITION_ID); + OlapTable table = tableWithBaseSchemaVersion(2, TInvertedIndexFileStorageFormat.V3, + Map.of(V2_PARTITION_ID, TInvertedIndexFileStorageFormat.V2)); + + job.configureBaseShadowSchemaVersion(table); + + Assert.assertEquals(Map.of(TInvertedIndexFileStorageFormat.V2, 3, + TInvertedIndexFileStorageFormat.V3, 4), + job.getBaseShadowSchemaVersionByFormat()); + } + + @Test + public void testBaseShadowVersionsRejectMissingPartitionFormat() throws Exception { + CloudSchemaChangeJobV2 job = newJob(V2_PARTITION_ID); + OlapTable table = tableWithBaseSchemaVersion(2, TInvertedIndexFileStorageFormat.V3, new HashMap<>()); + + try { + job.configureBaseShadowSchemaVersion(table); + Assert.fail("schema change must reject a partition without a resolved format"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("missing inverted index storage format")); + } + } + + @Test + public void testBaseShadowVersionsUseResolvedLegacyFormat() throws Exception { + CloudSchemaChangeJobV2 job = newJob(V2_PARTITION_ID); + OlapTable table = tableWithBaseSchemaVersion(2, TInvertedIndexFileStorageFormat.V3, new HashMap<>()); + Mockito.when(table.getInvertedIndexFileStorageFormatForPartition(V2_PARTITION_ID)) + .thenReturn(TInvertedIndexFileStorageFormat.V2); + + job.configureBaseShadowSchemaVersion(table); + + Assert.assertEquals(Map.of(TInvertedIndexFileStorageFormat.V2, 3, + TInvertedIndexFileStorageFormat.V3, 4), + job.getBaseShadowSchemaVersionByFormat()); + } + + @Test + public void testBaseShadowVersionPlanSurvivesJobSerialization() throws Exception { + CloudSchemaChangeJobV2 job = newJob(); + Map shadowVersions = Map.of( + TInvertedIndexFileStorageFormat.V2, 3, + TInvertedIndexFileStorageFormat.V3, 4); + job.setBaseShadowSchemaVersionByFormat(shadowVersions); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(bytes)) { + job.write(output); + } + + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + CloudSchemaChangeJobV2 replayedJob = (CloudSchemaChangeJobV2) AlterJobV2.read(input); + Assert.assertEquals(shadowVersions, replayedJob.getBaseShadowSchemaVersionByFormat()); + } + } + + private CloudSchemaChangeJobV2 newJob(long... partitionIds) { + CloudSchemaChangeJobV2 job = new CloudSchemaChangeJobV2("", 1L, 2L, 3L, "tbl", 4L); + job.addIndexSchema(SHADOW_INDEX_ID, BASE_INDEX_ID, "__doris_shadow_tbl", 3, 4, (short) 1, null); + for (long partitionId : partitionIds) { + job.addPartitionShadowIndex(partitionId, SHADOW_INDEX_ID, Mockito.mock(MaterializedIndex.class)); + } + return job; + } + + private OlapTable tableWithBaseSchemaVersion(int schemaVersion, + TInvertedIndexFileStorageFormat currentFormat, + Map partitionFormats) { + OlapTable table = Mockito.mock(OlapTable.class); + MaterializedIndexMeta baseIndexMeta = Mockito.mock(MaterializedIndexMeta.class); + PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(table.getBaseIndexId()).thenReturn(BASE_INDEX_ID); + Mockito.when(table.getIndexMetaByIndexId(BASE_INDEX_ID)).thenReturn(baseIndexMeta); + Mockito.when(baseIndexMeta.getSchemaVersion()).thenReturn(schemaVersion); + Mockito.when(table.getPartitionInvertedIndexFileStorageFormat()).thenReturn(currentFormat); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + Mockito.when(table.getInvertedIndexFileStorageFormatForPartition(Mockito.anyLong())) + .thenAnswer(invocation -> partitionFormats.get(invocation.getArgument(0))); + return table; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java index 99831c65578064..cd3461162b5edd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogRecycleBinTest.java @@ -27,6 +27,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.util.URI; import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.utframe.UtFrameUtils; @@ -490,12 +491,15 @@ public void testRecoverPartition() throws Exception { new DataProperty(TStorageMedium.HDD), new ReplicaAllocation((short) 3), false, - false + false, + TInvertedIndexFileStorageFormat.V3 ); recycleBin.recoverPartition(CatalogTestUtil.testDbId1, olapTable, CatalogTestUtil.testPartition1, -1, null); Assert.assertFalse(recycleBin.isRecyclePartition(CatalogTestUtil.testDbId1, CatalogTestUtil.testTableId1, CatalogTestUtil.testPartitionId1)); Assert.assertNotNull(olapTable.getPartition(CatalogTestUtil.testPartition1)); + Assert.assertEquals(TInvertedIndexFileStorageFormat.V3, olapTable.getPartitionInfo() + .getInvertedIndexFileStorageFormat(CatalogTestUtil.testPartitionId1)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/EnvTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/EnvTest.java index fe3f4aebadde40..d11b34176158d3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/EnvTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/EnvTest.java @@ -24,6 +24,9 @@ import org.apache.doris.meta.MetaContext; import org.apache.doris.mysql.authenticate.ldap.LdapManager; import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.persist.ModifyTablePropertyOperationLog; +import org.apache.doris.persist.OperationType; +import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.persist.meta.MetaHeader; import org.junit.After; @@ -183,4 +186,25 @@ public void testSetLdapDefaultRolesConfigRefreshesLdapCache() throws Exception { LdapConfig.ldap_default_roles = oldLdapDefaultRoles; } } + + @Test + public void testReplayLegacyModifyTablePropertyWithoutSchemaVersions() throws Exception { + FakeEditLog fakeEditLog = new FakeEditLog(); + FakeEnv fakeEnv = new FakeEnv(); + try { + Env env = CatalogTestUtil.createTestCatalog(); + Database db = env.getInternalCatalog().getDbOrMetaException(CatalogTestUtil.testDbId1); + OlapTable table = (OlapTable) db.getTableOrMetaException(CatalogTestUtil.testTableId1); + ModifyTablePropertyOperationLog log = GsonUtils.GSON.fromJson( + "{\"dbId\":" + db.getId() + ",\"tableId\":" + table.getId() + + ",\"properties\":{\"in_memory\":\"false\"}}", + ModifyTablePropertyOperationLog.class); + + Assert.assertNull(log.getIndexIdToSchemaVersion()); + env.replayModifyTableProperty(OperationType.OP_MODIFY_TABLE_PROPERTIES, log); + } finally { + fakeEnv.close(); + fakeEditLog.close(); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java index f7f7bb88beb8ab..872589ebe4ad85 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java @@ -29,6 +29,7 @@ import org.apache.doris.common.util.UnitTestUtil; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; @@ -232,6 +233,7 @@ public void testResetPartitionIdForRestore() { DataProperty origDataProperty = new DataProperty(TStorageMedium.SSD, 1735689600000L, "s3_policy"); ReplicaAllocation origReplicaAlloc = new ReplicaAllocation((short) 3); partitionInfo.addPartition(origPartId, origDataProperty, origReplicaAlloc, true, true); + partitionInfo.setInvertedIndexFileStorageFormat(origPartId, TInvertedIndexFileStorageFormat.V3); Map partitionIdMap = Maps.newHashMap(); long newPartId = 2000L; @@ -251,6 +253,8 @@ public void testResetPartitionIdForRestore() { Assert.assertEquals(1735689600000L, newDataProperty.getCooldownTimeMs()); Assert.assertEquals("s3_policy", newDataProperty.getStoragePolicy()); Assert.assertTrue(partitionInfo.getIsInMemory(newPartId)); + Assert.assertEquals(TInvertedIndexFileStorageFormat.V3, + partitionInfo.getInvertedIndexFileStorageFormat(newPartId)); } } @@ -261,6 +265,7 @@ public void testResetPartitionIdForRestoreInCloudMode() { DataProperty origDataProperty = new DataProperty(TStorageMedium.SSD, 1735689600000L, "s3_policy"); ReplicaAllocation origReplicaAlloc = new ReplicaAllocation((short) 3); partitionInfo.addPartition(origPartId, origDataProperty, origReplicaAlloc, true, true); + partitionInfo.setInvertedIndexFileStorageFormat(origPartId, TInvertedIndexFileStorageFormat.V2); Map partitionIdMap = Maps.newHashMap(); long newPartId = 2000L; @@ -284,9 +289,57 @@ public void testResetPartitionIdForRestoreInCloudMode() { Assert.assertEquals("", newDataProperty.getStoragePolicy()); Assert.assertTrue(newDataProperty.isMutable()); Assert.assertFalse(partitionInfo.getIsInMemory(newPartId)); + Assert.assertEquals(TInvertedIndexFileStorageFormat.V2, + partitionInfo.getInvertedIndexFileStorageFormat(newPartId)); } } + @Test + public void testGetInvertedIndexFileStorageFormatForPartition() { + PartitionInfo partitionInfo = new PartitionInfo(PartitionType.RANGE); + OlapTable olapTable = new OlapTable(1, "tbl", Lists.newArrayList(), KeysType.DUP_KEYS, + partitionInfo, null); + olapTable.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2); + + // Tables created before the rolling-format property was added have no partition map. + Assert.assertEquals(TInvertedIndexFileStorageFormat.V2, + olapTable.getInvertedIndexFileStorageFormatForPartition(10L)); + + olapTable.setPartitionInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V3); + Assert.assertNull(olapTable.getInvertedIndexFileStorageFormatForPartition(10L)); + + partitionInfo.setInvertedIndexFileStorageFormat(10L, TInvertedIndexFileStorageFormat.V2); + Assert.assertEquals(TInvertedIndexFileStorageFormat.V2, + olapTable.getInvertedIndexFileStorageFormatForPartition(10L)); + } + + @Test + public void testReplacePartitionPreservesInvertedIndexStorageFormat() { + long oldPartitionId = 10L; + long newPartitionId = 20L; + PartitionInfo partitionInfo = new SinglePartitionInfo(); + OlapTable table = new OlapTable(1L, "tbl", Lists.newArrayList(), KeysType.DUP_KEYS, + partitionInfo, null); + Partition oldPartition = new Partition(oldPartitionId, "p1", + new MaterializedIndex(1L, MaterializedIndex.IndexState.NORMAL), null); + Partition newPartition = new Partition(newPartitionId, "p1", + new MaterializedIndex(1L, MaterializedIndex.IndexState.NORMAL), null); + table.addPartition(oldPartition); + partitionInfo.addPartition(oldPartitionId, new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), false, true); + partitionInfo.setInvertedIndexFileStorageFormat(oldPartitionId, TInvertedIndexFileStorageFormat.V2); + + RecyclePartitionParam recyclePartitionParam = new RecyclePartitionParam(); + Assert.assertSame(oldPartition, table.replacePartition(newPartition, recyclePartitionParam)); + + Assert.assertNull(partitionInfo.getInvertedIndexFileStorageFormat(oldPartitionId)); + Assert.assertEquals(TInvertedIndexFileStorageFormat.V2, + partitionInfo.getInvertedIndexFileStorageFormat(newPartitionId)); + Assert.assertEquals(TInvertedIndexFileStorageFormat.V2, + recyclePartitionParam.invertedIndexFileStorageFormat); + Assert.assertSame(newPartition, table.getPartition("p1")); + } + @Test public void testBuildVariantEnableFlattenNestedWithLegacyPropertyKey() throws IOException { Map properties = Maps.newHashMap(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TablePropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/TablePropertyTest.java index dd250e44e9f9c4..a9e925cfec2461 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TablePropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TablePropertyTest.java @@ -35,6 +35,15 @@ public class TablePropertyTest { private static final String REPLICATION_ALLOCATION = "tag.location.group_0: 1, tag.location.group_1: 1, tag.location.group_2: 1"; + @Test + public void testPartitionInvertedIndexStorageFormat() { + Map properties = Maps.newHashMap(); + properties.put(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, "V3"); + TableProperty tableProperty = new TableProperty(properties); + Assert.assertEquals(org.apache.doris.thrift.TInvertedIndexFileStorageFormat.V3, + tableProperty.getPartitionInvertedIndexFileStorageFormat()); + } + // A non-whitelisted dynamic_partition.* key is ignored (skipped via continue), so it is not // collected at all and the table is neither built as dynamic nor flagged as incomplete. @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java new file mode 100644 index 00000000000000..c3b606a0f9e4f8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.datasource; + +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.common.DdlException; +import org.apache.doris.proto.OlapFile; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +public class CloudInternalCatalogTest { + private static final long TABLE_ID = 10L; + private static final long PARTITION_ID = 20L; + private static final long INDEX_ID = 30L; + + @Test + public void testGetPartitionIndexTabletMetaUsesLegacySchemaVersion() throws Exception { + OlapFile.TabletMetaCloudPB tablet = tabletMeta(100L, + OlapFile.InvertedIndexStorageFormatPB.V2, false); + + try (MockedStatic mockedProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + mockedProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(proxy.getTablet(Mockito.any())).thenReturn(getTabletResponse(tablet)); + + CloudInternalCatalog catalog = new CloudInternalCatalog(); + OlapFile.TabletMetaCloudPB tabletMeta = catalog.getPartitionIndexTabletMeta(TABLE_ID, PARTITION_ID, + INDEX_ID, 100L); + + Assert.assertEquals(100L, tabletMeta.getTabletId()); + Assert.assertEquals(7, catalog.getTabletSchemaVersion(tabletMeta)); + Mockito.verify(proxy).getTablet(Mockito.any()); + } + } + + @Test + public void testGetPartitionIndexTabletMetaRejectsWrongIdentity() throws Exception { + OlapFile.TabletMetaCloudPB tablet = tabletMeta(100L, + OlapFile.InvertedIndexStorageFormatPB.V2, true).toBuilder() + .setPartitionId(PARTITION_ID + 1).build(); + + try (MockedStatic mockedProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + mockedProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(proxy.getTablet(Mockito.any())).thenReturn(getTabletResponse(tablet)); + + try { + new CloudInternalCatalog().getPartitionIndexTabletMeta(TABLE_ID, PARTITION_ID, INDEX_ID, 100L); + Assert.fail("wrong tablet identity must fail"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("invalid source tablet meta")); + } + } + } + + @Test + public void testGetPartitionInvertedIndexStorageFormat() throws Exception { + OlapFile.TabletMetaCloudPB tablet = tabletMeta(100L, + OlapFile.InvertedIndexStorageFormatPB.V3, true); + + try (MockedStatic mockedProxyStatic = Mockito.mockStatic(MetaServiceProxy.class)) { + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + mockedProxyStatic.when(MetaServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(proxy.getTablet(Mockito.any())).thenReturn(getTabletResponse(tablet)); + + TInvertedIndexFileStorageFormat format = new CloudInternalCatalog() + .getPartitionInvertedIndexStorageFormat(TABLE_ID, PARTITION_ID, INDEX_ID, 100L); + + Assert.assertEquals(TInvertedIndexFileStorageFormat.V3, format); + } + } + + @Test + public void testGetTabletSchemaVersionRejectsInconsistentVersions() throws Exception { + OlapFile.TabletMetaCloudPB tabletMeta = tabletMeta(100L, + OlapFile.InvertedIndexStorageFormatPB.V2, true).toBuilder().setSchemaVersion(8).build(); + + try { + new CloudInternalCatalog().getTabletSchemaVersion(tabletMeta); + Assert.fail("inconsistent tablet meta and schema versions must fail"); + } catch (DdlException e) { + Assert.assertTrue(e.getMessage().contains("inconsistent schema versions")); + } + } + + private Cloud.GetTabletResponse getTabletResponse(OlapFile.TabletMetaCloudPB tabletMeta) { + return Cloud.GetTabletResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder().setCode(Cloud.MetaServiceCode.OK)) + .setTabletMeta(tabletMeta) + .build(); + } + + private OlapFile.TabletMetaCloudPB tabletMeta(long tabletId, + OlapFile.InvertedIndexStorageFormatPB invertedIndexStorageFormat, boolean withMetaSchemaVersion) { + OlapFile.TabletSchemaCloudPB schema = OlapFile.TabletSchemaCloudPB.newBuilder() + .setSchemaVersion(7) + .setInvertedIndexStorageFormat(invertedIndexStorageFormat) + .build(); + OlapFile.TabletMetaCloudPB.Builder builder = OlapFile.TabletMetaCloudPB.newBuilder() + .setTableId(TABLE_ID) + .setPartitionId(PARTITION_ID) + .setIndexId(INDEX_ID) + .setTabletId(tabletId) + .setSchemaHash(123) + .setSchema(schema); + if (withMetaSchemaVersion) { + builder.setSchemaVersion(7); + } + return builder.build(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java index 2f196d8554fb08..b5dcdce746203f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java @@ -330,7 +330,11 @@ public void testStoragePageSize() throws AnalysisException { @Test public void testAnalyzeInvertedIndexFileStorageFormat() throws AnalysisException { String originFormat = Config.inverted_index_storage_format; + String originDeployMode = Config.deploy_mode; + String originCloudUniqueId = Config.cloud_unique_id; try { + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; Config.inverted_index_storage_format = "V3"; TInvertedIndexFileStorageFormat result = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(null); Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result); @@ -376,8 +380,60 @@ public void testAnalyzeInvertedIndexFileStorageFormat() throws AnalysisException "errCode = 2, detailMessage = unknown inverted index storage format: unknown_format", e.getMessage()); } + + Map propertiesWithPartitionV3 = new HashMap<>(); + propertiesWithPartitionV3.put(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, "v3"); + result = PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat(propertiesWithPartitionV3); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V3, result); + Assertions.assertTrue(propertiesWithPartitionV3.isEmpty()); + + Map propertiesWithPartitionDefault = new HashMap<>(); + propertiesWithPartitionDefault.put(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, + "default"); + Config.inverted_index_storage_format = "V2"; + result = PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat(propertiesWithPartitionDefault); + Assertions.assertEquals(TInvertedIndexFileStorageFormat.V2, result); + + Map propertiesWithPartitionDefaultV1 = new HashMap<>(); + propertiesWithPartitionDefaultV1.put(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, + "default"); + Config.inverted_index_storage_format = "V1"; + try { + PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat(propertiesWithPartitionDefaultV1); + Assertions.fail("Expected an AnalysisException to be thrown"); + } catch (AnalysisException e) { + Assertions.assertEquals("errCode = 2, detailMessage = partition inverted index storage format " + + "only supports V2 and V3", e.getMessage()); + } + + Assertions.assertNull(PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat(new HashMap<>())); + + Map propertiesWithPartitionV1 = new HashMap<>(); + propertiesWithPartitionV1.put(PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, "v1"); + try { + PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat(propertiesWithPartitionV1); + Assertions.fail("Expected an AnalysisException to be thrown"); + } catch (AnalysisException e) { + Assertions.assertEquals("errCode = 2, detailMessage = unknown inverted index storage format: v1", + e.getMessage()); + } + + Config.deploy_mode = "local"; + Map propertiesWithPartitionFormatInLocalMode = new HashMap<>(); + propertiesWithPartitionFormatInLocalMode.put( + PropertyAnalyzer.PROPERTIES_PARTITION_INVERTED_INDEX_STORAGE_FORMAT, "v3"); + try { + PropertyAnalyzer.analyzePartitionInvertedIndexFileStorageFormat( + propertiesWithPartitionFormatInLocalMode); + Assertions.fail("Expected an AnalysisException to be thrown"); + } catch (AnalysisException e) { + Assertions.assertEquals("errCode = 2, detailMessage = partition.inverted_index_storage_format " + + "is only supported in cloud mode", e.getMessage()); + } } finally { Config.inverted_index_storage_format = originFormat; + Config.deploy_mode = originDeployMode; + Config.cloud_unique_id = originCloudUniqueId; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java index ffa9aba39de660..e7969d5d184949 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java @@ -32,6 +32,7 @@ import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; +import org.apache.doris.common.Config; import org.apache.doris.common.ConfigBase; import org.apache.doris.common.ConfigException; import org.apache.doris.common.DdlException; @@ -242,6 +243,28 @@ public void testNormal() throws DdlException, ConfigException { Assertions.assertEquals(tbl13.getSequenceMapCol(), "v1"); } + @Test + public void testPartitionInvertedIndexStorageFormatRejectedInLocalMode() { + String originDeployMode = Config.deploy_mode; + String originCloudUniqueId = Config.cloud_unique_id; + try { + Config.deploy_mode = "local"; + Config.cloud_unique_id = ""; + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> createTable("create table test.partition_inverted_index_storage_format_local\n" + + "(k1 int, v1 varchar(10))\n" + + "duplicate key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1', " + + "'partition.inverted_index_storage_format' = 'V3')")); + Assertions.assertTrue(exception.getMessage().contains( + "partition.inverted_index_storage_format is only supported in cloud mode")); + } finally { + Config.deploy_mode = originDeployMode; + Config.cloud_unique_id = originCloudUniqueId; + } + } + @Test public void testAbnormal() throws ConfigException { checkThrow(org.apache.doris.common.DdlException.class, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommandTest.java new file mode 100644 index 00000000000000..063d32aa8e2476 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowPartitionIdCommandTest.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.common.Config; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; + +public class ShowPartitionIdCommandTest { + private static final long DB_ID = 1L; + private static final long TABLE_ID = 2L; + private static final long PARTITION_ID = 3L; + + @Test + void testCloudFormatComesFromPartitionInfo() throws Exception { + String originalDeployMode = Config.deploy_mode; + String originalCloudUniqueId = Config.cloud_unique_id; + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + + Env env = Mockito.mock(Env.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + Partition partition = Mockito.mock(Partition.class); + PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); + ConnectContext context = Mockito.mock(ConnectContext.class); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(accessManager.checkGlobalPriv(Mockito.same(context), Mockito.eq(PrivPredicate.ADMIN))) + .thenReturn(true); + Mockito.when(context.getEnv()).thenReturn(env); + Mockito.when(env.getInternalCatalog()).thenReturn(catalog); + Mockito.when(catalog.getDbIds()).thenReturn(Collections.singletonList(DB_ID)); + Mockito.when(catalog.getDbNullable(DB_ID)).thenReturn(database); + Mockito.when(database.getTables()).thenReturn(Collections.singletonList(table)); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getId()).thenReturn(DB_ID); + Mockito.when(table.getPartition(PARTITION_ID)).thenReturn(partition); + Mockito.when(table.getName()).thenReturn("tbl"); + Mockito.when(table.getId()).thenReturn(TABLE_ID); + Mockito.when(partition.getName()).thenReturn("p1"); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + Mockito.when(partitionInfo.getInvertedIndexFileStorageFormat(PARTITION_ID)) + .thenReturn(TInvertedIndexFileStorageFormat.V2); + + try (MockedStatic envMock = Mockito.mockStatic(Env.class); + MockedStatic contextMock = Mockito.mockStatic(ConnectContext.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(env); + contextMock.when(ConnectContext::get).thenReturn(context); + + ShowResultSet result = new ShowPartitionIdCommand(PARTITION_ID).doRun(context, null); + + Assertions.assertEquals("V2", result.getResultRows().get(0).get(5)); + Mockito.verify(partitionInfo).getInvertedIndexFileStorageFormat(PARTITION_ID); + } finally { + Config.deploy_mode = originalDeployMode; + Config.cloud_unique_id = originalCloudUniqueId; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyDynamicPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyDynamicPartitionInfoTest.java index a54f91dd8267fd..506cafbbf05dee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyDynamicPartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyDynamicPartitionInfoTest.java @@ -18,6 +18,7 @@ package org.apache.doris.persist; import org.apache.doris.catalog.DynamicPartitionProperty; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; import org.junit.After; import org.junit.Assert; @@ -30,6 +31,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.HashMap; +import java.util.Map; public class ModifyDynamicPartitionInfoTest { private String fileName = "./ModifyTablePropertyOperationLogTest"; @@ -80,4 +82,23 @@ public void testToSql() { Assert.assertTrue(modifyDynamicPartitionInfo.toSql().contains("\"dynamic_partition.time_unit\" = \"day\"")); Assert.assertTrue(modifyDynamicPartitionInfo.toSql().contains("\"dynamic_partition.start\" = \"-3\"")); } + + @Test + public void testPartitionInvertedIndexStorageFormatPersistence() throws IOException { + Map formats = Map.of( + 10L, TInvertedIndexFileStorageFormat.V2, + 20L, TInvertedIndexFileStorageFormat.V3); + ModifyTablePropertyOperationLog log = new ModifyTablePropertyOperationLog(100L, 200L, "test", + Map.of("partition.inverted_index_storage_format", "V3"), Map.of(), formats); + + File file = new File(fileName); + file.createNewFile(); + try (DataOutputStream output = new DataOutputStream(new FileOutputStream(file))) { + log.write(output); + } + try (DataInputStream input = new DataInputStream(new FileInputStream(file))) { + Assert.assertEquals(formats, + ModifyTablePropertyOperationLog.read(input).getPartitionIdToInvertedIndexFileStorageFormat()); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplCloudTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplCloudTest.java index a5a28080315ca9..40999f874531db 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplCloudTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendServiceImplCloudTest.java @@ -17,10 +17,15 @@ package org.apache.doris.service; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.cloud.CacheHotspotManager; import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.common.Config; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.nereids.trees.plans.commands.info.AddPartitionLikeOp; +import org.apache.doris.thrift.TAddOrDropPartitionsRequest; import org.apache.doris.thrift.TGetTabletReplicaInfosRequest; import org.apache.doris.thrift.TGetTabletReplicaInfosResult; import org.apache.doris.thrift.TStatusCode; @@ -30,6 +35,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.lang.reflect.Method; import java.util.Collections; public class FrontendServiceImplCloudTest { @@ -75,4 +81,73 @@ public void testGetTabletReplicaInfosNullJobReturnsCancelledWithoutNpe() { Config.cloud_unique_id = originalCloudUniqueId; } } + + @Test + public void testAddPartitionForRemoteInsertOverwritePreservesSourceSchema() throws Exception { + Env env = Mockito.mock(Env.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(catalog.getDbNullable("db")).thenReturn(database); + Mockito.when(database.getTableNullable("table")).thenReturn(table); + Mockito.when(table.writeLockIfExist()).thenReturn(true); + Mockito.when(table.getName()).thenReturn("table"); + + try (MockedStatic envMock = Mockito.mockStatic(Env.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(env); + envMock.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + + invokeAddOrDropPartitions(newAddPartitionRequest(), true); + + Mockito.verify(env).addPartitionLikeForInsertOverwrite(Mockito.same(database), Mockito.eq("table"), + Mockito.any(AddPartitionLikeOp.class)); + Mockito.verify(env, Mockito.never()).addPartitionLike(Mockito.any(Database.class), Mockito.anyString(), + Mockito.any(AddPartitionLikeOp.class)); + Mockito.verify(table).writeUnlock(); + } + } + + @Test + public void testAddPartitionForExistingRemoteRequestUsesDefaultSchema() throws Exception { + Env env = Mockito.mock(Env.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Database database = Mockito.mock(Database.class); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(catalog.getDbNullable("db")).thenReturn(database); + Mockito.when(database.getTableNullable("table")).thenReturn(table); + Mockito.when(table.writeLockIfExist()).thenReturn(true); + Mockito.when(table.getName()).thenReturn("table"); + + try (MockedStatic envMock = Mockito.mockStatic(Env.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(env); + envMock.when(Env::getCurrentInternalCatalog).thenReturn(catalog); + + invokeAddOrDropPartitions(newAddPartitionRequest(), false); + + Mockito.verify(env).addPartitionLike(Mockito.same(database), Mockito.eq("table"), + Mockito.any(AddPartitionLikeOp.class)); + Mockito.verify(env, Mockito.never()).addPartitionLikeForInsertOverwrite(Mockito.any(Database.class), + Mockito.anyString(), Mockito.any(AddPartitionLikeOp.class)); + Mockito.verify(table).writeUnlock(); + } + } + + private TAddOrDropPartitionsRequest newAddPartitionRequest() { + TAddOrDropPartitionsRequest request = new TAddOrDropPartitionsRequest(); + request.setDb("db"); + request.setTbl("table"); + request.setPartitionNames(Collections.singletonList("source_partition")); + request.setTempPartitionNames(Collections.singletonList("temp_partition")); + request.setIsDrop(false); + request.setIsTemp(true); + return request; + } + + private void invokeAddOrDropPartitions(TAddOrDropPartitionsRequest request, boolean isInsertOverwrite) + throws Exception { + Method method = FrontendServiceImpl.class.getDeclaredMethod("addOrDropPartitionsImpl", + TAddOrDropPartitionsRequest.class, boolean.class); + method.setAccessible(true); + method.invoke(new FrontendServiceImpl(Mockito.mock(ExecuteEnv.class)), request, isInsertOverwrite); + } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 39bdbf022fad34..670fd7b8c79a4a 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -2073,6 +2073,7 @@ service FrontendService { TMasterAddressResult getMasterAddress(1: TMasterAddressRequest request) TAddOrDropPartitionsResult addOrDropPartitions(1: TAddOrDropPartitionsRequest request) + TAddOrDropPartitionsResult addPartitionsForInsertOverwrite(1: TAddOrDropPartitionsRequest request) TReplacePartitionsResult replacePartitions(1: TReplacePartitionsRequest request) TInsertOverwriteRegisterResult registerInsertOverwriteTask(1: TInsertOverwriteRegisterRequest request) diff --git a/regression-test/data/external_table_p0/tvf/test_partitions_tvf.out b/regression-test/data/external_table_p0/tvf/test_partitions_tvf.out index afc87c82420995..e732af15431adb 100644 --- a/regression-test/data/external_table_p0/tvf/test_partitions_tvf.out +++ b/regression-test/data/external_table_p0/tvf/test_partitions_tvf.out @@ -1,9 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !desc -- Buckets int No false \N NONE +CommittedVersion bigint No false \N NONE CooldownTime text No false \N NONE DataSize text No false \N NONE DistributionKey text No false \N NONE +InvertedIndexStorageFormat text No false \N NONE IsInMemory boolean No false \N NONE IsMutable boolean No false \N NONE LastConsistencyCheckTime text No false \N NONE @@ -14,6 +16,7 @@ Range text No false \N NONE RemoteStoragePolicy text No false \N NONE ReplicaAllocation text No false \N NONE ReplicationNum int No false \N NONE +RowCount bigint No false \N NONE State text No false \N NONE StorageMedium text No false \N NONE SyncWithBaseTables boolean No false \N NONE diff --git a/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_format.groovy b/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_format.groovy new file mode 100644 index 00000000000000..5b3b164003e245 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_format.groovy @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.Http + +suite("test_cloud_show_inverted_index_storage_format", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + + docker(options) { + def assertPartitionFormats = { tableName, expectedPartitionCount, expectedFormat -> + def partitions = sql_return_maparray("SHOW PARTITIONS FROM ${tableName}") + assertEquals(expectedPartitionCount, partitions.size()) + partitions.each { partition -> + assertEquals(expectedFormat, partition.InvertedIndexStorageFormat) + def partitionDetails = sql_return_maparray("SHOW PARTITION ${partition.PartitionId}") + assertEquals(1, partitionDetails.size()) + assertEquals(expectedFormat, partitionDetails[0].InvertedIndexStorageFormat) + + def tablets = sql_return_maparray( + "SHOW TABLETS FROM ${tableName} PARTITION(${partition.PartitionName})") + assertTrue(!tablets.isEmpty()) + tablets.each { tablet -> + def tabletMeta = Http.GET(tablet.MetaUrl, true, false) + assertEquals(expectedFormat, tabletMeta.schema.inverted_index_storage_format) + } + } + } + + sql "DROP TABLE IF EXISTS test_cloud_show_inverted_format_range" + sql """ + CREATE TABLE test_cloud_show_inverted_format_range ( + k INT, + v VARCHAR(20), + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k) + PARTITION BY RANGE(k) ( + PARTITION p1 VALUES [("0"), ("10")), + PARTITION p2 VALUES [("10"), ("20")), + PARTITION p3 VALUES [("20"), ("30")) + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_cloud_show_inverted_format_range VALUES + (1, 'range-1a'), (2, 'range-1b'), (3, 'range-1c'), + (11, 'range-2a'), (12, 'range-2b'), (13, 'range-2c'), + (21, 'range-3a'), (22, 'range-3b'), (23, 'range-3c') + """ + + sql "DROP TABLE IF EXISTS test_cloud_show_inverted_format_list" + sql """ + CREATE TABLE test_cloud_show_inverted_format_list ( + k INT, + category VARCHAR(20), + INDEX idx_category (category) USING INVERTED + ) + DUPLICATE KEY(k, category) + PARTITION BY LIST(category) ( + PARTITION p_east VALUES IN ('east', 'north'), + PARTITION p_west VALUES IN ('west', 'south'), + PARTITION p_central VALUES IN ('central', 'overseas') + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ) + """ + sql """ + INSERT INTO test_cloud_show_inverted_format_list VALUES + (1, 'east'), (2, 'north'), (3, 'west'), + (4, 'south'), (5, 'central'), (6, 'overseas') + """ + + sql "DROP TABLE IF EXISTS test_cloud_show_inverted_format_auto" + sql """ + CREATE TABLE test_cloud_show_inverted_format_auto ( + k INT, + category VARCHAR(20), + INDEX idx_category (category) USING INVERTED + ) + DUPLICATE KEY(k, category) + AUTO PARTITION BY LIST(category) () + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_cloud_show_inverted_format_auto VALUES + (1, 'book'), (2, 'book'), (3, 'music'), + (4, 'music'), (5, 'video'), (6, 'video') + """ + + sql "DROP TABLE IF EXISTS test_cloud_show_inverted_format_dynamic" + sql """ + CREATE TABLE test_cloud_show_inverted_format_dynamic ( + k INT, + dt DATE, + v VARCHAR(20), + INDEX idx_v (v) USING INVERTED + ) + DUPLICATE KEY(k, dt) + PARTITION BY RANGE(dt) () + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-3", + "dynamic_partition.end" = "3", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.create_history_partition" = "true", + "inverted_index_storage_format" = "V3" + ) + """ + sql """ + INSERT INTO test_cloud_show_inverted_format_dynamic + SELECT 1, date_sub(curdate(), INTERVAL 3 DAY), 'dynamic-0' + UNION ALL SELECT 2, date_sub(curdate(), INTERVAL 2 DAY), 'dynamic-1a' + UNION ALL SELECT 3, date_sub(curdate(), INTERVAL 2 DAY), 'dynamic-1b' + UNION ALL SELECT 4, date_sub(curdate(), INTERVAL 1 DAY), 'dynamic-2a' + UNION ALL SELECT 5, date_sub(curdate(), INTERVAL 1 DAY), 'dynamic-2b' + UNION ALL SELECT 6, curdate(), 'dynamic-3a' + UNION ALL SELECT 7, curdate(), 'dynamic-3b' + UNION ALL SELECT 8, date_add(curdate(), INTERVAL 1 DAY), 'dynamic-4' + UNION ALL SELECT 9, date_add(curdate(), INTERVAL 2 DAY), 'dynamic-5' + UNION ALL SELECT 10, date_add(curdate(), INTERVAL 3 DAY), 'dynamic-6' + """ + sql "SYNC" + + def rangeCount = sql "SELECT COUNT(*) FROM test_cloud_show_inverted_format_range" + def listCount = sql "SELECT COUNT(*) FROM test_cloud_show_inverted_format_list" + def autoCount = sql "SELECT COUNT(*) FROM test_cloud_show_inverted_format_auto" + def dynamicCount = sql "SELECT COUNT(*) FROM test_cloud_show_inverted_format_dynamic" + assertEquals(9L, rangeCount[0][0]) + assertEquals(6L, listCount[0][0]) + assertEquals(6L, autoCount[0][0]) + assertEquals(10L, dynamicCount[0][0]) + + assertPartitionFormats("test_cloud_show_inverted_format_range", 3, "V3") + assertPartitionFormats("test_cloud_show_inverted_format_list", 3, "V2") + assertPartitionFormats("test_cloud_show_inverted_format_auto", 3, "V3") + assertPartitionFormats("test_cloud_show_inverted_format_dynamic", 7, "V3") + + } +} diff --git a/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_storage_format_recycle_truncate.groovy b/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_storage_format_recycle_truncate.groovy new file mode 100644 index 00000000000000..9e970f9524aee9 --- /dev/null +++ b/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_storage_format_recycle_truncate.groovy @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.Http + +suite("test_partition_cloud_inverted_index_storage_format_recycle_truncate", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + options.extraHosts += [ + 'host.docker.internal:host-gateway', + 'metrics-test-bucket.host.docker.internal:host-gateway', + ] + + docker(options) { + def assertPartitionFormat = { tableName, partitionName, expectedFormat -> + def partitions = sql_return_maparray( + "SHOW PARTITIONS FROM ${tableName} WHERE PartitionName = '${partitionName}'") + assertEquals(1, partitions.size()) + assertEquals(expectedFormat, partitions[0].InvertedIndexStorageFormat) + def partitionDetails = sql_return_maparray("SHOW PARTITION ${partitions[0].PartitionId}") + assertEquals(1, partitionDetails.size()) + assertEquals(expectedFormat, partitionDetails[0].InvertedIndexStorageFormat) + + def tablets = sql_return_maparray( + "SHOW TABLETS FROM ${tableName} PARTITION(${partitionName})") + assertTrue(!tablets.isEmpty()) + def tabletMetas = tablets.collect { tablet -> + Http.GET(tablet.MetaUrl, true, false) + } + def baseTabletMetas = tabletMetas.findAll { tabletMeta -> + tabletMeta.schema.index?.any { index -> index.index_type == "INVERTED" } + } + def rollupTabletMetas = tabletMetas.findAll { tabletMeta -> + !tabletMeta.schema.index?.any { index -> index.index_type == "INVERTED" } + } + assertEquals(1, baseTabletMetas.size()) + assertEquals(1, rollupTabletMetas.size()) + baseTabletMetas.each { tabletMeta -> + assertEquals(expectedFormat, tabletMeta.schema.inverted_index_storage_format) + } + rollupTabletMetas.each { tabletMeta -> + assertTrue(!tabletMeta.schema.index?.any { index -> index.index_type == "INVERTED" }) + } + } + + def getTabletIds = { tableName, partitionName -> + def tablets = sql_return_maparray( + "SHOW TABLETS FROM ${tableName} PARTITION(${partitionName})") + assertTrue(!tablets.isEmpty()) + def baseTablets = tablets.findAll { tablet -> + def tabletMeta = Http.GET(tablet.MetaUrl, true, false) + tabletMeta.schema.index?.any { index -> index.index_type == "INVERTED" } + } + assertEquals(1, baseTablets.size()) + baseTablets.collect { it.TabletId } + } + + def assertMixedPartitionFormats = { tableName -> + assertPartitionFormat(tableName, "p_v2", "V2") + assertPartitionFormat(tableName, "p_v3", "V3") + } + + sql "DROP TABLE IF EXISTS test_cloud_partition_inv_idx_fmt_recycle_trunc" + sql """ + CREATE TABLE test_cloud_partition_inv_idx_fmt_recycle_trunc ( + k DATE NOT NULL, + v VARCHAR(100) NULL, + w INT NULL, + INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") + ) ENGINE=OLAP + DUPLICATE KEY(k) + PARTITION BY RANGE(k) ( + PARTITION p_v2 VALUES LESS THAN ("2024-01-01") + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ) + """ + sql """ALTER TABLE test_cloud_partition_inv_idx_fmt_recycle_trunc + ADD ROLLUP r_v(v, w)""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE ROLLUP + WHERE TableName = 'test_cloud_partition_inv_idx_fmt_recycle_trunc' + ORDER BY CreateTime DESC LIMIT 1""" + time 600 + } + assertPartitionFormat("test_cloud_partition_inv_idx_fmt_recycle_trunc", "p_v2", "V2") + + sql """ALTER TABLE test_cloud_partition_inv_idx_fmt_recycle_trunc + SET ("partition.inverted_index_storage_format" = "V3")""" + sql """ALTER TABLE test_cloud_partition_inv_idx_fmt_recycle_trunc + ADD PARTITION p_v3 VALUES [("2024-01-01"), ("2025-01-01"))""" + sql """ + INSERT INTO test_cloud_partition_inv_idx_fmt_recycle_trunc VALUES + ("2023-01-01", "v2 recycle token", 1), + ("2024-01-01", "v3 recycle token", 2) + """ + assertMixedPartitionFormats("test_cloud_partition_inv_idx_fmt_recycle_trunc") + + sql """ALTER TABLE test_cloud_partition_inv_idx_fmt_recycle_trunc + DROP PARTITION p_v2""" + sql """RECOVER PARTITION p_v2 FROM test_cloud_partition_inv_idx_fmt_recycle_trunc""" + assertMixedPartitionFormats("test_cloud_partition_inv_idx_fmt_recycle_trunc") + + sql "DROP TABLE test_cloud_partition_inv_idx_fmt_recycle_trunc" + sql "RECOVER TABLE test_cloud_partition_inv_idx_fmt_recycle_trunc" + assertMixedPartitionFormats("test_cloud_partition_inv_idx_fmt_recycle_trunc") + + def v2TabletIdsBeforeTruncate = getTabletIds( + "test_cloud_partition_inv_idx_fmt_recycle_trunc", "p_v2") + def v3TabletIdsBeforeTruncate = getTabletIds( + "test_cloud_partition_inv_idx_fmt_recycle_trunc", "p_v3") + sql "TRUNCATE TABLE test_cloud_partition_inv_idx_fmt_recycle_trunc" + def v2TabletIdsAfterTruncate = getTabletIds( + "test_cloud_partition_inv_idx_fmt_recycle_trunc", "p_v2") + def v3TabletIdsAfterTruncate = getTabletIds( + "test_cloud_partition_inv_idx_fmt_recycle_trunc", "p_v3") + assertEquals(v2TabletIdsBeforeTruncate.size(), v2TabletIdsAfterTruncate.size()) + assertEquals(v3TabletIdsBeforeTruncate.size(), v3TabletIdsAfterTruncate.size()) + assertTrue(v2TabletIdsBeforeTruncate.every { tabletId -> !v2TabletIdsAfterTruncate.contains(tabletId) }) + assertTrue(v3TabletIdsBeforeTruncate.every { tabletId -> !v3TabletIdsAfterTruncate.contains(tabletId) }) + assertMixedPartitionFormats("test_cloud_partition_inv_idx_fmt_recycle_trunc") + + sql """ + INSERT INTO test_cloud_partition_inv_idx_fmt_recycle_trunc VALUES + ("2023-01-02", "v2 truncate token", 1), + ("2024-01-02", "v3 truncate token", 2) + """ + def rows = sql """ + SELECT CAST(k AS STRING), v + FROM test_cloud_partition_inv_idx_fmt_recycle_trunc + WHERE v MATCH_ANY "truncate" ORDER BY k + """ + assertEquals([[ + "2023-01-02", "v2 truncate token" + ], [ + "2024-01-02", "v3 truncate token" + ]], rows) + } +} diff --git a/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_storage_format_rollout.groovy b/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_storage_format_rollout.groovy new file mode 100644 index 00000000000000..4e7c999166bfaa --- /dev/null +++ b/regression-test/suites/cloud_p0/test_partition_cloud_inverted_index_storage_format_rollout.groovy @@ -0,0 +1,453 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.Http + +suite("test_partition_cloud_inverted_index_storage_format_rollout", "p0, docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + options.extraHosts += [ + 'host.docker.internal:host-gateway', + 'metrics-test-bucket.host.docker.internal:host-gateway', + ] + options.feConfigs += [ + 'enable_debug_points=true', + ] + + docker(options) { + def assertPartitionFormat = { tableName, partitionName, expectedFormat -> + def partitions = sql_return_maparray( + "SHOW PARTITIONS FROM ${tableName} WHERE PartitionName = '${partitionName}'") + assertEquals(1, partitions.size()) + assertEquals(expectedFormat, partitions[0].InvertedIndexStorageFormat) + def partitionDetails = sql_return_maparray("SHOW PARTITION ${partitions[0].PartitionId}") + assertEquals(1, partitionDetails.size()) + assertEquals(expectedFormat, partitionDetails[0].InvertedIndexStorageFormat) + + def tablets = sql_return_maparray( + "SHOW TABLETS FROM ${tableName} PARTITION(${partitionName})") + assertTrue(!tablets.isEmpty()) + def baseTabletMetas = tablets.collect { tablet -> + Http.GET(tablet.MetaUrl, true, false) + }.findAll { tabletMeta -> + tabletMeta.schema.index?.any { index -> index.index_type == "INVERTED" } + } + assertEquals(1, baseTabletMetas.size()) + baseTabletMetas.each { tabletMeta -> + assertEquals(expectedFormat, tabletMeta.schema.inverted_index_storage_format) + } + } + + def getBaseTabletMeta = { tableName, partitionName -> + def tablets = sql_return_maparray("SHOW TABLETS FROM ${tableName} PARTITION(${partitionName})") + assertTrue(!tablets.isEmpty()) + def baseTabletMetas = tablets.collect { tablet -> + Http.GET(tablet.MetaUrl, true, false) + }.findAll { tabletMeta -> + tabletMeta.schema.index?.any { index -> index.index_type == "INVERTED" } + } + assertEquals(1, baseTabletMetas.size()) + return baseTabletMetas[0] + } + + def assertBaseTabletColumns = { tableName, partitionName, expectedColumns -> + def actualColumns = getBaseTabletMeta(tableName, partitionName).schema.column.collect { column -> + [column.name, column.type == "DATEV2" ? "DATE" : column.type] + } + assertEquals(expectedColumns, actualColumns) + } + + sql "DROP TABLE IF EXISTS test_cloud_partition_inverted_index_storage_format_rollout" + sql "DROP TABLE IF EXISTS test_cloud_partition_inverted_index_storage_format_initial" + sql "DROP TABLE IF EXISTS test_cloud_partition_inverted_index_storage_format_list" + sql "DROP TABLE IF EXISTS test_cloud_partition_inverted_index_storage_format_auto" + sql "DROP TABLE IF EXISTS test_cloud_partition_inverted_index_storage_format_schema_change" + sql "DROP TABLE IF EXISTS test_cloud_partition_inverted_index_storage_format_dynamic" + + sql """ + CREATE TABLE test_cloud_partition_inverted_index_storage_format_rollout ( + k DATE NOT NULL, + v VARCHAR(100) NULL, + INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") + ) ENGINE=OLAP + DUPLICATE KEY(k) + PARTITION BY RANGE(k) ( + PARTITION p_old VALUES LESS THAN ("2024-01-01") + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ) + """ + + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_rollout ADD PARTITION p_default VALUES [("2026-01-01"), ("2027-01-01"))""" + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_rollout", "p_old", "V2") + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_rollout", "p_default", "V2") + + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_rollout SET ("partition.inverted_index_storage_format" = "V3")""" + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_rollout ADD PARTITION p_new VALUES [("2024-01-01"), ("2025-01-01"))""" + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_rollout", "p_new", "V3") + + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_rollout SET ("partition.inverted_index_storage_format" = "V2")""" + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_rollout ADD PARTITION p_downgrade VALUES [("2025-01-01"), ("2026-01-01"))""" + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_rollout", "p_downgrade", "V2") + + sql "set enable_memtable_on_sink_node = true" + try { + sql """ + INSERT INTO test_cloud_partition_inverted_index_storage_format_rollout VALUES + ("2023-01-01", "old token"), + ("2024-01-01", "new token"), + ("2025-01-01", "downgrade token") + """ + } finally { + sql "set enable_memtable_on_sink_node = false" + } + def mixedFormatRows = sql """ + SELECT CAST(k AS STRING), v + FROM test_cloud_partition_inverted_index_storage_format_rollout + WHERE v MATCH_ANY 'token' ORDER BY k + """ + assertEquals([ + ["2023-01-01", "old token"], + ["2024-01-01", "new token"], + ["2025-01-01", "downgrade token"] + ], mixedFormatRows) + + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_rollout SET ("partition.inverted_index_storage_format" = "V3")""" + sql """TRUNCATE TABLE test_cloud_partition_inverted_index_storage_format_rollout PARTITION p_old""" + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_rollout", "p_old", "V2") + sql """INSERT OVERWRITE TABLE test_cloud_partition_inverted_index_storage_format_rollout PARTITION(p_old) VALUES ("2023-01-02", "old rewritten")""" + def rewrittenRows = sql """ + SELECT CAST(k AS STRING), v + FROM test_cloud_partition_inverted_index_storage_format_rollout + WHERE v MATCH_ANY 'rewritten' ORDER BY k + """ + assertEquals([["2023-01-02", "old rewritten"]], rewrittenRows) + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_rollout", "p_old", "V2") + + sql """ + CREATE TABLE test_cloud_partition_inverted_index_storage_format_initial ( + k DATE NOT NULL, + v VARCHAR(100) NULL, + INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") + ) ENGINE=OLAP + DUPLICATE KEY(k) + PARTITION BY RANGE(k) ( + PARTITION p_initial VALUES LESS THAN ("2024-01-01") + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2", + "partition.inverted_index_storage_format" = "V3" + ) + """ + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_initial", "p_initial", "V3") + + sql """ + CREATE TABLE test_cloud_partition_inverted_index_storage_format_list ( + k INT NOT NULL, + category VARCHAR(20) NOT NULL, + v VARCHAR(100) NULL, + INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") + ) ENGINE=OLAP + DUPLICATE KEY(k, category) + PARTITION BY LIST(category) ( + PARTITION p_old VALUES IN ("old") + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ) + """ + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_list", "p_old", "V2") + + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_list + SET ("partition.inverted_index_storage_format" = "V3")""" + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_list + ADD PARTITION p_new VALUES IN ("new")""" + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_list", "p_old", "V2") + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_list", "p_new", "V3") + + sql """ + INSERT INTO test_cloud_partition_inverted_index_storage_format_list VALUES + (1, "old", "old list token"), + (2, "new", "new list token") + """ + def listRows = sql """ + SELECT category, v + FROM test_cloud_partition_inverted_index_storage_format_list + WHERE v MATCH_ANY 'token' ORDER BY category + """ + assertEquals([[ + "new", "new list token" + ], [ + "old", "old list token" + ]], listRows) + + sql """ + CREATE TABLE test_cloud_partition_inverted_index_storage_format_auto ( + k VARCHAR(20) NOT NULL, + v VARCHAR(100) NULL, + INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") + ) ENGINE=OLAP + DUPLICATE KEY(k) + AUTO PARTITION BY LIST(k) () + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ) + """ + sql """INSERT INTO test_cloud_partition_inverted_index_storage_format_auto VALUES ("old", "auto old")""" + def autoPartitions = sql_return_maparray( + "SHOW PARTITIONS FROM test_cloud_partition_inverted_index_storage_format_auto") + assertEquals(1, autoPartitions.size()) + def oldAutoPartitionName = autoPartitions[0].PartitionName + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_auto", oldAutoPartitionName, "V2") + + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_auto SET ("partition.inverted_index_storage_format" = "V3")""" + sql """INSERT INTO test_cloud_partition_inverted_index_storage_format_auto VALUES ("new", "auto new")""" + autoPartitions = sql_return_maparray( + "SHOW PARTITIONS FROM test_cloud_partition_inverted_index_storage_format_auto") + assertEquals(2, autoPartitions.size()) + def newAutoPartition = autoPartitions.find { it.PartitionName != oldAutoPartitionName } + assertNotNull(newAutoPartition) + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_auto", newAutoPartition.PartitionName, "V3") + def autoRows = sql """ + SELECT k, v FROM test_cloud_partition_inverted_index_storage_format_auto + WHERE v MATCH_ANY 'auto' ORDER BY k + """ + assertEquals([["new", "auto new"], ["old", "auto old"]], autoRows) + + sql """ + CREATE TABLE test_cloud_partition_inverted_index_storage_format_schema_change ( + k DATE NOT NULL, + c1 INT NULL, + c2 VARCHAR(100) NULL, + INDEX idx_c2 (c2) USING INVERTED PROPERTIES("parser" = "english") + ) ENGINE=OLAP + DUPLICATE KEY(k) + PARTITION BY RANGE(k) ( + PARTITION p_v2 VALUES LESS THAN ("2024-01-01") + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2", + "light_schema_change" = "true" + ) + """ + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_schema_change + ADD COLUMN c3 INT NULL""" + assertBaseTabletColumns( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v2", + [["k", "DATE"], ["c1", "INT"], ["c2", "VARCHAR"]]) + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_schema_change + SET ("partition.inverted_index_storage_format" = "V3")""" + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_schema_change + ADD PARTITION p_v3 VALUES [("2024-01-01"), ("2025-01-01"))""" + assertBaseTabletColumns( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v3", + [["k", "DATE"], ["c1", "INT"], ["c2", "VARCHAR"], ["c3", "INT"]]) + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_schema_change + DROP PARTITION p_v2""" + sql """RECOVER PARTITION p_v2 FROM test_cloud_partition_inverted_index_storage_format_schema_change""" + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v2", "V2") + assertBaseTabletColumns( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v2", + [["k", "DATE"], ["c1", "INT"], ["c2", "VARCHAR"]]) + sql """ + INSERT INTO test_cloud_partition_inverted_index_storage_format_schema_change (k, c1, c2) VALUES + ("2023-01-01", 1, "v2 schema change token") + """ + sql """ + INSERT INTO test_cloud_partition_inverted_index_storage_format_schema_change VALUES + ("2024-01-01", 2, "v3 schema change token", 3) + """ + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_schema_change + ADD ROLLUP r_schema_change(c1, c2)""" + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE ROLLUP + WHERE TableName = 'test_cloud_partition_inverted_index_storage_format_schema_change' + ORDER BY CreateTime DESC LIMIT 1""" + time 600 + } + def pV2SourceSchemaVersion = getBaseTabletMeta( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v2").schema.schema_version + def pV3SourceSchemaVersion = getBaseTabletMeta( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v3").schema.schema_version + assertTrue(pV2SourceSchemaVersion < pV3SourceSchemaVersion) + assertBaseTabletColumns( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v2", + [["k", "DATE"], ["c1", "INT"], ["c2", "VARCHAR"]]) + assertBaseTabletColumns( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v3", + [["k", "DATE"], ["c1", "INT"], ["c2", "VARCHAR"], ["c3", "INT"]]) + + def assertShadowIndexes = { partitionName -> + def partitions = sql_return_maparray( + "SHOW PARTITIONS FROM test_cloud_partition_inverted_index_storage_format_schema_change " + + "WHERE PartitionName = '${partitionName}'") + assertEquals(1, partitions.size()) + def tablets = sql_return_maparray( + "SHOW TABLETS FROM test_cloud_partition_inverted_index_storage_format_schema_change " + + "PARTITION(${partitionName})") + assertTrue(!tablets.isEmpty()) + def tabletInfo = sql_return_maparray("SHOW TABLET ${tablets[0].TabletId}") + assertEquals(1, tabletInfo.size()) + def procPath = "/dbs/${tabletInfo[0].DbId}/${tabletInfo[0].TableId}/partitions/${partitions[0].PartitionId}" + def indexes = sql_return_maparray("SHOW PROC \"${procPath}\"") + assertEquals(4, indexes.size()) + assertTrue(indexes.any { it.IndexName == "test_cloud_partition_inverted_index_storage_format_schema_change" }) + assertTrue(indexes.any { it.IndexName == "r_schema_change" }) + def shadowIndexes = indexes.findAll { it.State == "SHADOW" } + assertEquals(2, shadowIndexes.size()) + assertTrue(shadowIndexes.any { + it.IndexName == "__doris_shadow_test_cloud_partition_inverted_index_storage_format_schema_change" + }) + assertTrue(shadowIndexes.any { it.IndexName == "__doris_shadow_r_schema_change" }) + } + + GetDebugPoint().enableDebugPointForAllFEs("FE.SchemaChangeJobV2.runRunning.block") + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_schema_change + MODIFY COLUMN c1 BIGINT NULL""" + def schemaChangeState = "" + for (def retry = 0; retry < 120; retry++) { + def jobs = sql_return_maparray("""SHOW ALTER TABLE COLUMN + WHERE TableName = 'test_cloud_partition_inverted_index_storage_format_schema_change' + ORDER BY CreateTime DESC LIMIT 1""") + if (!jobs.isEmpty()) { + schemaChangeState = jobs[0].State + if (schemaChangeState == "RUNNING") { + break + } + } + sleep(1000) + } + assertEquals("RUNNING", schemaChangeState) + assertShadowIndexes("p_v2") + assertShadowIndexes("p_v3") + + GetDebugPoint().disableDebugPointForAllFEs("FE.SchemaChangeJobV2.runRunning.block") + waitForSchemaChangeDone { + sql """SHOW ALTER TABLE COLUMN + WHERE TableName = 'test_cloud_partition_inverted_index_storage_format_schema_change' + ORDER BY CreateTime DESC LIMIT 1""" + time 600 + } + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v2", "V2") + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v3", "V3") + def pV2ShadowSchemaVersion = getBaseTabletMeta( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v2").schema.schema_version + def pV3ShadowSchemaVersion = getBaseTabletMeta( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v3").schema.schema_version + assertTrue(pV2ShadowSchemaVersion > pV2SourceSchemaVersion) + assertTrue(pV3ShadowSchemaVersion > pV3SourceSchemaVersion) + assertTrue(pV2ShadowSchemaVersion < pV3ShadowSchemaVersion) + assertBaseTabletColumns( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v2", + [["k", "DATE"], ["c1", "BIGINT"], ["c2", "VARCHAR"], ["c3", "INT"]]) + assertBaseTabletColumns( + "test_cloud_partition_inverted_index_storage_format_schema_change", "p_v3", + [["k", "DATE"], ["c1", "BIGINT"], ["c2", "VARCHAR"], ["c3", "INT"]]) + def schemaChangeRows = sql """ + SELECT CAST(k AS STRING), c2 + FROM test_cloud_partition_inverted_index_storage_format_schema_change + WHERE c2 MATCH_ANY 'schema change token' ORDER BY k + """ + assertEquals([ + ["2023-01-01", "v2 schema change token"], + ["2024-01-01", "v3 schema change token"] + ], schemaChangeRows) + + def oldDynamicPartitionCheckInterval = getFeConfig('dynamic_partition_check_interval_seconds') + try { + setFeConfig('dynamic_partition_check_interval_seconds', 1) + sql """ + CREATE TABLE test_cloud_partition_inverted_index_storage_format_dynamic ( + k DATE NOT NULL, + v VARCHAR(100) NULL, + INDEX idx_v (v) USING INVERTED PROPERTIES("parser" = "english") + ) ENGINE=OLAP + DUPLICATE KEY(k) + PARTITION BY RANGE(k) () + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "DAY", + "dynamic_partition.start" = "-1", + "dynamic_partition.end" = "1", + "dynamic_partition.prefix" = "p", + "dynamic_partition.buckets" = "1", + "dynamic_partition.create_history_partition" = "true", + "inverted_index_storage_format" = "V2" + ) + """ + def dynamicPartitions = sql_return_maparray( + "SHOW PARTITIONS FROM test_cloud_partition_inverted_index_storage_format_dynamic") + assertEquals(3, dynamicPartitions.size()) + def initialDynamicPartitionNames = dynamicPartitions.collect { it.PartitionName } + initialDynamicPartitionNames.each { partitionName -> + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_dynamic", partitionName, "V2") + } + + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_dynamic SET ("partition.inverted_index_storage_format" = "V3")""" + sql """ALTER TABLE test_cloud_partition_inverted_index_storage_format_dynamic SET ("dynamic_partition.end" = "3")""" + for (def retry = 0; retry < 120; retry++) { + dynamicPartitions = sql_return_maparray( + "SHOW PARTITIONS FROM test_cloud_partition_inverted_index_storage_format_dynamic") + if (dynamicPartitions.size() == 5) { + break + } + sleep(1000) + } + assertEquals(5, dynamicPartitions.size()) + dynamicPartitions.each { partition -> + def expectedFormat = initialDynamicPartitionNames.contains(partition.PartitionName) ? "V2" : "V3" + assertPartitionFormat( + "test_cloud_partition_inverted_index_storage_format_dynamic", + partition.PartitionName, expectedFormat) + } + } finally { + setFeConfig('dynamic_partition_check_interval_seconds', oldDynamicPartitionCheckInterval) + } + } +} diff --git a/regression-test/suites/external_table_p0/tvf/test_partitions_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_partitions_tvf.groovy index 112a131502f8d8..a6be21f6da88d3 100644 --- a/regression-test/suites/external_table_p0/tvf/test_partitions_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_partitions_tvf.groovy @@ -71,6 +71,12 @@ suite("test_partitions_tvf", "p0,external") { // SyncWithBaseTables assertEquals(true, res[0][18]); assertEquals(true, res2[0][18]); + // CommittedVersion + assertEquals(res[0][20], res2[0][20]); + // RowCount + assertEquals(res[0][21], res2[0][21]); + // InvertedIndexStorageFormat + assertEquals(res[0][22], res2[0][22]); // test exception