Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement alter database in table model #14256

Open
wants to merge 35 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
d8e62e2
partial
Caideyipi Nov 29, 2024
4db0fce
Update ClusterConfigTaskExecutor.java
Caideyipi Nov 29, 2024
9125326
Update ConfigNodeRPCServiceProcessor.java
Caideyipi Nov 29, 2024
14a0e40
refactor
Caideyipi Nov 29, 2024
2bd528f
Merge branch 'master' of https://github.com/apache/iotdb into alter-db
Caideyipi Nov 29, 2024
71bb32d
framework
Caideyipi Nov 29, 2024
5f712bf
partial
Caideyipi Nov 29, 2024
e6bc36d
partial
Caideyipi Nov 29, 2024
7760c18
next
Caideyipi Nov 29, 2024
60461d4
Update AlterDatabaseProcedure.java
Caideyipi Nov 29, 2024
9dd3d78
Update AlterDatabaseProcedure.java
Caideyipi Nov 29, 2024
7e4d735
partial
Caideyipi Nov 29, 2024
4ef6a7c
next
Caideyipi Nov 29, 2024
db25105
Merge branch 'master' of https://github.com/apache/iotdb into alter-db
Caideyipi Nov 29, 2024
0b4711d
continue
Caideyipi Nov 29, 2024
04596ce
Update ProcedureManager.java
Caideyipi Nov 29, 2024
2c9cad8
Update DataNodeInternalRPCServiceImpl.java
Caideyipi Nov 29, 2024
1f4558c
Link
Caideyipi Nov 29, 2024
240765c
backdoor
Caideyipi Nov 29, 2024
9c04d64
replace
Caideyipi Nov 29, 2024
636e391
replace
Caideyipi Nov 29, 2024
d606008
Update ClusterSchemaInfo.java
Caideyipi Nov 29, 2024
3ddeae5
Merge branch 'master' of https://github.com/apache/iotdb into alter-db
Caideyipi Nov 29, 2024
0fdbfe3
Update TsTableInternalRPCType.java
Caideyipi Nov 29, 2024
383eed5
Update SchemaUtils.java
Caideyipi Nov 29, 2024
4729b11
Update ClusterSchemaInfo.java
Caideyipi Nov 29, 2024
22edbc2
Update IoTDBTableIT.java
Caideyipi Nov 29, 2024
4fb5941
next
Caideyipi Nov 29, 2024
54f7a59
Update ClusterSchemaManager.java
Caideyipi Nov 29, 2024
2203d98
Update TableConfigTaskVisitor.java
Caideyipi Nov 29, 2024
17ff49f
Update ClusterSchemaInfo.java
Caideyipi Nov 29, 2024
10ddd33
Update ClusterSchemaInfo.java
Caideyipi Nov 29, 2024
d250c56
Update IoTDBTableIT.java
Caideyipi Nov 29, 2024
ce8a2f8
Update ConfigMTree.java
Caideyipi Nov 29, 2024
9f3a8d0
Fix
Caideyipi Nov 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -299,10 +299,15 @@ public void testManageTable() {
assertEquals(tableNames.length, cnt);
}

// Will not affect the manual "6600000"
statement.execute("alter database test2 set properties ttl=6600000");
statement.execute("alter database test2 set properties ttl=DEFAULT");

statement.execute("alter table table3 set properties ttl=1000000");
statement.execute("alter table table3 set properties ttl=DEFAULT");

// The table3's ttl shall be also 3000000
ttls = new String[] {"INF", "6600000"};
// The table3's ttl shall be "INF"
try (final ResultSet resultSet = statement.executeQuery("SHOW tables")) {
int cnt = 0;
ResultSetMetaData metaData = resultSet.getMetaData();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@
import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterOrDropTableProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.table.AlterDatabaseProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.table.DeleteDevicesProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure;
Expand Down Expand Up @@ -1683,6 +1684,30 @@ public TDeleteTableDeviceResp deleteDevices(final TDeleteTableDeviceReq req) {
}
}

public TSStatus alterDatabase(final TDatabaseSchema schema, final boolean isGeneratedByPipe) {
long procedureId;
synchronized (this) {
final Pair<Long, Boolean> procedureIdDuplicatePair =
checkDuplicateTableTask(
schema.getName(), null, null, null, ProcedureType.ALTER_DATABASE_PROCEDURE);
procedureId = procedureIdDuplicatePair.getLeft();

if (procedureId == -1) {
if (Boolean.TRUE.equals(procedureIdDuplicatePair.getRight())) {
return RpcUtils.getStatus(
TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
"Some other task is operating table with same name.");
}
procedureId =
this.executor.submitProcedure(new AlterDatabaseProcedure(schema, isGeneratedByPipe));
}
}
final List<TSStatus> procedureStatus = new ArrayList<>();
return waitingProcedureFinished(Collections.singletonList(procedureId), procedureStatus)
? StatusUtils.OK
: procedureStatus.get(0);
}

private TSStatus executeWithoutDuplicate(
final String database,
final TsTable table,
Expand Down Expand Up @@ -1756,6 +1781,12 @@ public Pair<Long, Boolean> checkDuplicateTableTask(
return new Pair<>(-1L, true);
}
break;
case ALTER_DATABASE_PROCEDURE:
final AlterDatabaseProcedure alterDatabaseProcedure = (AlterDatabaseProcedure) procedure;
if (database.equals(alterDatabaseProcedure.getSchema().getName())) {
return new Pair<>(-1L, true);
}
break;
case DELETE_DATABASE_PROCEDURE:
final DeleteDatabaseProcedure deleteDatabaseProcedure =
(DeleteDatabaseProcedure) procedure;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ public TSStatus setDatabase(
public TSStatus alterDatabase(
final DatabaseSchemaPlan databaseSchemaPlan, final boolean isGeneratedByPipe) {
TSStatus result;
TDatabaseSchema databaseSchema = databaseSchemaPlan.getSchema();
final TDatabaseSchema databaseSchema = databaseSchemaPlan.getSchema();

if (!isDatabaseExist(databaseSchema.getName())) {
// Reject if Database doesn't exist
Expand All @@ -219,7 +219,7 @@ public TSStatus alterDatabase(

if (databaseSchema.isSetMinSchemaRegionGroupNum()) {
// Validate alter SchemaRegionGroupNum
int minSchemaRegionGroupNum =
final int minSchemaRegionGroupNum =
getMinRegionGroupNum(databaseSchema.getName(), TConsensusGroupType.SchemaRegion);
if (databaseSchema.getMinSchemaRegionGroupNum() <= minSchemaRegionGroupNum) {
result = new TSStatus(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode());
Expand All @@ -246,6 +246,11 @@ public TSStatus alterDatabase(
}
}

// For table model's ttl
if (databaseSchema.isIsTableModel() && databaseSchema.isSetTTL()) {
return configManager.getProcedureManager().alterDatabase(databaseSchema, isGeneratedByPipe);
}

// Alter DatabaseSchema
try {
result =
Expand Down Expand Up @@ -1333,6 +1338,22 @@ public synchronized Pair<TSStatus, TsTable> updateTableProperties(
return new Pair<>(RpcUtils.SUCCESS_STATUS, updatedTable);
}

public List<TsTable> getAlteredTablesByDatabase(final String database, final long ttl)
throws MetadataException {
return clusterSchemaInfo.getAllTablesUnderSpecificDatabase(database).stream()
.filter(
table -> {
if (Boolean.TRUE
.toString()
.equals(table.getPropValue(TsTable.TTL_DEFAULT).orElse(null))) {
table.addProp(TsTable.TTL_PROPERTY, String.valueOf(ttl));
return true;
}
return false;
})
.collect(Collectors.toList());
}

public void clearSchemaQuotaCache() {
schemaQuotaStatistics.clear();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -246,10 +246,33 @@ public TSStatus alterDatabase(final DatabaseSchemaPlan plan) {
currentSchema.getMaxDataRegionGroupNum());
}

if (alterSchema.isSetTTL()) {
currentSchema.setTTL(alterSchema.getTTL());
mTree.getAllTablesUnderSpecificDatabase(partialPathName).stream()
.map(Pair::getLeft)
.filter(
table ->
Boolean.TRUE
.toString()
.equals(table.getPropValue(TsTable.TTL_DEFAULT).orElse(null)))
.forEach(
table ->
table.addProp(
TsTable.TTL_PROPERTY,
alterSchema.getTTL() == Long.MAX_VALUE
? TTL_INFINITE
: String.valueOf(alterSchema.getTTL())));
LOGGER.info(
"[SetTTL] The ttl of Database: {} is adjusted to: {}",
currentSchema.getName(),
currentSchema.getTTL());
}

mTree
.getDatabaseNodeByDatabasePath(partialPathName)
.getAsMNode()
.setDatabaseSchema(currentSchema);

result.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode());
} catch (final MetadataException e) {
LOGGER.error(ERROR_NAME, e);
Expand Down Expand Up @@ -1231,6 +1254,20 @@ public Optional<TsTable> getTsTableIfExists(final String database, final String
}
}

public List<TsTable> getAllTablesUnderSpecificDatabase(final String database)
throws MetadataException {
databaseReadWriteLock.readLock().lock();
try {
return mTree
.getAllTablesUnderSpecificDatabase(getQualifiedDatabasePartialPath(database))
.stream()
.map(Pair::getLeft)
.collect(Collectors.toList());
} finally {
databaseReadWriteLock.readLock().unlock();
}
}

public TSStatus addTableColumn(final AddTableColumnPlan plan) {
databaseReadWriteLock.writeLock().lock();
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -826,9 +826,17 @@ public void setTableProperties(
(k, v) -> {
if (Objects.nonNull(v)) {
table.addProp(k, v);
} else if (k.equals(TsTable.TTL_PROPERTY)
&& databaseNode.getDatabaseSchema().isSetTTL()) {
table.addProp(k, String.valueOf(databaseNode.getDatabaseSchema().getTTL()));
if (k.equals(TsTable.TTL_PROPERTY)) {
table.addProp(TsTable.TTL_DEFAULT, Boolean.FALSE.toString());
}
} else if (k.equals(TsTable.TTL_PROPERTY)) {
table.addProp(TsTable.TTL_DEFAULT, Boolean.TRUE.toString());
if (databaseNode.getDatabaseSchema().isSetTTL()
&& databaseNode.getDatabaseSchema().getTTL() != Long.MAX_VALUE) {
table.addProp(k, String.valueOf(databaseNode.getDatabaseSchema().getTTL()));
} else {
table.removeProp(k);
}
} else {
table.removeProp(k);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,12 @@ public class DeleteDatabaseProcedure

private TDatabaseSchema deleteDatabaseSchema;

public DeleteDatabaseProcedure(boolean isGeneratedByPipe) {
public DeleteDatabaseProcedure(final boolean isGeneratedByPipe) {
super(isGeneratedByPipe);
}

public DeleteDatabaseProcedure(TDatabaseSchema deleteDatabaseSchema, boolean isGeneratedByPipe) {
public DeleteDatabaseProcedure(
final TDatabaseSchema deleteDatabaseSchema, final boolean isGeneratedByPipe) {
super(isGeneratedByPipe);
this.deleteDatabaseSchema = deleteDatabaseSchema;
}
Expand All @@ -78,12 +79,13 @@ public TDatabaseSchema getDeleteDatabaseSchema() {
return deleteDatabaseSchema;
}

public void setDeleteDatabaseSchema(TDatabaseSchema deleteDatabaseSchema) {
public void setDeleteDatabaseSchema(final TDatabaseSchema deleteDatabaseSchema) {
this.deleteDatabaseSchema = deleteDatabaseSchema;
}

@Override
protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteStorageGroupState state)
protected Flow executeFromState(
final ConfigNodeProcedureEnv env, final DeleteStorageGroupState state)
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
if (deleteDatabaseSchema == null) {
return Flow.NO_MORE_STATE;
Expand Down Expand Up @@ -113,11 +115,11 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteStorageGroupSt
deleteDatabaseSchema.getName());

// Submit RegionDeleteTasks
OfferRegionMaintainTasksPlan dataRegionDeleteTaskOfferPlan =
final OfferRegionMaintainTasksPlan dataRegionDeleteTaskOfferPlan =
new OfferRegionMaintainTasksPlan();
List<TRegionReplicaSet> regionReplicaSets =
final List<TRegionReplicaSet> regionReplicaSets =
env.getAllReplicaSets(deleteDatabaseSchema.getName());
List<TRegionReplicaSet> schemaRegionReplicaSets = new ArrayList<>();
final List<TRegionReplicaSet> schemaRegionReplicaSets = new ArrayList<>();
regionReplicaSets.forEach(
regionReplicaSet -> {
// Clear heartbeat cache along the way
Expand Down Expand Up @@ -147,12 +149,12 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteStorageGroupSt
}

// try sync delete schemaengine region
DataNodeAsyncRequestContext<TConsensusGroupId, TSStatus> asyncClientHandler =
final DataNodeAsyncRequestContext<TConsensusGroupId, TSStatus> asyncClientHandler =
new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.DELETE_REGION);
Map<Integer, RegionDeleteTask> schemaRegionDeleteTaskMap = new HashMap<>();
final Map<Integer, RegionDeleteTask> schemaRegionDeleteTaskMap = new HashMap<>();
int requestIndex = 0;
for (TRegionReplicaSet schemaRegionReplicaSet : schemaRegionReplicaSets) {
for (TDataNodeLocation dataNodeLocation :
for (final TRegionReplicaSet schemaRegionReplicaSet : schemaRegionReplicaSets) {
for (final TDataNodeLocation dataNodeLocation :
schemaRegionReplicaSet.getDataNodeLocations()) {
asyncClientHandler.putRequest(requestIndex, schemaRegionReplicaSet.getRegionId());
asyncClientHandler.putNodeLocation(requestIndex, dataNodeLocation);
Expand All @@ -165,7 +167,7 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteStorageGroupSt
if (!schemaRegionDeleteTaskMap.isEmpty()) {
CnToDnInternalServiceAsyncRequestManager.getInstance()
.sendAsyncRequestWithRetry(asyncClientHandler);
for (Map.Entry<Integer, TSStatus> entry :
for (final Map.Entry<Integer, TSStatus> entry :
asyncClientHandler.getResponseMap().entrySet()) {
if (entry.getValue().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
LOG.info(
Expand All @@ -183,7 +185,7 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteStorageGroupSt

if (!schemaRegionDeleteTaskMap.isEmpty()) {
// submit async schemaengine region delete task for failed sync execution
OfferRegionMaintainTasksPlan schemaRegionDeleteTaskOfferPlan =
final OfferRegionMaintainTasksPlan schemaRegionDeleteTaskOfferPlan =
new OfferRegionMaintainTasksPlan();
schemaRegionDeleteTaskMap
.values()
Expand Down Expand Up @@ -215,7 +217,7 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteStorageGroupSt
new ProcedureException("[DeleteDatabaseProcedure] Delete DatabaseSchema failed"));
}
}
} catch (ConsensusException | TException | IOException e) {
} catch (final ConsensusException | TException | IOException e) {
if (isRollbackSupported(state)) {
setFailure(
new ProcedureException(
Expand All @@ -238,7 +240,8 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteStorageGroupSt
}

@Override
protected void rollbackState(ConfigNodeProcedureEnv env, DeleteStorageGroupState state)
protected void rollbackState(
final ConfigNodeProcedureEnv env, final DeleteStorageGroupState state)
throws IOException, InterruptedException {
switch (state) {
case PRE_DELETE_DATABASE:
Expand All @@ -254,7 +257,7 @@ protected void rollbackState(ConfigNodeProcedureEnv env, DeleteStorageGroupState
}

@Override
protected boolean isRollbackSupported(DeleteStorageGroupState state) {
protected boolean isRollbackSupported(final DeleteStorageGroupState state) {
switch (state) {
case PRE_DELETE_DATABASE:
case INVALIDATE_CACHE:
Expand All @@ -265,12 +268,12 @@ protected boolean isRollbackSupported(DeleteStorageGroupState state) {
}

@Override
protected DeleteStorageGroupState getState(int stateId) {
protected DeleteStorageGroupState getState(final int stateId) {
return DeleteStorageGroupState.values()[stateId];
}

@Override
protected int getStateId(DeleteStorageGroupState deleteStorageGroupState) {
protected int getStateId(final DeleteStorageGroupState deleteStorageGroupState) {
return deleteStorageGroupState.ordinal();
}

Expand All @@ -284,7 +287,7 @@ public String getDatabase() {
}

@Override
public void serialize(DataOutputStream stream) throws IOException {
public void serialize(final DataOutputStream stream) throws IOException {
stream.writeShort(
isGeneratedByPipe
? ProcedureType.PIPE_ENRICHED_DELETE_DATABASE_PROCEDURE.getTypeCode()
Expand All @@ -294,17 +297,17 @@ public void serialize(DataOutputStream stream) throws IOException {
}

@Override
public void deserialize(ByteBuffer byteBuffer) {
public void deserialize(final ByteBuffer byteBuffer) {
super.deserialize(byteBuffer);
try {
deleteDatabaseSchema = ThriftConfigNodeSerDeUtils.deserializeTDatabaseSchema(byteBuffer);
} catch (ThriftSerDeException e) {
LOG.error("Error in deserialize DeleteStorageGroupProcedure", e);
} catch (final ThriftSerDeException e) {
LOG.error("Error in deserialize DeleteDatabaseProcedure", e);
}
}

@Override
public boolean equals(Object that) {
public boolean equals(final Object that) {
if (that instanceof DeleteDatabaseProcedure) {
DeleteDatabaseProcedure thatProc = (DeleteDatabaseProcedure) that;
return thatProc.getProcId() == this.getProcId()
Expand Down
Loading
Loading