Skip to content

Commit

Permalink
storage: Flush SYSTEM.CATALOG table after DROP VIEW #TASK-5663
Browse files Browse the repository at this point in the history
  • Loading branch information
@j-coll
j-coll committed Feb 27, 2024
1 parent 2d71c7d commit 9ccb5d9
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 28 deletions.
2 changes: 1 addition & 1 deletion ...-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ private void dropView(VariantHadoopDBAdaptor dbAdaptor) throws Exception {
throw new IllegalStateException("Variants table '" + dbAdaptor.getVariantTable() + "' doesn't exist");
}

VariantPhoenixSchemaManager.dropTable(dbAdaptor.getHBaseManager(), dbAdaptor.getVariantTable(), true);
VariantPhoenixSchemaManager.dropView(dbAdaptor.getHBaseManager(), dbAdaptor.getVariantTable(), true);

}

Expand Down
2 changes: 1 addition & 1 deletion ...doop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ private void delete(String[] args) throws IOException, SQLException, ClassNotFou
LOGGER.info("[DRY-RUN] drop phoenix view '{}'", table);
} else {
LOGGER.info("Drop phoenix view '{}'", table);
VariantPhoenixSchemaManager.dropTable(hBaseManager, table, true);
VariantPhoenixSchemaManager.dropView(hBaseManager, table, true);
}
}
}
Expand Down
23 changes: 22 additions & 1 deletion ...c/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
Expand Down Expand Up @@ -210,11 +211,31 @@ public String buildDropTable(String tableName, PTableType tableType, boolean ifE
return sb.toString();
}

public void dropTable(Connection con, String tableName, PTableType tableType, boolean ifExists, boolean cascade) throws SQLException {
public void dropTable(org.apache.hadoop.hbase.client.Connection hbaseCon, Connection con, String tableName, PTableType tableType,
boolean ifExists, boolean cascade) throws SQLException, IOException {
String sql = buildDropTable(tableName, tableType, ifExists, cascade);
logger.info("Dropping phoenix {}: {}", tableType, tableName);
logger.info(sql);
execute(con, sql);

try (Admin admin = hbaseCon.getAdmin()) {
// Flush the SYSTEM.CATALOG table to avoid "unexpected errors" when creating a new table with the same name
// This was first observed when running tests in with Phoenix 5.1
TableName systemCatalog;
if (PhoenixHelper.isNamespaceMappingEnabled(PTableType.SYSTEM, conf)) {
systemCatalog = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME,
PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE);
} else {
systemCatalog = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME
+ "." + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE);
}
if (admin.tableExists(systemCatalog)) {
logger.info("Flushing phoenix system catalog table '" + systemCatalog + "'");
admin.flush(systemCatalog);
} else {
logger.info("System catalog table '" + systemCatalog + "' does not exist, unable to flush it.");
}
}
}

public void addMissingColumns(Connection con, String tableName, Collection<Column> newColumns, PTableType tableType)
Expand Down
11 changes: 4 additions & 7 deletions ...g/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,16 +346,13 @@ private void createTableIfNeeded() throws SQLException {
}
}

public void dropTable(boolean ifExists) throws SQLException {
phoenixHelper.dropTable(con, variantsTableName, VariantPhoenixSchema.DEFAULT_TABLE_TYPE, ifExists, true);
}

public static void dropTable(HBaseManager hBaseManager, String variantsTableName, boolean ifExists)
throws SQLException, ClassNotFoundException {
public static void dropView(HBaseManager hBaseManager, String variantsTableName, boolean ifExists)
throws SQLException, ClassNotFoundException, IOException {
// VariantStorageMetadataManager not needed for dropping table
try (VariantPhoenixSchemaManager manager =
new VariantPhoenixSchemaManager(hBaseManager.getConf(), variantsTableName, null, hBaseManager)) {
manager.dropTable(ifExists);
manager.phoenixHelper.dropTable(hBaseManager.getConnection(), manager.con, manager.variantsTableName,
PTableType.VIEW, ifExists, true);
}
}

Expand Down
23 changes: 5 additions & 18 deletions ...ore/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@
import org.apache.zookeeper.server.ZooKeeperServer;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.rules.ExternalResource;
import org.opencb.biodata.models.variant.VariantFileMetadata;
import org.opencb.biodata.models.variant.avro.VariantType;
Expand All @@ -89,11 +88,11 @@
import org.opencb.opencga.storage.core.variant.VariantStorageTest;
import org.opencb.opencga.storage.hadoop.HBaseCompat;
import org.opencb.opencga.storage.hadoop.utils.HBaseManager;
import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper;
import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema;
import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchemaManager;
import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor;
import org.opencb.opencga.storage.hadoop.variant.index.IndexUtils;
import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema;
import org.opencb.opencga.storage.hadoop.variant.utils.HBaseVariantTableNameGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -493,21 +492,9 @@ default void clearDB(String tableName) throws Exception {

default void deleteTable(String tableName) throws Exception {
LoggerFactory.getLogger(HadoopVariantStorageTest.class).info("Drop table " + tableName);
PhoenixHelper phoenixHelper = new PhoenixHelper(configuration.get());
try (java.sql.Connection con = phoenixHelper.openJdbcConnection()) {
if (phoenixHelper.tableExists(con, tableName)) {
phoenixHelper.dropTable(con, tableName, VariantPhoenixSchema.DEFAULT_TABLE_TYPE, true, true);
// Flush the SYSTEM.CATALOG table to avoid "unexpected errors" when creating a new table with the same name
TableName systemCatalog = TableName.valueOf("SYSTEM:CATALOG");
if (!utility.get().getConnection().getAdmin().tableExists(systemCatalog)) {
systemCatalog = TableName.valueOf("SYSTEM.CATALOG");
}
if (utility.get().getConnection().getAdmin().tableExists(systemCatalog)) {
try (Admin admin = utility.get().getConnection().getAdmin();
) {
admin.flush(systemCatalog);
}
}
if (HBaseVariantTableNameGenerator.isValidVariantsTable(tableName)) {
try (HBaseManager hbaseManager = new HBaseManager(configuration.get(), utility.get().getConnection())) {
VariantPhoenixSchemaManager.dropView(hbaseManager, tableName, true);
}
}
utility.get().deleteTableIfAny(TableName.valueOf(tableName));
Expand Down

0 comments on commit 9ccb5d9

Please sign in to comment.