From 9ccb5d96107b1cab3014c2cfd2211168a67ec219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 27 Feb 2024 09:25:20 +0000 Subject: [PATCH] storage: Flush SYSTEM.CATALOG table after DROP VIEW #TASK-5663 --- .../storage/hadoop/app/PhoenixMain.java | 2 +- .../hadoop/app/VariantEngineUtilsMain.java | 2 +- .../adaptors/phoenix/PhoenixHelper.java | 23 ++++++++++++++++++- .../phoenix/VariantPhoenixSchemaManager.java | 11 ++++----- .../variant/HadoopVariantStorageTest.java | 23 ++++--------------- 5 files changed, 33 insertions(+), 28 deletions(-) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java index b663d8728e7..1b513b5eed8 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/PhoenixMain.java @@ -153,7 +153,7 @@ private void dropView(VariantHadoopDBAdaptor dbAdaptor) throws Exception { throw new IllegalStateException("Variants table '" + dbAdaptor.getVariantTable() + "' doesn't exist"); } - VariantPhoenixSchemaManager.dropTable(dbAdaptor.getHBaseManager(), dbAdaptor.getVariantTable(), true); + VariantPhoenixSchemaManager.dropView(dbAdaptor.getHBaseManager(), dbAdaptor.getVariantTable(), true); } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java index 93b63ef9e5c..b4ff45f08df 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/app/VariantEngineUtilsMain.java @@ -171,7 +171,7 @@ private void delete(String[] args) throws IOException, SQLException, ClassNotFou LOGGER.info("[DRY-RUN] drop phoenix view '{}'", table); } else { LOGGER.info("Drop phoenix view '{}'", table); - VariantPhoenixSchemaManager.dropTable(hBaseManager, table, true); + VariantPhoenixSchemaManager.dropView(hBaseManager, table, true); } } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java index 3595927a738..a7a87cad4e7 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/PhoenixHelper.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -210,11 +211,31 @@ public String buildDropTable(String tableName, PTableType tableType, boolean ifE return sb.toString(); } - public void dropTable(Connection con, String tableName, PTableType tableType, boolean ifExists, boolean cascade) throws SQLException { + public void dropTable(org.apache.hadoop.hbase.client.Connection hbaseCon, Connection con, String tableName, PTableType tableType, + boolean ifExists, boolean cascade) throws SQLException, IOException { String sql = buildDropTable(tableName, tableType, ifExists, cascade); logger.info("Dropping phoenix {}: {}", tableType, tableName); logger.info(sql); execute(con, sql); + + try (Admin admin = hbaseCon.getAdmin()) { + // Flush the SYSTEM.CATALOG table to avoid "unexpected errors" when creating a new table with the same name + // This was first observed when running tests in with Phoenix 5.1 + TableName systemCatalog; + if (PhoenixHelper.isNamespaceMappingEnabled(PTableType.SYSTEM, conf)) { + systemCatalog = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE); + } else { + systemCatalog = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME + + "." + PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE); + } + if (admin.tableExists(systemCatalog)) { + logger.info("Flushing phoenix system catalog table '" + systemCatalog + "'"); + admin.flush(systemCatalog); + } else { + logger.info("System catalog table '" + systemCatalog + "' does not exist, unable to flush it."); + } + } } public void addMissingColumns(Connection con, String tableName, Collection newColumns, PTableType tableType) diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java index ccbb18d7bd7..ae8ce82ee84 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/adaptors/phoenix/VariantPhoenixSchemaManager.java @@ -346,16 +346,13 @@ private void createTableIfNeeded() throws SQLException { } } - public void dropTable(boolean ifExists) throws SQLException { - phoenixHelper.dropTable(con, variantsTableName, VariantPhoenixSchema.DEFAULT_TABLE_TYPE, ifExists, true); - } - - public static void dropTable(HBaseManager hBaseManager, String variantsTableName, boolean ifExists) - throws SQLException, ClassNotFoundException { + public static void dropView(HBaseManager hBaseManager, String variantsTableName, boolean ifExists) + throws SQLException, ClassNotFoundException, IOException { // VariantStorageMetadataManager not needed for dropping table try (VariantPhoenixSchemaManager manager = new VariantPhoenixSchemaManager(hBaseManager.getConf(), variantsTableName, null, hBaseManager)) { - manager.dropTable(ifExists); + manager.phoenixHelper.dropTable(hBaseManager.getConnection(), manager.con, manager.variantsTableName, + PTableType.VIEW, ifExists, true); } } diff --git a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java index 23e681bd003..787ea7c4412 100644 --- a/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java +++ b/opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java @@ -73,7 +73,6 @@ import org.apache.zookeeper.server.ZooKeeperServer; import org.junit.Assert; import org.junit.Assume; -import org.junit.Before; import org.junit.rules.ExternalResource; import org.opencb.biodata.models.variant.VariantFileMetadata; import org.opencb.biodata.models.variant.avro.VariantType; @@ -89,11 +88,11 @@ import org.opencb.opencga.storage.core.variant.VariantStorageTest; import org.opencb.opencga.storage.hadoop.HBaseCompat; import org.opencb.opencga.storage.hadoop.utils.HBaseManager; -import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper; -import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema; +import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchemaManager; import org.opencb.opencga.storage.hadoop.variant.executors.MRExecutor; import org.opencb.opencga.storage.hadoop.variant.index.IndexUtils; import org.opencb.opencga.storage.hadoop.variant.index.sample.SampleIndexSchema; +import org.opencb.opencga.storage.hadoop.variant.utils.HBaseVariantTableNameGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -493,21 +492,9 @@ default void clearDB(String tableName) throws Exception { default void deleteTable(String tableName) throws Exception { LoggerFactory.getLogger(HadoopVariantStorageTest.class).info("Drop table " + tableName); - PhoenixHelper phoenixHelper = new PhoenixHelper(configuration.get()); - try (java.sql.Connection con = phoenixHelper.openJdbcConnection()) { - if (phoenixHelper.tableExists(con, tableName)) { - phoenixHelper.dropTable(con, tableName, VariantPhoenixSchema.DEFAULT_TABLE_TYPE, true, true); - // Flush the SYSTEM.CATALOG table to avoid "unexpected errors" when creating a new table with the same name - TableName systemCatalog = TableName.valueOf("SYSTEM:CATALOG"); - if (!utility.get().getConnection().getAdmin().tableExists(systemCatalog)) { - systemCatalog = TableName.valueOf("SYSTEM.CATALOG"); - } - if (utility.get().getConnection().getAdmin().tableExists(systemCatalog)) { - try (Admin admin = utility.get().getConnection().getAdmin(); - ) { - admin.flush(systemCatalog); - } - } + if (HBaseVariantTableNameGenerator.isValidVariantsTable(tableName)) { + try (HBaseManager hbaseManager = new HBaseManager(configuration.get(), utility.get().getConnection())) { + VariantPhoenixSchemaManager.dropView(hbaseManager, tableName, true); } } utility.get().deleteTableIfAny(TableName.valueOf(tableName));