storage: Skip tests requiring solr when incompatible with hadoop. #TA…

…SK-5663
opencb · Feb 21, 2024 · 0544c25 · 0544c25
1 parent ddf6ceb
commit 0544c25
Show file tree

Hide file tree

Showing 14 changed files with 162 additions and 24 deletions.
diff --git a/...ysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java b/...ysis/src/test/java/org/opencb/opencga/analysis/variant/manager/VariantOperationsTest.java
@@ -126,7 +126,6 @@ public VariantOperationsTest(String storageEngine) {
     public static OpenCGATestExternalResource opencga = new OpenCGATestExternalResource();
     public static HadoopVariantStorageTest.HadoopExternalResource hadoopExternalResource;
 
-    @ClassRule
     public static VariantSolrExternalResource solrExternalResource = new VariantSolrExternalResource();
 
     private static String storageEngine;
@@ -170,13 +169,23 @@ public void tearDown() {
         }
     }
 
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+            solrExternalResource.before();
+        }
+    }
+
     @AfterClass
     public static void afterClass() {
         opencga.after();
         if (hadoopExternalResource != null) {
             hadoopExternalResource.after();
             hadoopExternalResource = null;
         }
+        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+            solrExternalResource.after();
+        }
     }
 
     private void loadDataset() throws Throwable {
@@ -194,7 +203,11 @@ private void loadDataset() throws Throwable {
         }
 
         catalogManager = opencga.getCatalogManager();
-        variantStorageManager = opencga.getVariantStorageManager(solrExternalResource);
+        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+            variantStorageManager = opencga.getVariantStorageManager(solrExternalResource);
+        } else {
+            variantStorageManager = opencga.getVariantStorageManager();
+        }
         toolRunner = new ToolRunner(opencga.getOpencgaHome().toString(), catalogManager, variantStorageManager);
 
         opencga.clearStorageDB(DB_NAME);
@@ -210,9 +223,10 @@ private void loadDataset() throws Throwable {
         }
 
         setUpCatalogManager();
+        if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
         solrExternalResource.configure(variantStorageManager.getVariantStorageEngine(STUDY, token));
-        solrExternalResource.configure(variantStorageManager.getVariantStorageEngineForStudyOperation(STUDY, new ObjectMap(), token));
-
+            solrExternalResource.configure(variantStorageManager.getVariantStorageEngineForStudyOperation(STUDY, new ObjectMap(), token));
+        }
 
         file = opencga.createFile(STUDY, "variant-test-file.vcf.gz", token);
 //            variantStorageManager.index(STUDY, file.getId(), opencga.createTmpOutdir("_index"), new ObjectMap(VariantStorageOptions.ANNOTATE.key(), true), token);
@@ -323,7 +337,7 @@ public void testVariantFileReload() throws Exception {
 
     @Test
     public void testVariantSecondaryAnnotationIndex() throws Exception {
-
+        Assume.assumeTrue(HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable());
         for (String sample : samples) {
             SampleInternalVariantSecondaryAnnotationIndex index = catalogManager.getSampleManager().get(STUDY, sample, new QueryOptions(), token).first().getInternal().getVariant().getSecondaryAnnotationIndex();
             assertEquals(IndexStatus.NONE, index.getStatus().getId());

diff --git a/...ge-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClass.java b/...ge-core/src/main/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClass.java
@@ -483,4 +483,14 @@ private static Genotype parseGenotype(String gt) {
         }
         return genotype;
     }
+
+    public static Set<GenotypeClass> classify(String gt) {
+        Set<GenotypeClass> genotypeClasses = new HashSet<>();
+        for (GenotypeClass value : values()) {
+            if (value.test(gt)) {
+                genotypeClasses.add(value);
+            }
+        }
+        return genotypeClasses;
+    }
 }
diff --git a/...ore/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClassTest.java b/...ore/src/test/java/org/opencb/opencga/storage/core/variant/adaptors/GenotypeClassTest.java
@@ -5,7 +5,9 @@
 import org.opencb.opencga.core.testclassification.duration.ShortTests;
 
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 
@@ -28,7 +30,41 @@ public void testGenotypes() throws Exception {
         assertEquals(Arrays.asList("./1", "1/."), GenotypeClass.HET_MISS.filter(gts));
         assertEquals(Arrays.asList("./.", "."), GenotypeClass.MISS.filter(gts));
         assertEquals(Arrays.asList("0/2", "2/2", "2/3", "2/."), GenotypeClass.SEC_ALT.filter(gts));
+    }
+
+    @Test
+    public void classify() {
+        checkClassify("0/0", GenotypeClass.HOM_REF);
+        checkClassify("0|0", GenotypeClass.HOM_REF);
+        checkClassify("0", GenotypeClass.HOM_REF);
+        checkClassify("1/1", GenotypeClass.MAIN_ALT, GenotypeClass.HOM_ALT);
+        checkClassify("1/1/1", GenotypeClass.MAIN_ALT, GenotypeClass.HOM_ALT);
+        checkClassify("1", GenotypeClass.MAIN_ALT, GenotypeClass.HOM_ALT);
+        checkClassify("0/1", GenotypeClass.MAIN_ALT, GenotypeClass.HET_REF, GenotypeClass.HET);
+        checkClassify("1/0", GenotypeClass.MAIN_ALT, GenotypeClass.HET_REF, GenotypeClass.HET);
+        checkClassify("0|1", GenotypeClass.MAIN_ALT, GenotypeClass.HET_REF, GenotypeClass.HET);
+        checkClassify("1|0", GenotypeClass.MAIN_ALT, GenotypeClass.HET_REF, GenotypeClass.HET);
+        checkClassify("1/2", GenotypeClass.MAIN_ALT, GenotypeClass.SEC, GenotypeClass.HET, GenotypeClass.HET_ALT);
+        checkClassify("1/4", GenotypeClass.MAIN_ALT, GenotypeClass.SEC, GenotypeClass.HET, GenotypeClass.HET_ALT);
+        checkClassify("3/4", GenotypeClass.SEC_ALT, GenotypeClass.SEC);
+        checkClassify("3/4/5", GenotypeClass.SEC_ALT, GenotypeClass.SEC, GenotypeClass.HET);
+        checkClassify("561/941", GenotypeClass.SEC_ALT, GenotypeClass.SEC);
+        checkClassify("561/1", GenotypeClass.MAIN_ALT, GenotypeClass.SEC, GenotypeClass.HET, GenotypeClass.HET_ALT);
+        checkClassify("0/2", GenotypeClass.SEC_ALT, GenotypeClass.SEC);
+        checkClassify("0/3", GenotypeClass.SEC_ALT, GenotypeClass.SEC);
+        checkClassify("3/3", GenotypeClass.SEC_ALT, GenotypeClass.SEC);
+        checkClassify("1/.", GenotypeClass.MAIN_ALT, GenotypeClass.HET_MISS, GenotypeClass.HET);
+        checkClassify("0/.");
+        checkClassify("./.", GenotypeClass.MISS);
+        checkClassify(".", GenotypeClass.MISS);
+        checkClassify("NA", GenotypeClass.NA);
+        checkClassify("THIS_IS_NOT_A_GENOTYPE");
+    }
 
+    private void checkClassify(String gt, GenotypeClass... expected) {
+        Set<GenotypeClass> actual = GenotypeClass.classify(gt);
+//        System.out.println("GenotypeClass.classify(" + gt + ") = " + actual);
+        assertEquals(new HashSet<>(Arrays.asList(expected)), actual);
     }
 
     @Test

diff --git a/...c/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java b/...c/test/java/org/opencb/opencga/storage/core/variant/solr/VariantSolrExternalResource.java
@@ -60,9 +60,7 @@ public VariantSolrExternalResource(boolean embeded) {
     }
 
     @Override
-    protected void before() throws Throwable {
-        super.before();
-
+    public void before() throws Exception {
         Path rootDir = Paths.get("target/test-data", "junit-variant-solr-" + TimeUtils.getTimeMillis());
         Files.createDirectories(rootDir);
 
@@ -95,7 +93,7 @@ protected void before() throws Throwable {
     }
 
     @Override
-    protected void after() {
+    public void after() {
         super.after();
         try {
             if (embeded) {

diff --git a/...age-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java b/...age-hadoop-compat-api/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompatApi.java
@@ -23,8 +23,9 @@ public static HBaseCompatApi getInstance() {
         return instance;
     }
 
-    public abstract void available(Configuration configuration) throws IOException;
-
     public abstract PhoenixCompatApi getPhoenixCompat();
 
+    public abstract void available(Configuration configuration) throws IOException;
+
+    public abstract boolean isSolrTestingAvailable();
 }
diff --git a/...e-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/...e-hadoop-compat-hbase2.0/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java
@@ -14,6 +14,11 @@ public void available(Configuration configuration) throws IOException {
         HBaseAdmin.available(configuration);
     }
 
+    @Override
+    public boolean isSolrTestingAvailable() {
+        return false;
+    }
+
     @Override
     public PhoenixCompatApi getPhoenixCompat() {
         return new PhoenixCompat();

diff --git a/...e-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/...e-hadoop-compat-hbase2.2/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java
@@ -13,6 +13,11 @@ public void available(Configuration configuration) throws IOException {
         HBaseAdmin.available(configuration);
     }
 
+    @Override
+    public boolean isSolrTestingAvailable() {
+        return false;
+    }
+
     @Override
     public PhoenixCompatApi getPhoenixCompat() {
         return new PhoenixCompat();

diff --git a/...e-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java b/...e-hadoop-compat-hbase2.4/src/main/java/org/opencb/opencga/storage/hadoop/HBaseCompat.java
@@ -14,6 +14,11 @@ public void available(Configuration configuration) throws IOException {
         HBaseAdmin.available(configuration);
     }
 
+    @Override
+    public boolean isSolrTestingAvailable() {
+        return true;
+    }
+
     @Override
     public PhoenixCompatApi getPhoenixCompat() {
         return new PhoenixCompat();

diff --git a/...java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java b/...java/org/opencb/opencga/storage/hadoop/variant/HadoopLocalLoadVariantStoragePipeline.java
@@ -331,8 +331,8 @@ protected void loadFromProto(URI input, URI outdir, ArchiveTableHelper helper, P
             // Update list of loaded genotypes
             this.loadedGenotypes = sampleIndexDBLoader.getLoadedGenotypes();
             this.sampleIndexVersion = sampleIndexDBLoader.getSampleIndexVersion();
-            this.largestVariantLength = largestVariantTask.getMaxLength();
         }
+        this.largestVariantLength = largestVariantTask.getMaxLength();
     }
 
     protected void loadFromAvro(URI input, URI outdir, ArchiveTableHelper helper, ProgressLogger progressLogger)
@@ -390,13 +390,13 @@ protected void loadFromAvroWithArchive(URI input, URI outdir, ArchiveTableHelper
             throw new StorageEngineException("Error loading file " + input, e);
         }
 
-        logLoadResults(variantReader.getVariantFileMetadata(), resolver, hadoopDBWriter);
         if (sampleIndexDBLoader != null) {
             // Update list of loaded genotypes
             this.loadedGenotypes = sampleIndexDBLoader.getLoadedGenotypes();
             this.sampleIndexVersion = sampleIndexDBLoader.getSampleIndexVersion();
-            this.largestVariantLength = largestVariantTask.getMaxLength();
         }
+        this.largestVariantLength = largestVariantTask.getMaxLength();
+        logLoadResults(variantReader.getVariantFileMetadata(), resolver, hadoopDBWriter);
     }
 
     protected void loadFromAvroWithoutArchive(URI input, URI outdir, ArchiveTableHelper helper, ProgressLogger progressLogger)
@@ -438,13 +438,13 @@ protected void loadFromAvroWithoutArchive(URI input, URI outdir, ArchiveTableHel
             throw new StorageEngineException("Error loading file " + input, e);
         }
 
-        logLoadResults(variantReader.getVariantFileMetadata(), resolver, hadoopDBWriter);
         if (sampleIndexDBLoader != null) {
             // Update list of loaded genotypes
             this.loadedGenotypes = sampleIndexDBLoader.getLoadedGenotypes();
             this.sampleIndexVersion = sampleIndexDBLoader.getSampleIndexVersion();
-            this.largestVariantLength = largestVariantTask.getMaxLength();
         }
+        this.largestVariantLength = largestVariantTask.getMaxLength();
+        logLoadResults(variantReader.getVariantFileMetadata(), resolver, hadoopDBWriter);
     }
 
     private void logLoadResults(VariantFileMetadata variantFileMetadata,

diff --git a/...va/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageSearchIntersectTest.java b/...va/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageSearchIntersectTest.java
@@ -33,6 +33,9 @@
 @Category(LongTests.class)
 public class HadoopVariantStorageSearchIntersectTest extends VariantStorageSearchIntersectTest implements HadoopVariantStorageTest {
 
+    @ClassRule
+    public static HadoopSolrSupport solrSupport = new HadoopSolrSupport();
+
     @ClassRule
     public static ExternalResource externalResource = new HadoopExternalResource();
 

diff --git a/...ore/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java b/...ore/src/test/java/org/opencb/opencga/storage/hadoop/variant/HadoopVariantStorageTest.java
@@ -72,6 +72,8 @@
 import org.apache.zookeeper.server.PrepRequestProcessor;
 import org.apache.zookeeper.server.ZooKeeperServer;
 import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
 import org.junit.rules.ExternalResource;
 import org.opencb.biodata.models.variant.VariantFileMetadata;
 import org.opencb.biodata.models.variant.avro.VariantType;
@@ -85,6 +87,7 @@
 import org.opencb.opencga.storage.core.variant.VariantStorageEngine;
 import org.opencb.opencga.storage.core.variant.VariantStorageOptions;
 import org.opencb.opencga.storage.core.variant.VariantStorageTest;
+import org.opencb.opencga.storage.hadoop.HBaseCompat;
 import org.opencb.opencga.storage.hadoop.utils.HBaseManager;
 import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.PhoenixHelper;
 import org.opencb.opencga.storage.hadoop.variant.adaptors.phoenix.VariantPhoenixSchema;
@@ -115,6 +118,18 @@ public interface HadoopVariantStorageTest /*extends VariantStorageManagerTestUti
 //    Set<HadoopVariantStorageEngine> managers = new ConcurrentHashSet<>();
     AtomicReference<HadoopVariantStorageEngine> manager = new AtomicReference<>();
 
+    class HadoopSolrSupport extends ExternalResource {
+        @Override
+        protected void before() throws Throwable {
+            super.before();
+            Assume.assumeTrue(isSolrTestingAvailable());
+        }
+
+        public static boolean isSolrTestingAvailable() {
+            return HBaseCompat.getInstance().isSolrTestingAvailable();
+        }
+    }
+
     class HadoopExternalResource extends ExternalResource implements HadoopVariantStorageTest {
 
         Logger logger = LoggerFactory.getLogger(this.getClass());

diff --git a/...src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java b/...src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java
@@ -2,10 +2,7 @@
 
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Test;
+import org.junit.*;
 import org.junit.experimental.categories.Category;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -21,6 +18,7 @@
 import org.opencb.opencga.storage.core.variant.io.VariantExporter;
 import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory;
 import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource;
+import org.opencb.opencga.storage.hadoop.HBaseCompat;
 import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageEngine;
 import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest;
 import org.opencb.opencga.storage.hadoop.variant.VariantHbaseTestUtils;
@@ -55,7 +53,6 @@ public HadoopVariantExporterTest(String name, Boolean exportToLocal) {
     @ClassRule
     public static HadoopExternalResource externalResource = new HadoopExternalResource();
 
-    @ClassRule
     public static VariantSolrExternalResource solr = new VariantSolrExternalResource();
 
     private static HadoopVariantStorageEngine variantStorageEngine;
@@ -65,7 +62,12 @@ public HadoopVariantExporterTest(String name, Boolean exportToLocal) {
     @BeforeClass
     public static void beforeClass() throws Exception {
         variantStorageEngine = externalResource.getVariantStorageEngine();
-        solr.configure(variantStorageEngine);
+        if (HBaseCompat.getInstance().isSolrTestingAvailable()) {
+            solr.before();
+            solr.configure(variantStorageEngine);
+        } else {
+            System.out.println("Skip embedded solr tests");
+        }
 
 //        URI inputUri = VariantStorageBaseTest.getResourceUri("sample1.genome.vcf");
         URI inputUri = VariantStorageBaseTest.getResourceUri("platinum/1K.end.platinum-genomes-vcf-NA12877_S1.genome.vcf.gz");
@@ -87,12 +89,21 @@ public static void beforeClass() throws Exception {
                         .append(VariantStorageOptions.STATS_CALCULATE.key(), false)
         );
 
-        variantStorageEngine.secondaryIndex();
+        if (HBaseCompat.getInstance().isSolrTestingAvailable()) {
+            variantStorageEngine.secondaryIndex();
+        }
 
         VariantHbaseTestUtils.printVariants(variantStorageEngine.getDBAdaptor(), newOutputUri());
 
     }
 
+    @AfterClass
+    public static void afterClass() {
+        if (HBaseCompat.getInstance().isSolrTestingAvailable()) {
+            solr.after();
+        }
+    }
+
     @Before
     public void before() throws Exception {
         // Do not clean database!

diff --git a/...t/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopSolrTestingSupportTest.java b/...t/java/org/opencb/opencga/storage/hadoop/variant/search/HadoopSolrTestingSupportTest.java
@@ -0,0 +1,33 @@
+package org.opencb.opencga.storage.hadoop.variant.search;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.opencb.opencga.core.testclassification.duration.ShortTests;
+import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource;
+import org.opencb.opencga.storage.hadoop.variant.HadoopVariantStorageTest;
+
+@Category(ShortTests.class)
+public class HadoopSolrTestingSupportTest {
+
+    @Test
+    public void testSupported() {
+        VariantSolrExternalResource externalResource = new VariantSolrExternalResource();
+        try {
+            externalResource.before();
+            externalResource.after();
+            if (!HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+                Assert.fail("Solr testing should not be available");
+            } else {
+                System.out.println("As expected :: Solr testing available");
+            }
+        } catch (Throwable throwable) {
+            if (HadoopVariantStorageTest.HadoopSolrSupport.isSolrTestingAvailable()) {
+                Assert.fail("Solr testing should be available");
+            } else {
+                System.out.println("As expected :: Solr testing not available: " + throwable.getMessage());
+            }
+        }
+    }
+
+}