Text Search

Signed-off-by: Varun Jain <[email protected]>
opensearch-project · Dec 19, 2023 · a9357f2 · a9357f2
1 parent 724bd36
commit a9357f2
Show file tree

Hide file tree

Showing 10 changed files with 297 additions and 81 deletions.
diff --git a/.github/workflows/backwards_compatibility_tests_workflow.yml b/.github/workflows/backwards_compatibility_tests_workflow.yml
@@ -13,19 +13,19 @@ jobs:
   Restart-Upgrade-BWCTests-NeuralSearch:
     strategy:
       matrix:
-        java: [ 11, 17 ]
+        java: [ 11, 17, 21 ]
         os: [ubuntu-latest]
-        bwc_version : [ "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0-SNAPSHOT"]
+        bwc_version : ["2.9.0", "2.10.0", "2.11.0-SNAPSHOT"]
         opensearch_version : [ "3.0.0-SNAPSHOT" ]
-        exclude:
-          - os: windows-latest
-            bwc_version: "2.0.1"
-          - os: windows-latest
-            bwc_version: "2.1.0"
-          - os: windows-latest
-            bwc_version: "2.2.1"
-          - os: windows-latest
-            bwc_version: "2.3.0"
+#        exclude:
+#          - os: windows-latest
+#            bwc_version: "2.0.1"
+#          - os: windows-latest
+#            bwc_version: "2.1.0"
+#          - os: windows-latest
+#            bwc_version: "2.2.1"
+#          - os: windows-latest
+#            bwc_version: "2.3.0"
 
     name: NeuralSearch Restart-Upgrade BWC Tests
     runs-on: ${{ matrix.os }}
@@ -86,7 +86,7 @@ jobs:
   Rolling-Upgrade-BWCTests-NeuralSearch:
     strategy:
       matrix:
-        java: [ 11, 17 ]
+        java: [ 11, 17, 21 ]
         os: [ubuntu-latest]
         bwc_version: [ "2.11.0-SNAPSHOT" ]
         opensearch_version: [ "3.0.0-SNAPSHOT" ]

diff --git a/gradle.properties b/gradle.properties
@@ -4,7 +4,7 @@
 #
 
 version=2.0.0
-systemProp.bwc.version=2.10.0
+systemProp.bwc.version=2.11.0
 
 # For fixing Spotless check with Java 17
 org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \

diff --git a/qa/build.gradle b/qa/build.gradle
@@ -4,6 +4,8 @@
  */
 
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.w3c.dom.Text
+
 import java.nio.file.Files
 
 apply plugin: 'opensearch.testclusters'
@@ -25,9 +27,18 @@ repositories {
     maven { url "https://plugins.gradle.org/m2/" }
 }
 
+configurations {
+    zipArchive
+}
+
+def knnJarDirectory = "$rootDir/build/dependencies/opensearch-knn"
+
 dependencies {
     api "org.opensearch:opensearch:${opensearch_version}"
-    api group: 'commons-lang', name: 'commons-lang', version: '2.6'
+    zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
+    zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
+    compileOnly fileTree(dir: knnJarDirectory, include: '*.jar')
+    //api group: 'commons-lang', name: 'commons-lang', version: '2.6'
     api "org.apache.logging.log4j:log4j-api:${versions.log4j}"
     api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
     api "junit:junit:${versions.junit}"
@@ -46,6 +57,7 @@ String default_bwc_version = System.getProperty("bwc.version")
 String neural_search_bwc_version = System.getProperty("tests.bwc.version", default_bwc_version)
 boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT")
 String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version
+String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version
 
 String os_platform = "linux"
 String artifact_type = "tar"
@@ -57,8 +69,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) {
     file_ext = "zip"
 }
 
-// Task to pull neural search plugin from archive
-task pullBwcPlugin {
+task deletetempDirectories {
     doFirst {
         File[] tempFiles = tmp_dir.listFiles()
         for (File child : tempFiles) {
@@ -67,6 +78,41 @@ task pullBwcPlugin {
             }
         }
     }
+}
+
+task pullMlCommonsBwcPlugin {
+    dependsOn "deletetempDirectories"
+
+    doLast {
+        ext{
+            if (isSnapshot) {
+                srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"
+            } else {
+                srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}"
+            }
+        }
+        ant.get(
+                src: srcUrl,
+                dest: tmp_dir.absolutePath,
+                httpusecaches: false
+        )
+        copy {
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
+            } else {
+                from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
+            }
+            into tmp_dir.absolutePath
+        }
+        copy {
+            from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-ml"))
+            into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")
+        }
+    }
+}
+
+task pullKnnBwcPlugin {
+    dependsOn "deletetempDirectories"
 
     doLast {
         ext{
@@ -90,15 +136,69 @@ task pullBwcPlugin {
             into tmp_dir.absolutePath
         }
         copy {
-            from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search"))
+            from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-knn"))
+            into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")
+        }
+    }
+}
+
+// Task to pull neural search plugin from archive
+task pullBwcPlugin {
+    dependsOn "deletetempDirectories"
+
+    doLast {
+        ext{
+            if (isSnapshot) {
+                srcUrl = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${neural_search_bwc_version_no_qualifier}/latest/${os_platform}/x64/${artifact_type}/dist/opensearch/opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"
+            } else {
+                srcUrl = "https://artifacts.opensearch.org/releases/bundle/opensearch/${neural_search_bwc_version}/opensearch-${neural_search_bwc_version}-${os_platform}-x64.${file_ext}"
+            }
+        }
+        ant.get(
+                src: srcUrl,
+                dest: tmp_dir.absolutePath,
+                httpusecaches: false
+        )
+        copy {
+            if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+                from zipTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
+            } else {
+                from tarTree(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}"))
+            }
+            into tmp_dir.absolutePath
+        }
+        copy {
+            from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}", "plugins", "opensearch-neural-search"))
             into java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search")
         }
         delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}"), java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-${neural_search_bwc_version_no_qualifier}-${os_platform}-x64.${file_ext}")
     }
 }
 
+task zipBwcMlCommonsPlugin(type: Zip) {
+    dependsOn "pullMlCommonsBwcPlugin"
+    from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml"))
+    destinationDirectory = tmp_dir
+    archiveFileName = "opensearch-ml-${neural_search_bwc_version_no_qualifier}.zip"
+    doLast {
+        delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-ml")
+    }
+}
+
+task zipBwcKnnPlugin(type: Zip) {
+    dependsOn "zipBwcMlCommonsPlugin"
+    dependsOn "pullKnnBwcPlugin"
+    from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn"))
+    destinationDirectory = tmp_dir
+    archiveFileName = "opensearch-knn-${neural_search_bwc_version_no_qualifier}.zip"
+    doLast {
+        delete java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-knn")
+    }
+}
+
 // Task to zip plugin from archive
 task zipBwcPlugin(type: Zip) {
+    dependsOn "zipBwcKnnPlugin"
     dependsOn "pullBwcPlugin"
     from(java.nio.file.Path.of(tmp_dir.absolutePath, "opensearch-neural-search"))
     destinationDirectory = tmp_dir
@@ -108,6 +208,7 @@ task zipBwcPlugin(type: Zip) {
     }
 }
 
+
 task bwcTestSuite {
     dependsOn ":qa:restart-upgrade:testRestartUpgrade"
     dependsOn ":qa:rolling-upgrade:testRollingUpgrade"

diff --git a/qa/restart-upgrade/build.gradle b/qa/restart-upgrade/build.gradle
@@ -13,59 +13,42 @@ String neural_search_bwc_version = System.getProperty("tests.bwc.version", defau
 boolean isSnapshot = neural_search_bwc_version.contains("-SNAPSHOT")
 String neural_search_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version
 String baseName = "neuralSearchBwcCluster-restart"
+String knn_bwc_version_no_qualifier = isSnapshot ? neural_search_bwc_version - "-SNAPSHOT" : neural_search_bwc_version
 
 // Creates a test cluster of previous version and loads k-NN plugin of bwcVersion
 testClusters {
     "${baseName}" {
         testDistribution = "ARCHIVE"
         versions = [neural_search_bwc_version, opensearch_version]
         numberOfNodes = 3
+        plugin(project.tasks.zipBwcMlCommonsPlugin.archiveFile)
+        plugin(project.tasks.zipBwcKnnPlugin.archiveFile)
         plugin(project.tasks.zipBwcPlugin.archiveFile)
         setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
         setting 'http.content_type.required', 'true'
-//        environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-//        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-//            // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies
-//            environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
-//            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-//        } else {
-//            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
-//        }
+        environment "LD_LIBRARY_PATH", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+            // While running on Windows OS, setting the PATH environment variable to include the paths to dlls of JNI libraries and windows dependencies
+            environment('PATH', System.getenv('PATH') + ";$rootDir/jni/release" + ";$rootDir/src/main/resources/windowsDependencies")
+            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib;${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        } else {
+            systemProperty "java.library.path", "${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/knnlib:${buildDir}/testclusters/${baseName}-0/distro/${knn_bwc_version_no_qualifier}-ARCHIVE/plugins/opensearch-knn/lib"
+        }
 
     }
 }
 
 // Task to run BWC tests against the old cluster
 task testAgainstOldCluster(type: StandaloneRestIntegTestTask) {
+    dependsOn "zipBwcMlCommonsPlugin"
+    dependsOn "zipBwcKnnPlugin"
     dependsOn "zipBwcPlugin"
     useCluster testClusters."${baseName}"
     systemProperty 'tests.rest.bwcsuite_cluster', 'old_cluster'
     systemProperty 'tests.is_old_cluster', 'true'
     systemProperty 'tests.skip_delete_model_index', 'true'
     systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
 
-    // Skip test if version is 1.0 or 1.1 as they are not supported in those versions
-    if (neural_search_bwc_version.startsWith("1.0") || neural_search_bwc_version.startsWith("1.1")) {
-        filter {
-//            excludeTestsMatching "org.opensearch.knn.bwc.ModelIT"
-//            excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexCustomMethodFieldMapping"
-//            excludeTestsMatching "org.opensearch.knn.bwc.WarmupIT.testKNNWarmupCustomMethodFieldMapping"
-        }
-    }
-
-    // Skip test if version is 1.2 or 1.3 as they are not supported in those versions
-    if (neural_search_bwc_version.startsWith("1.2") || neural_search_bwc_version.startsWith("1.3")) {
-        filter {
-            // excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testNullParametersOnUpgrade"
-        }
-    }
-
-    // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions
-    if (neural_search_bwc_version.startsWith("1.") || neural_search_bwc_version.startsWith("2.")) {
-        filter {
-            // excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade"
-        }
-    }
     nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
     nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
     systemProperty 'tests.security.manager', 'false'
@@ -87,28 +70,6 @@ task testRestartUpgrade(type: StandaloneRestIntegTestTask) {
     systemProperty 'tests.is_old_cluster', 'false'
     systemProperty 'tests.plugin_bwc_version', neural_search_bwc_version
 
-    // Skip test if version is 1.0 or 1.1 as they are not supported in those versions
-    if (neural_search_bwc_version.startsWith("1.0") || neural_search_bwc_version.startsWith("1.1")) {
-        filter {
-//            excludeTestsMatching "org.opensearch.knn.bwc.ModelIT"
-//            excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testKNNIndexCustomMethodFieldMapping"
-//            excludeTestsMatching "org.opensearch.knn.bwc.WarmupIT.testKNNWarmupCustomMethodFieldMapping"
-        }
-    }
-
-    // Skip test if version is 1.2 or 1.3 as they are not supported in those versions
-    if (neural_search_bwc_version.startsWith("1.2") || neural_search_bwc_version.startsWith("1.3")) {
-        filter {
-            //excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testNullParametersOnUpgrade"
-        }
-    }
-
-    // Skip test if version is 1.0, 1.1, 1.2 or 1.3 as they are not supported in those versions
-    if (neural_search_bwc_version.startsWith("1.") || neural_search_bwc_version.startsWith("2.")) {
-        filter {
-            //excludeTestsMatching "org.opensearch.knn.bwc.IndexingIT.testEmptyParametersOnUpgrade"
-        }
-    }
     nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}")
     nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}")
     systemProperty 'tests.security.manager', 'false'

diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java b/qa/restart-upgrade/src/test/java/org/opensearch/neuralsearch/bwc/TextSearch.java
@@ -5,5 +5,63 @@
 
 package org.opensearch.neuralsearch.bwc;
 
-public class TextSearch {
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import static org.opensearch.neuralsearch.TestUtils.NODES_BWC_CLUSTER;
+
+public class TextSearch extends AbstractRestartUpgradeRestTestCase{
+
+    private static final String PIPELINE_NAME = "nlp-pipeline";
+    private static String DOC_ID = "0";
+    private static final String TEST_FIELD = "test-field";
+    private static final String TEXT= "Hello world";
+
+    public void testIndex() throws Exception{
+        waitForClusterHealthGreen(NODES_BWC_CLUSTER);
+
+        if (isRunningAgainstOldCluster()){
+            String modelId= uploadTextEmbeddingModel();
+            loadModel(modelId);
+            createPipelineProcessor(modelId,PIPELINE_NAME);
+            createIndexWithConfiguration(
+                    testIndex,
+                    Files.readString(Path.of(classLoader.getResource("processor/IndexMappings.json").toURI())),
+                    PIPELINE_NAME
+            );
+            addDocument(testIndex, DOC_ID,TEST_FIELD,TEXT);
+        }else {
+            System.out.println("===========================================================================================Cluster Upgraded");
+            validateTestIndex();
+        }
+    }
+
+
+    private void validateTestIndex() throws Exception {
+        int docCount=getDocCount(testIndex);
+        assertEquals(1,docCount);
+        deleteIndex(testIndex);
+    }
+
+    private String uploadTextEmbeddingModel() throws Exception {
+        String requestBody = Files.readString(Path.of(classLoader.getResource("processor/UploadModelRequestBody.json").toURI()));
+        return registerModelGroupAndGetModelId(requestBody);
+    }
+
+    private String registerModelGroupAndGetModelId(String requestBody) throws Exception {
+        String modelGroupRegisterRequestBody = Files.readString(
+                Path.of(classLoader.getResource("processor/CreateModelGroupRequestBody.json").toURI())
+        ).replace("<MODEL_GROUP_NAME>", "public_model_" + RandomizedTest.randomAsciiAlphanumOfLength(8));
+
+        String modelGroupId=registerModelGroup(modelGroupRegisterRequestBody);
+
+        requestBody = requestBody.replace("<MODEL_GROUP_ID>", modelGroupId);
+
+        return uploadModelId(requestBody);
+    }
+
+    protected void createPipelineProcessor(String modelId, String pipelineName, ProcessorType processorType) throws Exception {
+        String requestBody=Files.readString(Path.of(classLoader.getResource("processor/PipelineConfiguration.json").toURI()));
+        createPipelineProcessor(requestBody,pipelineName,modelId);
+    }
 }
diff --git a/qa/restart-upgrade/src/test/resources/CreateModelGroupRequestBody.json b/qa/restart-upgrade/src/test/resources/CreateModelGroupRequestBody.json
@@ -0,0 +1,4 @@
+{
+  "name": "<MODEL_GROUP_NAME>",
+  "description": "This is a public model group"
+}