diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6012ee1..c1ffbf6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -28,6 +28,14 @@ functions: params: directory: "src" + "fetch secrets": + - command: subprocess.exec + type: setup + params: + working_dir: "src" + binary: bash + args: [.evergreen/fetch-secrets.sh] + "fetch repo": - command: shell.exec type: setup @@ -54,12 +62,10 @@ functions: add_expansions_to_env: true working_dir: "src/${DIR}/${REPO_NAME}" binary: bash - env: - atlas: ${workdir}/src/atlas/bin/atlas args: - ../run.sh - "setup atlas cli": + "setup local atlas": - command: subprocess.exec type: setup retry_on_failure: true @@ -67,45 +73,105 @@ functions: add_expansions_to_env: true working_dir: "src" binary: bash - env: - atlas: ${workdir}/src/atlas/bin/atlas args: - .evergreen/provision-atlas.sh + "setup remote atlas": + - command: subprocess.exec + type: setup + params: + add_expansions_to_env: true + working_dir: "src" + binary: bash + args: [.evergreen/setup-remote.sh] + pre_error_fails_task: true pre: - func: "fetch source" - - func: "setup atlas cli" + - func: "fetch secrets" tasks: - - name: test-semantic-kernel-python + - name: test-semantic-kernel-python-local + tags: [local] + commands: + - func: "fetch repo" + - func: "setup local atlas" + - func: "execute tests" + + - name: test-semantic-kernel-python-remote + tags: [remote] + commands: + - func: "fetch repo" + - func: "setup remote atlas" + - func: "execute tests" + + - name: test-semantic-kernel-csharp-local + tags: [local] + commands: + - func: "fetch repo" + - func: "setup local atlas" + - func: "execute tests" + + - name: test-semantic-kernel-csharp-remote + tags: [remote] + commands: + - func: "fetch repo" + - func: "setup remote atlas" + - func: "execute tests" + + - name: test-langchain-python-local + tags: [local] commands: - func: "fetch repo" + - func: "setup local atlas" - func: "execute tests" - - name: test-semantic-kernel-csharp + - name: test-langchain-python-remote + tags: [remote] commands: - func: "fetch repo" + - func: "setup remote atlas" - func: "execute tests" - - name: test-langchain-python + - name: test-chatgpt-retrieval-plugin-local + tags: [local] commands: - func: "fetch repo" + - func: "setup local atlas" - func: "execute tests" - - name: test-chatgpt-retrieval-plugin + - name: test-chatgpt-retrieval-plugin-remote + tags: [remote] commands: - func: "fetch repo" + - func: "setup remote atlas" - func: "execute tests" - - name: test-llama-index + - name: test-llama-index-local + tags: [local] commands: - func: "fetch repo" + - func: "setup local atlas" - func: "execute tests" - - name: test-docarray + - name: test-llama-index-remote commands: - func: "fetch repo" + - func: "setup remote atlas" + - func: "execute tests" + + - name: test-docarray-local + tags: [local] + commands: + - func: "fetch repo" + - func: "setup local atlas" + - func: "execute tests" + + - name: test-docarray-remote + tags: [remote] + commands: + - func: "fetch repo" + - func: "setup remote atlas" - func: "execute tests" buildvariants: @@ -121,7 +187,10 @@ buildvariants: run_on: - rhel87-small tasks: - - name: test-llama-index + - name: test-llama-index-local + - name: test-llama-index-remote + batchtime: 10080 # 1 week + - name: test-semantic-kernel-python-rhel display_name: Semantic-Kernel RHEL Python expansions: @@ -132,7 +201,10 @@ buildvariants: run_on: - rhel87-small tasks: - - name: test-semantic-kernel-python + - name: test-semantic-kernel-python-local + # TODO: INTPYTHON-430 + # - name: test-semantic-kernel-python-remote + # batchtime: 10080 # 1 week - name: test-semantic-kernel-csharp-rhel display_name: Semantic-Kernel RHEL CSharp @@ -144,7 +216,9 @@ buildvariants: run_on: - rhel87-small tasks: - - name: test-semantic-kernel-csharp + - name: test-semantic-kernel-csharp-local + - name: test-semantic-kernel-csharp-remote + batchtime: 10080 # 1 week - name: test-langchain-python-rhel display_name: Langchain RHEL Python @@ -156,7 +230,9 @@ buildvariants: run_on: - rhel87-small tasks: - - name: test-langchain-python + - name: test-langchain-python-local + - name: test-langchain-python-remote + batchtime: 10080 # 1 week - name: test-chatgpt-retrieval-plugin-rhel display_name: ChatGPT Retrieval Plugin @@ -168,7 +244,9 @@ buildvariants: run_on: - rhel87-small tasks: - - name: test-chatgpt-retrieval-plugin + - name: test-chatgpt-retrieval-plugin-local + - name: test-chatgpt-retrieval-plugin-remote + batchtime: 10080 # 1 week - name: test-llama-index-vectorstore-rhel display_name: LlamaIndex RHEL Vector Store @@ -180,7 +258,10 @@ buildvariants: run_on: - rhel87-small tasks: - - name: test-llama-index + - name: test-llama-index-local + # TODO: INTPYTHON-440 + # - name: test-llama-index-remote + # batchtime: 10080 # 1 week - name: test-docarray-rhel display_name: DocArray RHEL @@ -192,4 +273,5 @@ buildvariants: run_on: - rhel87-small tasks: - - name: test-docarray + - name: test-docarray-local + - name: test-docarray-remote diff --git a/.evergreen/fetch-secrets.sh b/.evergreen/fetch-secrets.sh new file mode 100644 index 0000000..2ca2980 --- /dev/null +++ b/.evergreen/fetch-secrets.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eu + +# Clone drivers-evergeen-tools. +git clone https://github.com/mongodb-labs/drivers-evergreen-tools + +# Get the secrets for drivers/ai-ml-pipeline-testing. +. drivers-evergreen-tools/.evergreen/secrets_handling/setup-secrets.sh drivers/ai-ml-pipeline-testing diff --git a/.evergreen/provision-atlas.sh b/.evergreen/provision-atlas.sh index 30dba7a..dc1e010 100644 --- a/.evergreen/provision-atlas.sh +++ b/.evergreen/provision-atlas.sh @@ -1,33 +1,14 @@ #!/bin/bash +set -eu . .evergreen/utils.sh -PYTHON_BINARY=$(find_python3) - -# Should be called from src -EVERGREEN_PATH=$(pwd)/.evergreen -TARGET_DIR=$(pwd)/$DIR -SCAFFOLD_SCRIPT=$EVERGREEN_PATH/scaffold_atlas.py - -set -ex -mkdir atlas - setup_local_atlas +scaffold_atlas -cd atlas - -$PYTHON_BINARY -m venv . -source ./bin/activate - -# Test server is up -$PYTHON_BINARY -m pip install pymongo -CONN_STRING=$CONN_STRING \ - $PYTHON_BINARY -c "from pymongo import MongoClient; import os; MongoClient(os.environ['CONN_STRING']).db.command('ping')" +# Get the secrets. +source secrets-export.sh -# Add database and index configurations -DATABASE=$DATABASE \ - CONN_STRING=$CONN_STRING \ - REPO_NAME=$REPO_NAME \ - DIR=$DIR \ - TARGET_DIR=$TARGET_DIR \ - $PYTHON_BINARY $SCAFFOLD_SCRIPT +# Create the env file +echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh +echo "export MONGODB_URI=$CONN_STRING" >> env.sh diff --git a/.evergreen/scaffold_atlas.py b/.evergreen/scaffold_atlas.py index 6e66250..645ec44 100644 --- a/.evergreen/scaffold_atlas.py +++ b/.evergreen/scaffold_atlas.py @@ -4,7 +4,8 @@ import logging import os from pathlib import Path -from typing import Any, Union +from time import sleep, monotonic +from typing import Any, Callable, Union from pymongo import MongoClient from pymongo.database import Database @@ -13,7 +14,7 @@ logging.basicConfig() logger = logging.getLogger(__file__) -logger.setLevel(logging.DEBUG if os.environ.get("DEBUG") else logging.INFO) +logger.setLevel(logging.DEBUG) DATABASE_NAME = os.environ.get("DATABASE") CONN_STRING = os.environ.get("CONN_STRING") @@ -41,12 +42,17 @@ def upload_data(db: Database, filename: Path) -> None: db.name, collection_name, ) + collections = [c["name"] for c in db.list_collections()] + if collection_name in collections: + logger.debug("Clearing existing collection", collection_name) + db[collection_name].delete_many({}) + if not isinstance(loaded_collection, list): loaded_collection = [loaded_collection] if loaded_collection: result: InsertManyResult = db[collection_name].insert_many(loaded_collection) logger.debug("Uploaded results for %s: %s", filename.name, result.inserted_ids) - else: + elif collection_name not in collections: logger.debug("Empty collection named %s created", collection_name) db.create_collection(collection_name) @@ -66,12 +72,87 @@ def create_index(client: MongoClient, filename: Path) -> None: index_name = loaded_index_configuration.pop("name") index_type = loaded_index_configuration.pop("type", None) + logger.debug( + "creating search index: %s on %s.%s...", + index_name, + database_name, + collection_name, + ) + collection = client[database_name][collection_name] search_index = SearchIndexModel( loaded_index_configuration, name=index_name, type=index_type ) - collection.create_search_index(search_index) + indexes = [index["name"] for index in collection.list_search_indexes()] + if index_name not in indexes: + collection.create_search_index(search_index) + + else: + logger.debug( + "search index already exists, updating: %s on %s.%s", + index_name, + database_name, + collection_name, + ) + collection.update_search_index(index_name, loaded_index_configuration) + + logger.debug("waiting for search index to be ready...") + wait_until_complete = 120 + _wait_for_predicate( + predicate=lambda: _is_index_ready(collection, index_name), + err=f"Index {index_name} update did not complete in {wait_until_complete}!", + timeout=wait_until_complete, + ) + logger.debug("waiting for search index to be ready... done.") + + logger.debug( + "creating search index: %s on %s.%s... done", + index_name, + database_name, + collection_name, + ) + + +def _is_index_ready(collection: Any, index_name: str) -> bool: + """Check for the index name in the list of available search indexes. + + This confirms that the specified index is of status READY. + + Args: + collection (Collection): MongoDB Collection to for the search indexes + index_name (str): Vector Search Index name + + Returns: + bool : True if the index is present and READY false otherwise + """ + search_indexes = collection.list_search_indexes(index_name) + + for index in search_indexes: + if index["status"] == "READY": + return True + return False + + +def _wait_for_predicate( + predicate: Callable, err: str, timeout: float = 120, interval: float = 0.5 +) -> None: + """Generic to block until the predicate returns true. + + Args: + predicate (Callable[, bool]): A function that returns a boolean value + err (str): Error message to raise if nothing occurs + timeout (float, optional): Wait time for predicate. Defaults to TIMEOUT. + interval (float, optional): Interval to check predicate. Defaults to DELAY. + + Raises: + TimeoutError: _description_ + """ + start = monotonic() + while not predicate(): + if monotonic() - start > timeout: + raise TimeoutError(err) + sleep(interval) def walk_directory(filepath) -> list[str]: diff --git a/.evergreen/setup-remote.sh b/.evergreen/setup-remote.sh new file mode 100644 index 0000000..551f548 --- /dev/null +++ b/.evergreen/setup-remote.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -eu + +source secrets-export.sh + +if [ -z "${DIR:-}" ]; then + echo "Must give a target dir!" + exit 1 +fi + +# Get the correct remote URI. +case $DIR in + llama-index-python-kvstore) + MONGODB_URI=$LLAMA_INDEX_MONGODB_URI + ;; + semantic-kernel-python) + MONGODB_URI=$SEMANTIC_KERNEL_MONGODB_URI + ;; + semantic-kernel-csharp) + MONGODB_URI=$SEMANTIC_KERNEL_MONGODB_URI + ;; + langchain-python) + MONGODB_URI=$LANGCHAIN_MONGODB_URI + ;; + chatgpt-retrieval-plugin) + MONGODB_URI=$CHATGPT_RETRIEVAL_PLUGIN_MONGODB_URI + ;; + llama-index-python-vectorstore) + MONGODB_URI=$LLAMA_INDEX_MONGODB_URI + ;; + docarray) + MONGODB_URI=$DOCARRAY_MONGODB_URI + ;; + *) + echo "Missing config in fetch-secrets.sh for DIR: $DIR" + exit 1 + ;; +esac +export MONGODB_URI + +# Create the env file +echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh +echo "export MONGODB_URI=$MONGODB_URI" >> env.sh + +# Ensure the remote database is populated. +. .evergreen/utils.sh + +CONN_STRING=$MONGODB_URI scaffold_atlas diff --git a/.evergreen/utils.sh b/.evergreen/utils.sh index a14b063..885d4f6 100644 --- a/.evergreen/utils.sh +++ b/.evergreen/utils.sh @@ -1,6 +1,6 @@ -#!/bin/bash -ex +#!/bin/bash -set -o xtrace +set -eu find_python3() { PYTHON="" @@ -17,7 +17,7 @@ find_python3() { elif [ -d "/Library/Frameworks/Python.Framework/Versions/3.7" ]; then PYTHON="/Library/Frameworks/Python.Framework/Versions/3.7/bin/python3" fi - elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + elif [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin PYTHON="C:/python/Python37/python.exe" else # Prefer our own toolchain, fall back to mongodb toolchain if it has Python 3.7+. @@ -115,3 +115,34 @@ fetch_local_atlas_uri() { export CONN_STRING=$CONN_STRING echo "$CONN_STRING" } + + +scaffold_atlas() { + PYTHON_BINARY=$(find_python3) + + # Should be called from src + EVERGREEN_PATH=$(pwd)/.evergreen + TARGET_DIR=$(pwd)/$DIR + SCAFFOLD_SCRIPT=$EVERGREEN_PATH/scaffold_atlas.py + + mkdir -p atlas + pushd atlas + + $PYTHON_BINARY -m venv . + source ./bin/activate + popd + + # Test server is up + $PYTHON_BINARY -m pip install pymongo + CONN_STRING=$CONN_STRING \ + $PYTHON_BINARY -c "from pymongo import MongoClient; import os; MongoClient(os.environ['CONN_STRING']).db.command('ping')" + + # Add database and index configurations + DATABASE=$DATABASE \ + CONN_STRING=$CONN_STRING \ + REPO_NAME=$REPO_NAME \ + DIR=$DIR \ + DEBUG="${DEBUG:-1}" \ + TARGET_DIR=$TARGET_DIR \ + $PYTHON_BINARY $SCAFFOLD_SCRIPT +} diff --git a/.gitignore b/.gitignore index fabf148..af6cdc2 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,8 @@ xunit-results/ # Miscellaneous .DS_Store +drivers-evergreen-tools + +# Secrets +secrets-export.sh +env.sh diff --git a/chatgpt-retrieval-plugin/run.sh b/chatgpt-retrieval-plugin/run.sh index dc1cba0..9927cc5 100644 --- a/chatgpt-retrieval-plugin/run.sh +++ b/chatgpt-retrieval-plugin/run.sh @@ -2,7 +2,11 @@ # chat-gpt-retrieval-plugin is a poetry run project -set -x +set -eu + +# Get the MONGODB_URI and OPENAI_API_KEY. +# shellcheck disable=SC2154 +. $workdir/src/env.sh # shellcheck disable=SC2154 . $workdir/src/.evergreen/utils.sh @@ -24,12 +28,11 @@ $PYTHON_BINARY -m poetry lock --no-update # Install from pyproject.toml into package specific environment $PYTHON_BINARY -m poetry install --with dev -# Run tests. Sensitive variables in Evergreen come from Evergeen project: ai-ml-pipeline-testing/ -# shellcheck disable=SC2154 -OPENAI_API_KEY=$openai_api_key \ +# Run tests. +MONGODB_URI="$MONGODB_URI" \ +OPENAI_API_KEY="$OPENAI_API_KEY" \ DATASTORE="mongodb" \ BEARER_TOKEN="staylowandkeepmoving" \ -MONGODB_URI=$(fetch_local_atlas_uri) \ MONGODB_DATABASE="chatgpt_retrieval_plugin_test_db" \ MONGODB_COLLECTION="chatgpt_retrieval_plugin_test_vectorstore" \ MONGODB_INDEX="vector_index" \ diff --git a/docarray/database/bespoke_name.json b/docarray/database/bespoke_name.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/docarray/database/bespoke_name.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/docarray/indexes/text_index_bespoke_name.json b/docarray/indexes/text_index_bespoke_name.json new file mode 100644 index 0000000..b65e8c2 --- /dev/null +++ b/docarray/indexes/text_index_bespoke_name.json @@ -0,0 +1,16 @@ + { + "mappings": { + "dynamic": false, + "fields": { + "text": [ + { + "type": "string" + } + ] + } + }, + "name": "text_index", + "type": "search", + "database": "docarray_test_db", + "collectionName": "bespoke_name" + } \ No newline at end of file diff --git a/docarray/indexes/vector_index_bespoke_name.json b/docarray/indexes/vector_index_bespoke_name.json new file mode 100644 index 0000000..74410f3 --- /dev/null +++ b/docarray/indexes/vector_index_bespoke_name.json @@ -0,0 +1,23 @@ +{ + "fields": [ + { + "numDimensions": 10, + "path": "embedding", + "similarity": "cosine", + "type": "vector" + }, + { + "path": "number", + "type": "filter" + }, + { + "path": "text", + "type": "filter" + } + ], + "name": "vector_index", + "type": "vectorSearch", + "database": "docarray_test_db", + "collectionName": "bespoke_name" + } + \ No newline at end of file diff --git a/docarray/indexes/vector_index_mydoc__list_docs__docs.json b/docarray/indexes/vector_index_mydoc__list_docs__docs.json index a4f986d..6e0fdf4 100644 --- a/docarray/indexes/vector_index_mydoc__list_docs__docs.json +++ b/docarray/indexes/vector_index_mydoc__list_docs__docs.json @@ -12,5 +12,3 @@ "database": "docarray_test_db", "collectionName": "mydoc__list_docs__docs" } - - diff --git a/docarray/run.sh b/docarray/run.sh index 315adb4..3477b5a 100644 --- a/docarray/run.sh +++ b/docarray/run.sh @@ -3,7 +3,11 @@ # Sets up a virtual environment (poetry) # Runs the mongodb tests of the upstream repo -set -x +set -eu + +# Get the MONGODB_URI. +# shellcheck disable=SC2154 +. $workdir/src/env.sh # shellcheck disable=SC2154 . $workdir/src/.evergreen/utils.sh @@ -23,6 +27,6 @@ poetry install --with dev --extras mongo # Run tests. Sensitive variables in Evergreen come from Evergeen project: ai-ml-pipeline-testing/ # shellcheck disable=SC2154 -MONGODB_URI=$docarray_mongodb_uri \ +MONGODB_URI="$MONGODB_URI" \ MONGODB_DATABASE="docarray_test_db" \ pytest -v tests/index/mongo_atlas diff --git a/langchain-python/indexes/langchain_test_retrievers_fulltext_index.json b/langchain-python/indexes/langchain_test_retrievers_fulltext_index.json index 51ac3a7..f9f458b 100644 --- a/langchain-python/indexes/langchain_test_retrievers_fulltext_index.json +++ b/langchain-python/indexes/langchain_test_retrievers_fulltext_index.json @@ -1,13 +1,13 @@ { "mappings": { - "dynamic": false, - "fields": { - "text": [ - { - "type": "string" - } - ] - } + "dynamic": false, + "fields": { + "text": [ + { + "type": "string" + } + ] + } }, "name": "text_index", "type": "search", diff --git a/langchain-python/indexes/langchain_test_retrievers_vector_index.json b/langchain-python/indexes/langchain_test_retrievers_vector_index.json index c13d4cf..747ef92 100644 --- a/langchain-python/indexes/langchain_test_retrievers_vector_index.json +++ b/langchain-python/indexes/langchain_test_retrievers_vector_index.json @@ -1,18 +1,14 @@ { "fields": [ { - "numDimensions": 1536, - "path": "embedding", - "similarity": "dotProduct", - "type": "vector" - }, - { - "path": "c", - "type": "filter" + "numDimensions": 1536, + "path": "embedding", + "similarity": "cosine", + "type": "vector" } ], "name": "vector_index", "type": "vectorSearch", "database": "langchain_test_db", "collectionName": "langchain_test_retrievers" -} \ No newline at end of file +} diff --git a/langchain-python/run.sh b/langchain-python/run.sh index 04aa371..19ea853 100644 --- a/langchain-python/run.sh +++ b/langchain-python/run.sh @@ -1,7 +1,11 @@ #!/bin/bash # WORKING_DIR = src/langchain-python/langchain -set -x +set -eu + +# Get the MONGODB_URI and OPENAI_API_KEY. +# shellcheck disable=SC2154 +. $workdir/src/env.sh # shellcheck disable=SC2154 . $workdir/src/.evergreen/utils.sh @@ -20,11 +24,8 @@ poetry lock --no-update poetry install --with dev -MONGODB_ATLAS_URI=$(fetch_local_atlas_uri) - -export MONGODB_ATLAS_URI -# shellcheck disable=SC2154 -export OPENAI_API_KEY=$openai_api_key +export MONGODB_ATLAS_URI=$MONGODB_URI +export OPENAI_API_KEY=$OPENAI_API_KEY make test diff --git a/llama-index-python-kvstore/run.sh b/llama-index-python-kvstore/run.sh index 70691bc..ca5e2db 100644 --- a/llama-index-python-kvstore/run.sh +++ b/llama-index-python-kvstore/run.sh @@ -1,11 +1,14 @@ #!/bin/sh -set -x +set -eu + +# Get the MONGODB_URI and OPENAI_API_KEY. +# shellcheck disable=SC2154 +. $workdir/src/env.sh # shellcheck disable=SC2154 . $workdir/src/.evergreen/utils.sh -CONN_STRING=$(fetch_local_atlas_uri) PYTHON_BINARY=$(find_python3) $PYTHON_BINARY -c "import sys; print(f'Python version found: {sys.version_info}')" @@ -29,10 +32,9 @@ $PYTHON_BINARY -m poetry lock --no-update # Install from pyproject.toml into package specific environment $PYTHON_BINARY -m poetry install --with dev -# Run tests. Sensitive variables in Evergreen come from Evergreen project: ai-ml-pipeline-testing/ -# shellcheck disable=SC2154 -OPENAI_API_KEY=$openai_api_key \ -MONGODB_URI=$CONN_STRING \ +# Run tests. +MONGODB_URI="$MONGODB_URI" \ +OPENAI_API_KEY="$OPENAI_API_KEY" \ MONGODB_DATABASE="llama_index_test_db" \ MONGODB_COLLECTION="llama_index_test_kvstore" \ $PYTHON_BINARY -m poetry run pytest -v tests diff --git a/llama-index-python-vectorstore/database/llama_index_test_filters.json b/llama-index-python-vectorstore/database/llama_index_test_filters.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/llama-index-python-vectorstore/database/llama_index_test_filters.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/llama-index-python-vectorstore/run.sh b/llama-index-python-vectorstore/run.sh index e3eaf02..1a517eb 100644 --- a/llama-index-python-vectorstore/run.sh +++ b/llama-index-python-vectorstore/run.sh @@ -1,6 +1,10 @@ #!/bin/sh -set -x +set -eu + +# Get the MONGODB_URI and OPENAI_API_KEY. +# shellcheck disable=SC2154 +. $workdir/src/env.sh # shellcheck disable=SC2154 . $workdir/src/.evergreen/utils.sh @@ -26,10 +30,9 @@ $PYTHON_BINARY -m poetry lock --no-update # Install from pyproject.toml into package specific environment $PYTHON_BINARY -m poetry install --with dev -# Run tests. Sensitive variables in Evergreen come from Evergreen project: ai-ml-pipeline-testing/ -# shellcheck disable=SC2154 -MONGODB_URI=$(fetch_local_atlas_uri) \ -OPENAI_API_KEY=$openai_api_key \ +# Run tests. +MONGODB_URI="$MONGODB_URI" \ +OPENAI_API_KEY="$OPENAI_API_KEY" \ MONGODB_DATABASE="llama_index_test_db" \ MONGODB_COLLECTION="llama_index_test_vectorstore" \ MONGODB_INDEX="vector_index" \ diff --git a/semantic-kernel-csharp/indexes/nearestSearch_default.json b/semantic-kernel-csharp/indexes/nearestSearch_default.json index f0ca670..aa932ff 100644 --- a/semantic-kernel-csharp/indexes/nearestSearch_default.json +++ b/semantic-kernel-csharp/indexes/nearestSearch_default.json @@ -1,15 +1,12 @@ { "collectionName": "nearestSearch", "database": "dotnetMSKNearestTest", - "mappings": { - "dynamic": true, - "fields": { - "embedding": { - "dimensions": 3, + "type": "vectorSearch", + "fields": [{ + "path": "embedding", + "numDimensions": 3, "similarity": "cosine", - "type": "knnVector" - } - } - }, + "type": "vector" + }], "name": "default" } \ No newline at end of file diff --git a/semantic-kernel-csharp/run.sh b/semantic-kernel-csharp/run.sh index 96fed2c..c88b9a9 100644 --- a/semantic-kernel-csharp/run.sh +++ b/semantic-kernel-csharp/run.sh @@ -1,6 +1,10 @@ #!/bin/bash -set -x +set -eu + +# Get the MONGODB_URI. +# shellcheck disable=SC2154 +. $workdir/src/env.sh # shellcheck disable=SC2154 . $workdir/src/.evergreen/utils.sh @@ -20,5 +24,5 @@ sed -i -e 's/"MongoDB Atlas cluster is required"/null/g' dotnet/src/IntegrationT # Run tests echo "Running MongoDBMemoryStoreTests" -MongoDB__ConnectionString=$(fetch_local_atlas_uri) \ +MongoDB__ConnectionString=$MONGODB_URI \ $DOTNET_SDK_PATH/dotnet test dotnet/src/IntegrationTests/IntegrationTests.csproj --filter SemanticKernel.IntegrationTests.Connectors.MongoDB.MongoDBMemoryStoreTests diff --git a/semantic-kernel-python/indexes/nearestSearch_default.json b/semantic-kernel-python/indexes/nearestSearch_default.json index 30a6347..096e153 100644 --- a/semantic-kernel-python/indexes/nearestSearch_default.json +++ b/semantic-kernel-python/indexes/nearestSearch_default.json @@ -1,15 +1,14 @@ { - "collectionName": "nearestSearch", - "database": "pyMSKTest", - "mappings": { - "dynamic": true, - "fields": { - "embedding": { - "dimensions": 3, - "similarity": "cosine", - "type": "knnVector" - } + "fields": [ + { + "numDimensions": 3, + "path": "embedding", + "similarity": "cosine", + "type": "vector" } - }, - "name": "default" + ], + "name": "default", + "type": "vectorSearch", + "collectionName": "nearestSearch", + "database": "pyMSKTest" } \ No newline at end of file diff --git a/semantic-kernel-python/run.sh b/semantic-kernel-python/run.sh index 4958719..fa5a16e 100644 --- a/semantic-kernel-python/run.sh +++ b/semantic-kernel-python/run.sh @@ -1,11 +1,14 @@ #!/bin/bash -set -x +set -eu + +# Get the MONGODB_URI and OPENAI_API_KEY. +# shellcheck disable=SC2154 +. $workdir/src/env.sh # shellcheck disable=SC2154 . $workdir/src/.evergreen/utils.sh -CONN_STRING=$(fetch_local_atlas_uri) PYTHON_BINARY=$(find_python3) # WORKING_DIR = src/semantic-kernel-python/semantic-kernel @@ -22,21 +25,21 @@ make install-sk make install-pre-commit # shellcheck disable=SC2154 -OPENAI_API_KEY=$openai_api_key \ +OPENAI_API_KEY="$OPENAI_API_KEY" \ OPENAI_ORG_ID="" \ AZURE_OPENAI_DEPLOYMENT_NAME="" \ AZURE_OPENAI_ENDPOINT="" \ AZURE_OPENAI_API_KEY="" \ - MONGODB_ATLAS_CONNECTION_STRING=$CONN_STRING \ + MONGODB_ATLAS_CONNECTION_STRING=$MONGODB_URI \ Python_Integration_Tests=1 \ uv run pytest tests/integration/memory/memory_stores/test_mongodb_atlas_memory_store.py -k test_collection_knn # shellcheck disable=SC2154 -OPENAI_API_KEY=$openai_api_key \ +OPENAI_API_KEY="$OPENAI_API_KEY" \ OPENAI_ORG_ID="" \ AZURE_OPENAI_DEPLOYMENT_NAME="" \ AZURE_OPENAI_ENDPOINT="" \ AZURE_OPENAI_API_KEY="" \ - MONGODB_ATLAS_CONNECTION_STRING=$CONN_STRING \ + MONGODB_ATLAS_CONNECTION_STRING=$MONGODB_URI \ Python_Integration_Tests=1 \ uv run pytest tests/integration/memory/memory_stores/test_mongodb_atlas_memory_store.py