Testing

.github/workflows/testing.yml

@@ -0,0 +1,73 @@
+name: tests
+
+on:
+  push:
+  pull_request:
+  # schedule:
+  #   - cron: '00 4 * * *'  # daily at 4AM
+
+jobs:
+  build:
+    name: Test conftrack with Python ${{ matrix.python-version }} (${{ matrix.dependencies }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+        dependencies: ["pip", "conda"]
+      fail-fast: false
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Set env vars
+        run: |
+          export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/}  # just the repo, as opposed to org/repo
+          echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV
+
+      - name: Checkout the code
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }} with conda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }}
+          auto-update-conda: true
+          miniconda-version: "latest"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install auxiliary dependencies
+        run: |
+          set -vxeo pipefail
+          python3 -m pip install --upgrade pip wheel
+
+      - name: Install dependencies with ${{ matrix.dependencies }}
+        if: matrix.dependencies == 'pip'
+        run: |
+          set -vxeo pipefail
+          python3 -m pip install -r requirements.txt
+          python3 -m pip install -r requirements-dev.txt
+
+      - name: Install dependencies with ${{ matrix.dependencies }}
+        if: matrix.dependencies == 'conda'
+        run: |
+          set -vxeo pipefail
+          conda install -y -c conda-forge doct jsonschema mongomock mongoquery pymongo pytest pyyaml requests six tornado ujson
+
+      - name: Install the package
+        run: |
+          set -vxeo pipefail
+          python3 -m pip install . -vv --no-deps
+
+      - name: Check installed dependencies
+        run: |
+          set -vxeo pipefail
+          conda env list
+          pip list
+          conda list
+
+      - name: Test with pytest
+        run: |
+          set -vxeuo pipefail
+          pytest -s -vv

amostra/ignition.py

@@ -11,7 +11,7 @@
 from .server.conf import load_configuration
 
 
-def start_server(config=None):
+def start_server(config=None, testing=False):
     """
     Amostra service startup script.
     Returns tornado event loop provided configuration.
@@ -56,7 +56,7 @@ def start_server(config=None):
         service_port = 7770
     print(args)
     db = db_connect(database=config['database'],
-                    mongo_uri=config['mongo_uri'])
+                    mongo_uri=config['mongo_uri'], testing=testing)
     application = tornado.web.Application([
         (r'/sample', SampleReferenceHandler),
         (r'/request', RequestReferenceHandler),

amostra/server/engine.py

@@ -10,7 +10,7 @@
 from .utils import compose_err_msg
 
 
-def db_connect(database, mongo_uri):
+def db_connect(database, mongo_uri, testing=False):
     """Helper function to deal with stateful connections to MongoDB
     Connection established lazily. Connects to the database on request.
     Same connection pool is used for all clients per recommended by
@@ -28,10 +28,14 @@ def db_connect(database, mongo_uri):
     multiple clients and makes no difference for a single client compared
     to pymongo
     """
-    try:
-        client = pymongo.MongoClient(uri)
-    except pymongo.errors.ConnectionFailure:
-        raise utils.AmostraException("Unable to connect to MongoDB server...")
+    if testing:
+        import mongomock
+        client = mongomock.MongoClient(mongo_uri)
+    else:
+        try:
+            client = pymongo.MongoClient(mongo_uri)
+        except pymongo.errors.ConnectionFailure:
+            raise utils.AmostraException("Unable to connect to MongoDB server...")
     database = client[database]
     return database
 
@@ -117,7 +121,7 @@ def post(self):
                                           "Invalid schema on document(s)",
                                           d)
                 uids.append(d['uid'])
-                res = database.sample.insert(d)
+                res = database.sample.insert_one(d)
         elif isinstance(data, dict):
             data = utils.default_timeuid(data)
             try:
@@ -128,7 +132,7 @@ def post(self):
                                       "Invalid schema on document(s)",
                                       data)
             uids.append(data['uid'])
-            res = database.sample.insert(data)
+            res = database.sample.insert_one(data)
             if not res:
                 raise compose_err_msg(500,
                                       'SampleHandler expects list or dict')
@@ -194,7 +198,7 @@ def post(self):
                                           "Invalid schema on document(s)",
                                           d)
                 try:
-                    database.request.insert(d)
+                    database.request.insert_one(d)
                     uids.append(d['uid'])
                 except pymongo.errors.PyMongoError:
                     raise compose_err_msg(500,
@@ -210,7 +214,7 @@ def post(self):
                                       "Invalid schema on document(s)",
                                       data)
             try:
-                database.request.insert(data)
+                database.request.insert_one(data)
                 uids.append(data['uid'])
             except pymongo.errors.PyMongoError:
                 raise compose_err_msg(500,
@@ -298,7 +302,7 @@ def post(self):
                     raise compose_err_msg(400,
                                           "Invalid schema on document(s)", d)
                 uids.append(d['uid'])
-                res = database.container.insert(d)
+                res = database.container.insert_one(d)
         elif isinstance(data, dict):
             data = utils.default_timeuid(data)
             try:
@@ -308,7 +312,7 @@ def post(self):
                 raise compose_err_msg(400,
                                       "Invalid schema on document(s)", data)
             uids.append(data['uid'])
-            res = database.container.insert(data)
+            res = database.container.insert_one(data)
             if not res:
                 raise compose_err_msg(500,
                                       'SampleHandler expects list or dict')

amostra/test/conftest.py

@@ -0,0 +1,115 @@
+import uuid
+import pytest
+import subprocess
+import contextlib
+import os
+from pathlib import Path
+import time as ttime
+import shutil
+import sys
+
+from amostra.client.commands import AmostraClient
+from amostra.client.local_commands import (
+    LocalContainerReference,
+    LocalRequestReference,
+    LocalSampleReference,
+)
+
+testing_config = {
+    "database": "mds_testing_disposable_{}".format(str(uuid.uuid4())),
+    "mongo_server": "localhost",
+    "mongo_uri": "mongodb://localhost",
+    "mongo_port": 27017,
+    "host": "localhost",
+    "port": 7770,
+    "timezone": "US/Eastern",
+    "mongo_user": "tom",
+    "mongo_pwd": "jerry",
+    "local_files": "/tmp/amostra_files",
+}
+
+
+@contextlib.contextmanager
+def amostra_startup():
+    try:
+        ps = subprocess.Popen(
+            [
+                sys.executable,
+                "-c",
+                f"from amostra.ignition import start_server; start_server(config={testing_config}, testing=True)",
+            ],
+        )
+        print(f"entire command: from amostra.ignition import start_server; start_server(config={testing_config}, testing=True)")
+        ttime.sleep(1.3)  # make sure the process is started
+        yield ps
+    finally:
+        ps.terminate()
+
+
+@pytest.fixture(scope="session")
+def amostra_server():
+    with amostra_startup() as amostra_fixture:
+        yield
+
+
+@pytest.fixture(scope="function")
+def amostra_client():
+    conn = AmostraClient(host=testing_config["host"], port=testing_config["port"])
+    return conn
+
+
+@pytest.fixture(scope="function")
+def amostra_local_container(request):
+    try:
+        usr_path = os.path.expanduser(testing_config["local_files"])
+        os.mkdir(usr_path)
+    except FileExistsError:
+        pass
+    local_container = LocalContainerReference(top_dir=testing_config["local_files"])
+
+    def clear_files():
+        try:
+            shutil.rmtree(Path(testing_config["local_files"]).expanduser())
+        except FileNotFoundError:
+            pass
+
+    request.addfinalizer(lambda: clear_files())
+    return local_container
+
+
+@pytest.fixture(scope="function")
+def amostra_local_request(request):
+    try:
+        usr_path = os.path.expanduser(testing_config["local_files"])
+        os.mkdir(usr_path)
+    except FileExistsError:
+        pass
+    local_request = LocalRequestReference(top_dir=testing_config["local_files"])
+
+    def clear_files():
+        try:
+            shutil.rmtree(Path(testing_config["local_files"]).expanduser())
+        except FileNotFoundError:
+            pass
+
+    request.addfinalizer(lambda: clear_files())
+    return local_request
+
+
+@pytest.fixture(scope="function")
+def amostra_local_sample(request):
+    try:
+        usr_path = os.path.expanduser(testing_config["local_files"])
+        os.mkdir(usr_path)
+    except FileExistsError:
+        pass
+    local_sample = LocalSampleReference(top_dir=testing_config["local_files"])
+
+    def clear_files():
+        try:
+            shutil.rmtree(Path(testing_config["local_files"]).expanduser())
+        except FileNotFoundError:
+            pass
+
+    request.addfinalizer(lambda: clear_files())
+    return local_sample