Merge pull request #81 from FrancisCrickInstitute/staging

Staging

plaque_assay/__init__.py

@@ -1,40 +1,5 @@
-from plaque_assay import main
-from plaque_assay import ingest
-from plaque_assay import stats
-from plaque_assay import utils
-from plaque_assay import experiment
-from plaque_assay import plate
-from plaque_assay import failure
-from plaque_assay import sample
-from plaque_assay import consts
-from plaque_assay import qc_criteria
-from plaque_assay import db_models
-from plaque_assay import errors
-from plaque_assay import titration
-from plaque_assay import titration_dilution
-from plaque_assay import titration_ingest
-from plaque_assay import titration_db_uploader
-from plaque_assay import titration_main
-from plaque_assay import titration_utils
+from plaque_assay import (consts, db_models, errors, experiment, failure,
+                          ingest, main, plate, qc_criteria, sample, stats,
+                          titration, utils)
 
-
-__all__ = [
-    "main",
-    "ingest",
-    "stats",
-    "utils",
-    "experiment",
-    "plate",
-    "failure",
-    "sample",
-    "consts",
-    "qc_criteria",
-    "db_models",
-    "errors",
-    "titration",
-    "titration_dilution",
-    "titration_ingest",
-    "titration_db_uploader",
-    "titration_main",
-    "titration_utils",
-]
+from .main import run

plaque_assay/db_models.py

@@ -1,7 +1,11 @@
 from typing import Any
 
 import sqlalchemy as sql
-from sqlalchemy.orm import declarative_base
+
+try:
+    from sqlalchemy.orm import declarative_base
+except ImportError:
+    from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()  # type: Any
 

plaque_assay/main.py

@@ -3,17 +3,15 @@
 the titration analysis.
 """
 
-
 import os
 from typing import List
 
 import sqlalchemy
-from plaque_assay.experiment import Experiment
-from plaque_assay.errors import DatabaseCredentialError
 
+from plaque_assay import ingest, utils
 from plaque_assay.db_uploader import AnalysisDatabaseUploader
-from plaque_assay import ingest
-from plaque_assay import utils
+from plaque_assay.errors import DatabaseCredentialError
+from plaque_assay.experiment import Experiment
 
 
 def create_engine(test: bool = True) -> sqlalchemy.engine.base.Engine:
@@ -49,7 +47,9 @@ def create_engine(test: bool = True) -> sqlalchemy.engine.base.Engine:
             "db credentials not found in environent.",
             "Need to set NE_USER, NE_HOST_{TEST,PROD}, NE_PASSWORD",
         )
-    engine = sqlalchemy.create_engine(f"mysql://{user}:{password}@{host}/serology")
+    engine = sqlalchemy.create_engine(
+        f"mysql+mysqldb://{user}:{password}@{host}/serology"
+    )
     return engine
 
 

plaque_assay/titration/__init__.py

@@ -0,0 +1,15 @@
+from . import consts, ingest, utils
+from .db_uploader import TitrationDatabaseUploader
+from .dilution import TitrationDilution
+from .main import run
+from .titration_class import Titration
+
+__all__ = [
+    "consts",
+    "utils",
+    "ingest",
+    "run",
+    "TitrationDatabaseUploader",
+    "TitrationDilution",
+    "Titration",
+]

plaque_assay/titration_dilution.py → plaque_assay/titration/dilution.py

@@ -5,7 +5,7 @@
 
 import pandas as pd
 
-from plaque_assay import titration_consts as consts
+from plaque_assay.titration import consts
 
 
 class TitrationDilution:

plaque_assay/titration_ingest.py → plaque_assay/titration/ingest.py

@@ -1,14 +1,13 @@
-from glob import glob
-import os
 import logging
+import os
+from glob import glob
 from typing import List
 
 import pandas as pd
 
-from plaque_assay import utils
-from plaque_assay import consts
-from plaque_assay import titration_consts
-from plaque_assay import titration_utils
+from plaque_assay import consts, utils
+from plaque_assay.titration import consts as titration_consts
+from plaque_assay.titration import utils as titration_utils
 
 
 def read_data_from_list(plate_list: List[str]) -> pd.DataFrame:

plaque_assay/titration_main.py → plaque_assay/titration/main.py

@@ -2,11 +2,11 @@
 
 import sqlalchemy
 
-from plaque_assay import titration_db_uploader
-from plaque_assay import titration_ingest
 from plaque_assay import utils
 from plaque_assay.main import create_engine
-from plaque_assay.titration import Titration
+
+from . import db_uploader, ingest
+from .titration_class import Titration
 
 
 def run(plate_list: List[str]) -> None:
@@ -24,14 +24,14 @@ def run(plate_list: List[str]) -> None:
     engine = create_engine(test=False)
     Session = sqlalchemy.orm.sessionmaker(bind=engine)
     session = Session()
-    lims_db_titration = titration_db_uploader.TitrationDatabaseUploader(session)
+    lims_db_titration = db_uploader.TitrationDatabaseUploader(session)
     workflow_id = utils.get_workflow_id_from_plate_list(plate_list)
     variant = utils.get_variant_from_plate_list(plate_list, session, titration=True)
     if lims_db_titration.already_uploaded(workflow_id):
         print(f"workflow_id: {workflow_id} already have results in the database")
         # still exist successfully so task is marked complete
         return None
-    dataset = titration_ingest.read_data_from_list(plate_list)
+    dataset = ingest.read_data_from_list(plate_list)
     titration = Titration(dataset, variant=variant)
     normalised_results = titration.get_normalised_results()
     final_results = titration.get_final_results()

plaque_assay/titration.py → plaque_assay/titration/titration_class.py

@@ -8,9 +8,9 @@
 
 import pandas as pd
 
-from plaque_assay.sample import Sample
-from plaque_assay.titration_dilution import TitrationDilution
 from plaque_assay import utils
+from plaque_assay.sample import Sample
+from plaque_assay.titration.dilution import TitrationDilution
 
 
 class Titration:

plaque_assay/titration_utils.py → plaque_assay/titration/utils.py

@@ -4,8 +4,8 @@
 
 from typing import Optional
 
-from plaque_assay import titration_consts
-from plaque_assay.utils import is_odd, is_even
+from plaque_assay.titration import consts
+from plaque_assay.utils import is_even, is_odd
 
 
 def pos_control_dilution(well) -> Optional[int]:
@@ -22,7 +22,7 @@ def pos_control_dilution(well) -> Optional[int]:
     H (even) | 3 | 1 |
              +---+---+
     """
-    if well not in titration_consts.TITRATION_POSITIVE_CONTROL_WELLS:
+    if well not in consts.TITRATION_POSITIVE_CONTROL_WELLS:
         return None
     row_int = ord(well[0]) - 64  # 1 indexed
     col_int = int(well[1:])

plaque_assay/utils.py

@@ -1,6 +1,6 @@
+import math
 import os
 import string
-import math
 from typing import List, Union
 
 import pandas as pd
@@ -10,7 +10,6 @@
 from plaque_assay.db_models import NE_available_strains
 from plaque_assay.errors import VariantLookupError
 
-
 RESULT_TO_INT = {
     "failed to fit model": -999,
     "no inhibition": -600,
@@ -284,6 +283,8 @@ def get_variant_from_plate_list(
         # swap T for S at beginning i.e "T01" -> "S01"
         # so they match those in NE_available_strains
         prefixes = [i.replace("T", "S") for i in prefixes]
+    else:
+        prefixes = ["S" + prefix[1:] for prefix in prefixes]
     prefix_1, prefix_2 = sorted(prefixes)
     # query table to return variant name (mutant_strain) for entry matching
     # both the plate barcode prefixes
@@ -296,10 +297,24 @@ def get_variant_from_plate_list(
         .first()
     )
     if return_val is None or len(return_val) != 1:
-        raise VariantLookupError(
-            "plate barcode prefixes do not match any known variants in the ",
-            f"LIMS database: {prefixes}",
+        # try without sample prefix if variant is found.
+        # this is because the NE_available_strains table has no guarantee
+        # that the prefixes will be registered wth an "S" prefix.
+        prefix_1 = prefix_1[1:]
+        prefix_2 = prefix_2[1:]
+        return_val = (
+            session.query(NE_available_strains.mutant_strain)
+            .filter(
+                NE_available_strains.plate_id_1 == prefix_1,
+                NE_available_strains.plate_id_2 == prefix_2,
+            )
+            .first()
         )
+        if return_val is None or len(return_val) != 1:
+            raise VariantLookupError(
+                "plate barcode prefixes do not match any known variants in the ",
+                f"LIMS database: {prefixes}",
+            )
     return return_val.mutant_strain
 
 

setup.py

@@ -12,7 +12,7 @@ def read_requirements():
     author="Scott Warchal",
     author_email="[email protected]",
     license="MIT",
-    packages=["plaque_assay"],
+    packages=["plaque_assay", "plaque_assay.titration"],
     python_requires=">=3.6",
     install_requires=read_requirements(),
     tests_require="pytest",

tests/dilution_1_4/test_integration_titration.py

@@ -1,15 +1,11 @@
-import os
 import datetime
+import os
 
 import pandas as pd
 import sqlalchemy
 
 import plaque_assay
-from plaque_assay import db_models
-from plaque_assay import titration
-from plaque_assay import titration_db_uploader
-from plaque_assay import titration_ingest
-
+from plaque_assay import db_models, titration
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 N_WELLS = 384  # half plate (16 rows, 12 cols)
@@ -20,7 +16,15 @@
 
 def create_plate_list(dir_name):
     paths = os.path.abspath(
-        os.path.join(CURRENT_DIR, "..", "test_data", "dilution_1_4", "titration_data", "real_data", dir_name)
+        os.path.join(
+            CURRENT_DIR,
+            "..",
+            "test_data",
+            "dilution_1_4",
+            "titration_data",
+            "real_data",
+            dir_name,
+        )
     )
     return [os.path.join(paths, i) for i in os.listdir(paths)]
 
@@ -92,11 +96,11 @@ def teardown_module():
 
 
 def run_titration_pipeline(plate_list):
-    dataset = titration_ingest.read_data_from_list(plate_list)
+    dataset = titration.ingest.read_data_from_list(plate_list)
     variant = plaque_assay.utils.get_variant_from_plate_list(
         plate_list, session, titration=True
     )
-    lims_db = titration_db_uploader.TitrationDatabaseUploader(session)
+    lims_db = titration.db_uploader.TitrationDatabaseUploader(session)
     workflow_id = plaque_assay.utils.get_workflow_id_from_plate_list(plate_list)
     if lims_db.already_uploaded(workflow_id):
         print(f"workflow_id: {workflow_id} already have results in the database")

tests/test_integration_titration.py

@@ -5,8 +5,7 @@
 import sqlalchemy
 
 import plaque_assay
-from plaque_assay import (db_models, titration, titration_db_uploader,
-                          titration_ingest)
+from plaque_assay import db_models, titration
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 N_WELLS = 384  # half plate (16 rows, 12 cols)
@@ -97,11 +96,11 @@ def teardown_module():
 
 
 def run_titration_pipeline(plate_list):
-    dataset = titration_ingest.read_data_from_list(plate_list)
+    dataset = titration.ingest.read_data_from_list(plate_list)
     variant = plaque_assay.utils.get_variant_from_plate_list(
         plate_list, session, titration=True
     )
-    lims_db = titration_db_uploader.TitrationDatabaseUploader(session)
+    lims_db = titration.db_uploader.TitrationDatabaseUploader(session)
     workflow_id = plaque_assay.utils.get_workflow_id_from_plate_list(plate_list)
     if lims_db.already_uploaded(workflow_id):
         print(f"workflow_id: {workflow_id} already have results in the database")