Merge branch 'master' of https://github.com/CovertLab/vivarium-ecoli …

…into metabolism-redux

# Conflicts:
#	ecoli/processes/metabolism_redux.py

.github/workflows/docs_deploy.yml

@@ -12,7 +12,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           sudo apt-get install pandoc

.github/workflows/docs_test.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           sudo apt-get install pandoc

.github/workflows/pytest.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ["3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python ${{ matrix.python-version }}
@@ -26,9 +26,6 @@ jobs:
         pip install numpy
         pip install Cython
         pip install -r requirements.txt
-    - name: Check dependencies
-      run: |
-        python -m scripts.test_requirements
     - name: Setup
       run: |
         python setup.py install

.github/workflows/pytest_master.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ["3.10"]
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python ${{ matrix.python-version }}

Makefile

@@ -7,14 +7,20 @@ compile:
 # The reconstruction/ecoli/dataclasses/process/*.py files were generated by
 #  write_ode_file.py in Parca code.
 # Fireworks writes launcher_20* and block_20*.
+# `aesara-cache clear` clears the current $(aesara-cache) cache, not others, and
+# leaves 3 small *.delete.me dirs for later in case they're in use, esp. on NFS.
 clean:
-	rm -fr fixtures
+	rm -fr fixtures cache
 	(cd reconstruction/ecoli/dataclasses/process && rm -f equilibrium_odes.py two_component_system_odes*.py)
-	find wholecell/ -name "*.pyc" -exec rm -rf {} \;
-	find wholecell/ -name "*.o" -exec rm -fr {} \;
-	find wholecell/ -name "*.so" -exec rm -fr {} \;
+	find . -name "*.pyc" -exec rm -rf {} +
+	find . -name "*.o" -exec rm -fr {} +
+	find . -name "*.so" -exec rm -fr {} +
 	rm -fr build
 	rm -fr launcher_20* block_20*
+	if [ "`aesara-cache | xargs du -sm | cut -f1`" -gt 30 ]; then \
+		echo "Clearing the aesara-cache since it's larger than threshold."; \
+		aesara-cache clear; \
+	fi
 
 # Delete just the *.so libraries then (re)compile them.
 # This is useful when switching to a different Python virtualenv.

README.md

@@ -8,6 +8,8 @@ Vivarium-ecoli is a port of the Covert Lab's
 [E. coli Whole Cell Model](https://github.com/CovertLab/wcEcoli) 
 to the [Vivarium framework](https://github.com/vivarium-collective/vivarium-core).
 
+Below is a high-level overview of the advantages and WIP features.
+
 The scope of this project is to migrate the Whole Cell Model's processes, and therefore takes 
 wcEcoli's `sim_data` as its starting point in the simulation pipeline.
 `sim_data` is a large configuration object created by the parameter calculator (ParCa). 
@@ -26,37 +28,25 @@ a Vivarium topology that is generated in `ecoli/experiments/ecoli_master_sim.py`
 
 pyenv lets you install and switch between multiple Python releases and multiple "virtual 
 environments", each with its own pip packages. Using pyenv, create a virtual environment 
-and install Python 3.8.5. For a tutorial on how to create a virtual environment, follow 
+and install Python 3.10.12. For a tutorial on how to create a virtual environment, follow 
 the instructions [here](https://github.com/CovertLab/wcEcoli/blob/master/docs/create-pyenv.md) 
 and stop once you reach the "Create the 'wcEcoli3' python virtual environment" step. Then, 
 run the following command in your terminal:
 
-    $ pyenv virtualenv 3.8.13 viv-ecoli && pyenv local viv-ecoli
+    $ pyenv virtualenv 3.10.12 viv-ecoli && pyenv local viv-ecoli
 
-Now, install `numpy` first to avoid installation errors with `stochastic-arrow`
-(check `requirements.py` for correct version):
+Now, install numpy (check `requirements.txt` for the exact version):
 
-    $ pip install numpy==1.23.1
+    $ pip install numpy
 
 Then install the remaining requirements:
 
-    $ pip install -r requirements.txt 
+    $ pip install -r requirements.txt
 
 And build the Cython components:
 
     $ make clean compile
 
-## Conda install
-
-To install using conda, create a conda environment with `numpy` (check `requirements.py` 
-for correct version), activate it, install the remaining dependencies, and build the Cython 
-components.
-
-    $ conda create -n WCM python==3.8.13 numpy==1.23.1
-    $ conda activate WCM
-    $ pip install -r requirements.txt
-    $ make clean compile
-
 ## Running the simulation
 
 To run the simulation, set the `PYTHONPATH` environment variable to the cloned repository and run

data/marA_binding/get_TU_ID.py

@@ -10,60 +10,53 @@
 monomer as well as the number of monomers incorporated into each complex.
 
 Required files:
-- tetFC.tsv: fold change for each gene
-- complexation_stoich.npy: complexation stoichiometric matrix
-- complexation_molecules.npy: list of molecules used by complexation process
-- TU_id_to_index.json: dictionary mapping RNA names to TU indexes (Example: {"EG10001_RNA[c]": 0})
+- tetFC.tsv: fold change for each gene, extracted from Figure 1 of
+ Viveiros M, Dupont M, Rodrigues L, Couto I, Davin-Regli A, et al. (2007)
+ Antibiotic Stress, Genetic Response and Altered Permeability of E. coli.
+ PLOS ONE 2(4): e365. https://doi.org/10.1371/journal.pone.0000365
 """
 
 import pandas as pd
+import pickle
 import json
 import numpy as np
+from ast import literal_eval
+
+from ecoli.library.sim_data import SIM_DATA_PATH_NO_OPERONS
 
 def main():
-    rnas = pd.read_table("reconstruction/ecoli/flat/rnas.tsv", comment='#')
-    model_degenes = []
+    # Load complexation and TU index data from sim_data
+    sim_data = pickle.load(open(SIM_DATA_PATH_NO_OPERONS, 'rb'))
+    if sim_data.operons_on:
+        raise TypeError('marA regulon only works with operons OFF')
+    bulk_names = sim_data.internal_state.bulk_molecules.bulk_data['id']
+    TU_id_to_index = {rna: idx for idx, rna in
+        enumerate(sim_data.process.transcription.rna_data['id'])}
+    comp_stoich = sim_data.process.complexation.stoich_matrix().astype(np.int64).T
+    comp_molecules = sim_data.process.complexation.molecule_names
 
+    rnas = pd.read_table("reconstruction/ecoli/flat/rnas.tsv", comment='#')
+    rnas['synonyms'] = rnas['synonyms'].apply(literal_eval)
+    rnas = rnas.explode('synonyms')
 
     # Use fold change from exposure to 1.5 mg/L tetracycline
     tet_FC = pd.read_table("data/marA_binding/tet_FC.tsv")
     tet_FC = tet_FC.loc[:,["Gene Name ", "1.5 mg/L tet."]]
     tet_FC.rename(columns={
         "Gene Name ": "Gene name", "1.5 mg/L tet.": "Fold change"}, inplace=True)
-    degenes = tet_FC.sort_values(
+    de_genes = tet_FC.sort_values(
         by="Fold change", ascending=False, ignore_index=True)
+    de_genes = de_genes.merge(rnas, left_on='Gene name', right_on='synonyms')
 
-    for i, gene in enumerate(degenes["Gene name"]):
-        # Cycle through gene synonyms in rnas.tsv to find EcoCyc name for DE genes
-        found = False
-        for j, synonyms in enumerate(rnas["synonyms"]):
-            if gene in synonyms:
-                model_degenes.append(
-                    degenes.iloc[i,].append(rnas.iloc[j,]).to_frame().T)
-                found = True
-        if not found:
-            print(gene)
-
-    # Concatenating at end is supposed to be more efficient than row-wise append
-    model_degenes = pd.concat(model_degenes, ignore_index=True)
     # Delete these two duplicates that are the incorrect genes
-    model_degenes = model_degenes.loc[~((model_degenes['Gene name']=="acrE") & (model_degenes['common_name']=='acrB'))]
-    model_degenes = model_degenes.loc[~((model_degenes['Gene name']=="acrB") & (model_degenes['common_name']=='gyrB'))]
-    with open("data/marA_binding/TU_id_to_index.json") as f:
-        TU_id_to_index = json.load(f)
+    de_genes = de_genes.loc[~((de_genes['Gene name']=="acrE") & (de_genes['common_name']=='acrB'))]
+    de_genes = de_genes.loc[~((de_genes['Gene name']=="acrB") & (de_genes['common_name']=='gyrB'))]
 
     # Get model RNAs names by appending the "[c]" suffix, then get TU index for RNA
-    TU_idx = [TU_id_to_index[rna_id + "[c]"] for rna_id in model_degenes["id"]]
-    model_degenes["TU_idx"] = TU_idx
-
-    with open("data/wcecoli_t0.json") as f:
-        initial_state = json.load(f)
-    bulk_names = list(initial_state["bulk"].keys())
+    TU_idx = [TU_id_to_index[rna_id + "[c]"] for rna_id in de_genes["id"]]
+    de_genes["TU_idx"] = TU_idx
 
     # Include complexes
-    # These numpy arrays were saved directly from complexation sim data
-    comp_stoich = np.load('data/marA_binding/complexation_stoich.npy')
-    comp_molecules = np.load('data/marA_binding/complexation_molecules.npy')
     comp_rxns = pd.DataFrame(comp_stoich, columns=comp_molecules)
 
     def recursive_search(complex_name, monomers_used):
@@ -126,20 +119,20 @@ def get_IDs(monomer_id):
 
     # Protein IDs have varied suffixes: brute force search
     all_ids = []
-    for monomer_id in model_degenes["monomer_ids"]:
+    for monomer_id in de_genes["monomer_ids"]:
         all_ids.append(get_IDs(monomer_id))
 
-    (model_degenes["bulk_ids"], model_degenes["monomers_used"], 
-        model_degenes["complex_ids"]) = zip(
+    (de_genes["bulk_ids"], de_genes["monomers_used"], 
+        de_genes["complex_ids"]) = zip(
             *[ids for ids in all_ids])
 
     # Add marR-tet complex
-    marR_complex_ids = model_degenes.loc[model_degenes['Gene name'] == 'marR', 'complex_ids'].to_numpy()[0]
+    marR_complex_ids = de_genes.loc[de_genes['Gene name'] == 'marR', 'complex_ids'].to_numpy()[0]
     marR_complex_ids.append('marR-tet[c]')
-    marR_monomers_used = model_degenes.loc[model_degenes['Gene name'] == 'marR', 'monomers_used'].to_numpy()[0]
+    marR_monomers_used = de_genes.loc[de_genes['Gene name'] == 'marR', 'monomers_used'].to_numpy()[0]
     marR_monomers_used.append(-2)
 
-    model_degenes.to_csv(
+    de_genes.to_csv(
         "data/marA_binding/model_degenes.csv", index=False)
 
 if __name__=="__main__":

data/marA_binding/model_degenes.csv

@@ -1,25 +1,25 @@
 Gene name,Fold change,id,common_name,synonyms,type,modified_forms,gene_id,monomer_ids,anticodon,coding_segments,TU_idx,bulk_ids,monomers_used,complex_ids
-MicF,3.29,MICF-RNA,small regulatory RNA MicF,"[""MicF"", ""Stc""]",miscRNA,[],EG30063,[],,[],2493,[],[],[]
-marA,2.99,EG11434_RNA,marA,"[""marA"", ""norB"", ""inaR"", ""cfxB"", ""nfxC"", ""soxQ"", ""b1531"", ""ECK1524""]",mRNA,[],EG11434,"[""PD00365""]",,[],1394,['PD00365[c]'],[],[]
-soxS,2.63,EG10958_RNA,soxS,"[""soxS"", ""b4062"", ""ECK4054""]",mRNA,[],EG10958,"[""PD00406""]",,[],944,['PD00406[c]'],[],[]
-yhiV,2.17,EG12241_RNA,mdtF,"[""mdtF"", ""yhiV"", ""b3514"", ""ECK3498""]",mRNA,[],EG12241,"[""YHIV-MONOMER""]",,[],2112,['YHIV-MONOMER[i]'],[-1],['TRANS-CPLX-204[i]']
-marR,2.1,EG11435_RNA,marR,"[""marR"", ""cfxB"", ""inaR"", ""soxQ"", ""b1530"", ""ECK1523""]",mRNA,[],EG11435,"[""PD00364""]",,[],1395,['PD00364[c]'],"[-2, -2]","['CPLX0-7710[c]', 'marR-tet[c]']"
-ompR,1.58,EG10672_RNA,ompR,"[""ompR"", ""cry"", ""kmt"", ""ompB"", ""b3405"", ""ECK3392""]",mRNA,[],EG10672,"[""OMPR-MONOMER""]",,[],662,['OMPR-MONOMER[c]'],[-2],['CPLX0-8285[c]']
-ompF,1.58,EG10671_RNA,ompF,"[""ompF"", ""nfxB"", ""tolF"", ""cmlB"", ""cry"", ""b0929"", ""ECK0920""]",mRNA,[],EG10671,"[""EG10671-MONOMER""]",,[],661,['EG10671-MONOMER[o]'],[-3],['CPLX0-7534[o]']
-envZ,1.49,EG10269_RNA,envZ,"[""envZ"", ""tpo"", ""perA"", ""ompB"", ""b3404"", ""ECK3391""]",mRNA,[],EG10269,"[""ENVZ-MONOMER""]",,[],262,['ENVZ-MONOMER[p]'],[-2],['ENVZ-CPLX[p]']
-tehA,1.47,EG11883_RNA,tehA,"[""tehA"", ""b1429"", ""ECK1422""]",mRNA,[],EG11883,"[""TEHA-MONOMER""]",,[],1807,['TEHA-MONOMER[i]'],[],[]
-acrA,1.42,EG11703_RNA,acrA,"[""acrA"", ""sipB"", ""Mb"", ""lir"", ""mbl"", ""mtcA"", ""nbsA"", ""b0463"", ""ECK0457""]",mRNA,[],EG11703,"[""EG11703-MONOMER""]",,[],1641,['EG11703-MONOMER[p]'],"[-6, -6]","['CPLX0-3932[i]', 'TRANS-CPLX-201[m]']"
-acrB,1.41,EG11704_RNA,acrB,"[""acrB"", ""acrE"", ""b0462"", ""ECK0456""]",mRNA,[],EG11704,"[""ACRB-MONOMER""]",,[],1642,['ACRB-MONOMER[p]'],[-3],['TRANS-CPLX-201[m]']
-ompX,1.28,EG12117_RNA,ompX,"[""ompX"", ""ybiG"", ""ompP"", ""b0814"", ""ECK0803""]",mRNA,[],EG12117,"[""EG12117-MONOMER""]",,[],2011,['EG12117-MONOMER[o]'],[],[]
-ompC,1.28,EG10670_RNA,ompC,"[""ompC"", ""meoA"", ""par"", ""b2215"", ""ECK2207""]",mRNA,[],EG10670,"[""EG10670-MONOMER""]",,[],660,['EG10670-MONOMER[o]'],[-3],['CPLX0-7533[o]']
-ompA,1.27,EG10669_RNA,ompA,"[""ompA"", ""con"", ""tolG"", ""tut"", ""b0957"", ""ECK0948""]",mRNA,[],EG10669,"[""EG10669-MONOMER""]",,[],659,['EG10669-MONOMER[p]'],[],[]
-acrR,1.26,EG12116_RNA,acrR,"[""acrR"", ""ybaH"", ""b0464"", ""ECK0458""]",mRNA,[],EG12116,"[""EG12116-MONOMER""]",,[],2010,['EG12116-MONOMER[c]'],[],[]
-mdfA,1.12,G6440_RNA,mdfA,"[""mdfA"", ""cmlA"", ""cmr"", ""b0842"", ""ECK0832""]",mRNA,[],G6440,"[""CMR-MONOMER""]",,[],3289,['CMR-MONOMER[i]'],[],[]
-acrD,1.12,EG10014_RNA,acrD,"[""acrD"", ""yffA"", ""b2470"", ""ECK2465""]",mRNA,[],EG10014,"[""ACRD-MONOMER""]",,[],11,['ACRD-MONOMER[i]'],[-3],['CPLX0-3932[i]']
-emrB,1.12,EG11439_RNA,emrB,"[""emrB"", ""b2686"", ""ECK2680""]",mRNA,[],EG11439,"[""EMRB-MONOMER""]",,[],1399,['EMRB-MONOMER[i]'],[-1],['CPLX0-2121[i]']
-rob,1.11,EG11366_RNA,rob,"[""rob"", ""cbpB"", ""robA"", ""b4396"", ""ECK4388""]",mRNA,[],EG11366,"[""PD04418""]",,[],1330,['PD04418[c]'],[],[]
-emrD,1.11,EG11693_RNA,emrD,"[""emrD"", ""yicQ"", ""b3673"", ""ECK3664""]",mRNA,[],EG11693,"[""EMRD-MONOMER""]",,[],1631,['EMRD-MONOMER[i]'],[],[]
-acrF,1.11,EG10267_RNA,acrF,"[""acrF"", ""envD"", ""b3266"", ""ECK3253""]",mRNA,[],EG10267,"[""ACRF-MONOMER""]",,[],260,['ACRF-MONOMER[m]'],[-3],['CPLX0-2141[i]']
-tolC,1.1,EG11009_RNA,tolC,"[""tolC"", ""weeA"", ""colE1-i"", ""mtcB"", ""mukA"", ""refI"", ""toc"", ""b3035"", ""ECK3026""]",mRNA,[],EG11009,"[""EG11009-MONOMER""]",,[],995,['EG11009-MONOMER[m]'],"[-3, -3, -3, -3, -3, -3, -3, -3, -3]","['CPLX0-2121[i]', 'CPLX0-2141[i]', 'CPLX0-2161[i]', 'CPLX0-3932[i]', 'CPLX0-7964[m]', 'TRANS-200-CPLX[m]', 'TRANS-CPLX-201[m]', 'TRANS-CPLX-202[i]', 'TRANS-CPLX-204[i]']"
-acrE,0.97,EG10266_RNA,acrE,"[""acrE"", ""envC"", ""b3265"", ""ECK3252""]",mRNA,[],EG10266,"[""EG10266-MONOMER""]",,[],259,['EG10266-MONOMER[i]'],[-6],['CPLX0-2141[i]']
-marB,0.97,EG11599_RNA,marB,"[""marB"", ""b1532"", ""ECK1525""]",mRNA,[],EG11599,"[""EG11599-MONOMER""]",,[],1543,['EG11599-MONOMER[p]'],[],[]
+MicF,3.29,MICF-RNA,small regulatory RNA MicF,MicF,miscRNA,[],EG30063,[],,[],2468,[],[],[]
+marA,2.99,EG11434_RNA,marA,marA,mRNA,[],EG11434,"[""PD00365""]",,[],1386,['PD00365[c]'],[],[]
+soxS,2.63,EG10958_RNA,soxS,soxS,mRNA,[],EG10958,"[""PD00406""]",,[],942,['PD00406[c]'],[],[]
+yhiV,2.17,EG12241_RNA,mdtF,yhiV,mRNA,[],EG12241,"[""YHIV-MONOMER""]",,[],2093,['YHIV-MONOMER[m]'],[-1],['TRANS-CPLX-204[i]']
+marR,2.1,EG11435_RNA,marR,marR,mRNA,[],EG11435,"[""PD00364""]",,[],1387,['PD00364[c]'],"[-2, -2]","['CPLX0-7710[c]', 'marR-tet[c]']"
+ompR,1.58,EG10672_RNA,ompR,ompR,mRNA,[],EG10672,"[""OMPR-MONOMER""]",,[],660,['OMPR-MONOMER[c]'],[-2],['CPLX0-8285[c]']
+ompF,1.58,EG10671_RNA,ompF,ompF,mRNA,[],EG10671,"[""EG10671-MONOMER""]",,[],659,['EG10671-MONOMER[o]'],[-3],['CPLX0-7534[o]']
+envZ,1.49,EG10269_RNA,envZ,envZ,mRNA,[],EG10269,"[""ENVZ-MONOMER""]",,[],261,['ENVZ-MONOMER[c]'],[-2],['ENVZ-CPLX[c]']
+tehA,1.47,EG11883_RNA,tehA,tehA,mRNA,[],EG11883,"[""TEHA-MONOMER""]",,[],1793,['TEHA-MONOMER[i]'],[],[]
+acrA,1.42,EG11703_RNA,acrA,acrA,mRNA,[],EG11703,"[""EG11703-MONOMER""]",,[],1632,['EG11703-MONOMER[p]'],"[-6, -6]","['CPLX0-3932[i]', 'TRANS-CPLX-201[m]']"
+acrB,1.41,EG11704_RNA,acrB,acrB,mRNA,[],EG11704,"[""ACRB-MONOMER""]",,[],1633,['ACRB-MONOMER[m]'],[-3],['TRANS-CPLX-201[m]']
+ompX,1.28,EG12117_RNA,ompX,ompX,mRNA,[],EG12117,"[""EG12117-MONOMER""]",,[],1995,['EG12117-MONOMER[o]'],[],[]
+ompC,1.28,EG10670_RNA,ompC,ompC,mRNA,[],EG10670,"[""EG10670-MONOMER""]",,[],658,['EG10670-MONOMER[o]'],[-3],['CPLX0-7533[o]']
+ompA,1.27,EG10669_RNA,ompA,ompA,mRNA,[],EG10669,"[""EG10669-MONOMER""]",,[],657,['EG10669-MONOMER[o]'],[],[]
+acrR,1.26,EG12116_RNA,acrR,acrR,mRNA,[],EG12116,"[""EG12116-MONOMER""]",,[],1994,['EG12116-MONOMER[c]'],[],[]
+mdfA,1.12,G6440_RNA,mdfA,mdfA,mRNA,[],G6440,"[""CMR-MONOMER""]",,[],3209,['CMR-MONOMER[i]'],[],[]
+acrD,1.12,EG10014_RNA,acrD,acrD,mRNA,[],EG10014,"[""ACRD-MONOMER""]",,[],10,['ACRD-MONOMER[i]'],[-3],['CPLX0-3932[i]']
+emrB,1.12,EG11439_RNA,emrB,emrB,mRNA,[],EG11439,"[""EMRB-MONOMER""]",,[],1391,['EMRB-MONOMER[i]'],[-1],['CPLX0-2121[i]']
+rob,1.11,EG11366_RNA,rob,rob,mRNA,[],EG11366,"[""PD04418""]",,[],1323,['PD04418[c]'],[],[]
+emrD,1.11,EG11693_RNA,emrD,emrD,mRNA,[],EG11693,"[""EMRD-MONOMER""]",,[],1622,['EMRD-MONOMER[i]'],[],[]
+acrF,1.11,EG10267_RNA,acrF,acrF,mRNA,[],EG10267,"[""ACRF-MONOMER""]",,[],259,['ACRF-MONOMER[m]'],[-3],['CPLX0-2141[i]']
+tolC,1.1,EG11009_RNA,tolC,tolC,mRNA,[],EG11009,"[""EG11009-MONOMER""]",,[],993,['EG11009-MONOMER[m]'],"[-3, -3, -3, -3, -3, -3, -3, -3, -3]","['CPLX0-2121[i]', 'CPLX0-2141[i]', 'CPLX0-2161[i]', 'CPLX0-3932[i]', 'CPLX0-7964[m]', 'TRANS-200-CPLX[m]', 'TRANS-CPLX-201[m]', 'TRANS-CPLX-202[i]', 'TRANS-CPLX-204[i]']"
+acrE,0.97,EG10266_RNA,acrE,acrE,mRNA,[],EG10266,"[""EG10266-MONOMER""]",,[],258,['EG10266-MONOMER[i]'],[-6],['CPLX0-2141[i]']
+marB,0.97,EG11599_RNA,marB,marB,mRNA,[],EG11599,"[""EG11599-MONOMER""]",,[],1534,['EG11599-MONOMER[p]'],[],[]