From c27ee5a090cd6320e8d5d976e8fd62dd5446340e Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Mon, 6 Mar 2023 20:54:36 +0100 Subject: [PATCH 01/10] feat: use Docker image of Memote --- .github/workflows/validation.yml | 47 +++++++++++--------------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 68255f0..f2038d4 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -19,23 +19,16 @@ jobs: with: fetch-depth: 1 - - name: Set up Python 3 - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - name: Install dependencies - run: pip install -r requirements.txt - - id: set-matrix - run: echo "matrix=$(python -c 'import runner; runner.matrix()')" >> $GITHUB_OUTPUT - - - name: Cache pip directory - uses: actions/cache@v3 - id: cache + name: Fetch list of repositories + uses: addnab/docker-run-action@v3 with: - path: ~/.cache/pip - key: ${{ github.sha }} + image: opencobra/memote:latest + options: -v ${{ github.workspace }}:/project + shell: bash + run: | + cd /project + echo "matrix=$(python -c 'import runner; runner.matrix()')" >> $GITHUB_OUTPUT validate: needs: setup @@ -55,23 +48,15 @@ jobs: with: fetch-depth: 1 - - name: Set up Python 3 - uses: actions/setup-python@v4 - with: - python-version: "3.x" - - - name: Restore pip cache directory - uses: actions/cache@v3 - id: cache - with: - path: ~/.cache/pip - key: ${{ github.sha }} - - - name: Install dependencies - run: pip install -r requirements.txt - - name: Validate repository - run: python -c 'import runner; runner.validate("${{ matrix.gem }}")' + uses: addnab/docker-run-action@v3 + with: + image: opencobra/memote:latest + options: -v ${{ github.workspace }}:/project + shell: bash + run: | + cd /project + python -c 'import runner; runner.validate("${{ matrix.gem }}")' - name: Update branch run: git pull --ff From cafbbffbc620f8bf4eb27f38075aa73224bb9ae0 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Mon, 6 Mar 2023 22:11:25 +0100 Subject: [PATCH 02/10] feat: use custom Docker image of Memote --- .github/workflows/validation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index f2038d4..12cbe02 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -23,7 +23,7 @@ jobs: name: Fetch list of repositories uses: addnab/docker-run-action@v3 with: - image: opencobra/memote:latest + image: ghcr.io/metabolicatlas/memote-docker:0.13 options: -v ${{ github.workspace }}:/project shell: bash run: | @@ -51,7 +51,7 @@ jobs: - name: Validate repository uses: addnab/docker-run-action@v3 with: - image: opencobra/memote:latest + image: ghcr.io/metabolicatlas/memote-docker:0.13 options: -v ${{ github.workspace }}:/project shell: bash run: | From 116dcaf10eb7d61a16355c8ab191f1f03838863a Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Mon, 6 Mar 2023 22:21:11 +0100 Subject: [PATCH 03/10] fix: pass gh token to container --- .github/workflows/validation.yml | 46 ++++++++++++++++++++------------ index.json | 0 runner.py | 3 ++- 3 files changed, 31 insertions(+), 18 deletions(-) create mode 100644 index.json diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index 12cbe02..a1df21d 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -8,6 +8,11 @@ on: jobs: setup: runs-on: ubuntu-latest + container: + image: ghcr.io/metabolicatlas/memote-docker:0.13 + volumes: + - ${{ github.workspace }}:/project:rw + options: --user root --workdir /project outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} env: @@ -19,20 +24,32 @@ jobs: with: fetch-depth: 1 - - id: set-matrix - name: Fetch list of repositories - uses: addnab/docker-run-action@v3 + - name: Fetch list of repositories + run: | + git config --global --add safe.directory /__w/standard-GEM-validation/standard-GEM-validation + python -c 'import runner; runner.matrix()' + + - name: Commit index of standard-GEMs + uses: stefanzweifel/git-auto-commit-action@v4 with: - image: ghcr.io/metabolicatlas/memote-docker:0.13 - options: -v ${{ github.workspace }}:/project - shell: bash - run: | - cd /project - echo "matrix=$(python -c 'import runner; runner.matrix()')" >> $GITHUB_OUTPUT + commit_user_name: validation-bot + commit_message: update index of standard-GEMs + file_pattern: results/index.json + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Define job matrix from index + id: set-matrix + run: echo "matrix=$(cat results/index.json)" >> $GITHUB_OUTPUT validate: needs: setup runs-on: ubuntu-latest + container: + image: ghcr.io/metabolicatlas/memote-docker:0.13 + volumes: + - ${{ github.workspace }}:/project:rw + options: --user root --workdir /project continue-on-error: true env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -49,14 +66,9 @@ jobs: fetch-depth: 1 - name: Validate repository - uses: addnab/docker-run-action@v3 - with: - image: ghcr.io/metabolicatlas/memote-docker:0.13 - options: -v ${{ github.workspace }}:/project - shell: bash - run: | - cd /project - python -c 'import runner; runner.validate("${{ matrix.gem }}")' + run: | + git config --global --add safe.directory /__w/standard-GEM-validation/standard-GEM-validation + python -c 'import runner; runner.validate("${{ matrix.gem }}")' - name: Update branch run: git pull --ff diff --git a/index.json b/index.json new file mode 100644 index 0000000..e69de29 diff --git a/runner.py b/runner.py index fed2198..31f9e8a 100644 --- a/runner.py +++ b/runner.py @@ -59,7 +59,8 @@ def releases(nameWithOwner): def matrix(): m = json.dumps(list(gem_repositories())) - print(m) + with open("results/index.json", "w") as file: + file.write(m) def gem_follows_standard(nameWithOwner, release, version): repo_standard = requests.get('https://raw.githubusercontent.com/{}/{}/.standard-GEM.md'.format(nameWithOwner, release)) From b40e69319a863897a08efdf93a19b744145a469d Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Wed, 8 Mar 2023 16:53:49 +0100 Subject: [PATCH 04/10] chore: cleanup --- .github/workflows/validation.yml | 4 ++-- requirements.txt | 5 ----- runner.py | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) delete mode 100644 requirements.txt diff --git a/.github/workflows/validation.yml b/.github/workflows/validation.yml index a1df21d..2d67b51 100644 --- a/.github/workflows/validation.yml +++ b/.github/workflows/validation.yml @@ -34,13 +34,13 @@ jobs: with: commit_user_name: validation-bot commit_message: update index of standard-GEMs - file_pattern: results/index.json + file_pattern: index.json env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Define job matrix from index id: set-matrix - run: echo "matrix=$(cat results/index.json)" >> $GITHUB_OUTPUT + run: echo "matrix=$(cat index.json)" >> $GITHUB_OUTPUT validate: needs: setup diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c5ff981..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -requests -yamllint -cobra -scipy -memote \ No newline at end of file diff --git a/runner.py b/runner.py index 31f9e8a..d568e78 100644 --- a/runner.py +++ b/runner.py @@ -59,7 +59,7 @@ def releases(nameWithOwner): def matrix(): m = json.dumps(list(gem_repositories())) - with open("results/index.json", "w") as file: + with open("index.json", "w") as file: file.write(m) def gem_follows_standard(nameWithOwner, release, version): From 32d044164b3946833b1d23e56c124d413bfb7d6c Mon Sep 17 00:00:00 2001 From: mihai-sysbio Date: Wed, 8 Mar 2023 15:54:24 +0000 Subject: [PATCH 05/10] chore: remove duplicate file --- index.json | 1 + 1 file changed, 1 insertion(+) diff --git a/index.json b/index.json index e69de29..c4d67cc 100644 --- a/index.json +++ b/index.json @@ -0,0 +1 @@ +["SysBioChalmers/yeast-GEM", "SysBioChalmers/Human-GEM", "SysBioChalmers/Fruitfly-GEM", "SysBioChalmers/Mouse-GEM", "SysBioChalmers/Sco-GEM", "SysBioChalmers/Zebrafish-GEM", "iAMB-RWTH-Aachen/Opol-GSMM", "SysBioChalmers/Worm-GEM", "haowang-bioinfo/Ecoli-GEM", "SysBioChalmers/Rat-GEM", "tibbdc/vna-GEM"] \ No newline at end of file From 0f90cbc311ba69b5aaacd65c9d6d6abb11ca064e Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Mon, 13 Mar 2023 11:58:30 +0100 Subject: [PATCH 06/10] fix: check file exists and size --- tests/cobra.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/cobra.py b/tests/cobra.py index 91c6f7c..ca337a6 100644 --- a/tests/cobra.py +++ b/tests/cobra.py @@ -1,6 +1,6 @@ import cobra import json -from os.path import exists +from os.path import exists, getsize def load(model_filename): print(' load model with cobrapy') @@ -9,10 +9,12 @@ def load(model_filename): try: cobra.io.load_yaml_model(model_filename + '.yml') cobra.io.read_sbml_model(model_filename + '.xml') - if exists(model_filename + '.mat'): - cobra.io.load_matlab_model(model_filename + '.mat') - if exists(model_filename + '.json'): - cobra.io.load_json_model(model_filename + '.json') + matFile = model_filename + '.matFile' + if exists(matFile) and getsize(matFile) > 0: + cobra.io.load_matlab_model(matFile) + jsonFile = model_filename + '.json' + if exists(jsonFile) and getsize(jsonFile) > 0: + cobra.io.load_json_model(jsonFile) is_valid_cobrapy = True except Exception as e: errors = json.dumps(str(e)) From aefb80dd9265ea0601d09c8542631fd435ffec94 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Mon, 13 Mar 2023 22:37:49 +0100 Subject: [PATCH 07/10] fix: memote params --- tests/cobra.py | 18 +++++++++++------- tests/memote.py | 8 ++++---- tests/yaml.py | 5 +++-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/cobra.py b/tests/cobra.py index ca337a6..467dd78 100644 --- a/tests/cobra.py +++ b/tests/cobra.py @@ -2,30 +2,34 @@ import json from os.path import exists, getsize -def load(model_filename): +def load(model_name): print(' load model with cobrapy') is_valid_cobrapy = False errors = '' try: - cobra.io.load_yaml_model(model_filename + '.yml') - cobra.io.read_sbml_model(model_filename + '.xml') - matFile = model_filename + '.matFile' + cobra.io.load_yaml_model(model_name + '.yml') + print("yml done") + cobra.io.read_sbml_model(model_name + '.xml') + print("xml done") + matFile = model_name + '.mat' if exists(matFile) and getsize(matFile) > 0: cobra.io.load_matlab_model(matFile) - jsonFile = model_filename + '.json' + print("mat done") + jsonFile = model_name + '.json' if exists(jsonFile) and getsize(jsonFile) > 0: cobra.io.load_json_model(jsonFile) + print("json done") is_valid_cobrapy = True except Exception as e: errors = json.dumps(str(e)) print(e) return {'cobrapy-load': { cobra.__version__ : is_valid_cobrapy, 'errors': errors } } -def validateSBML(model_filename): +def validateSBML(model_name): print(' validate sbml with cobrapy') is_valid_sbml = False try: - _, result = cobra.io.sbml.validate_sbml_model(model_filename + '.xml') + _, result = cobra.io.sbml.validate_sbml_model(model_name + '.xml') if result != {}: raise Exception(result) except Exception as e: diff --git a/tests/memote.py b/tests/memote.py index f140b17..4f8a2e9 100644 --- a/tests/memote.py +++ b/tests/memote.py @@ -2,13 +2,13 @@ import json import memote -def scoreAnnotationAndConsistency(model_filename): +def scoreAnnotationAndConsistency(model_name): print(' memote scoring') - memote_score = 'Scoring failed' + memote_score = False errors = '' try: - model = cobra.io.read_sbml_model(model_filename + '.xml') - _, results = memote.suite.api.test_model(model, None, True, None, {"basic", "annotation", "consistency"}) + model = cobra.io.read_sbml_model(model_name + '.xml') + _, results = memote.suite.api.test_model(model=model, results=True, exclusive=['basic', 'annotation', 'consistency']) processed_results = memote.suite.api.snapshot_report(results, None, False) results_json = json.loads(processed_results) memote_score = results_json['score']['total_score'] diff --git a/tests/yaml.py b/tests/yaml.py index 9c1d11a..8a2a4ad 100644 --- a/tests/yaml.py +++ b/tests/yaml.py @@ -2,18 +2,19 @@ from yamllint.config import YamlLintConfig import json -def validate(model_filename): +def validate(model_name): print(' validate YAML with yamllint') is_valid_yaml = False errors = '' try: conf = YamlLintConfig('{extends: default, rules: {line-length: disable}}') - with open(model_filename + '.yml', 'r') as file: + with open(model_name + '.yml', 'r') as file: errors = list(map(str, yamllint.linter.run(file, conf))) if len(errors) == 0: is_valid_yaml = True else: errors = json.dumps(errors) except Exception as e: + errors = json.dumps(str(e)) print(e) return {'yamllint': { yamllint.__version__ : str(is_valid_yaml), 'errors': errors } } \ No newline at end of file From 5b7bcdaf6c88d3a6485d65945b58deb33fa46a10 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Tue, 14 Mar 2023 09:35:18 +0100 Subject: [PATCH 08/10] refactor: split cobra load --- runner.py | 9 +++++--- tests/cobra.py | 61 ++++++++++++++++++++++++++++++++++--------------- tests/memote.py | 3 ++- tests/yaml.py | 8 +++---- 4 files changed, 55 insertions(+), 26 deletions(-) diff --git a/runner.py b/runner.py index d568e78..744dc21 100644 --- a/runner.py +++ b/runner.py @@ -9,7 +9,7 @@ API_TOKEN = environ['GH_TOKEN'] MODEL_FILENAME = 'model' MODEL_FORMATS = ['.yml', '.xml', '.mat', '.json'] -RELEASES = 10 +RELEASES = 1 header_auth = {'Authorization': 'token %s' % API_TOKEN} additional_branch_tags = [] @@ -91,8 +91,11 @@ def validate(nameWithOwner): with open(my_model, 'w') as file: file.write(response.text) test_results.update(tests.yaml.validate(model)) - test_results.update(tests.cobra.load(model)) - test_results.update(tests.cobra.validateSBML(model)) + test_results.update(tests.cobra.loadYaml(model)) + test_results.update(tests.cobra.loadSbml(model)) + test_results.update(tests.cobra.loadMatlab(model)) + test_results.update(tests.cobra.loadJson(model)) + test_results.update(tests.cobra.validateSbml(model)) test_results.update(tests.memote.scoreAnnotationAndConsistency(model)) else: print('is not following standard') diff --git a/tests/cobra.py b/tests/cobra.py index 467dd78..1f88b10 100644 --- a/tests/cobra.py +++ b/tests/cobra.py @@ -1,32 +1,57 @@ import cobra import json -from os.path import exists, getsize -def load(model_name): - print(' load model with cobrapy') - is_valid_cobrapy = False +def loadYaml(model_name): + print('load yaml') + is_valid = False errors = '' try: cobra.io.load_yaml_model(model_name + '.yml') - print("yml done") + is_valid = True + except Exception as e: + errors = json.dumps(str(e)) + print(e) + return {'cobrapy-load-yaml': { cobra.__version__ : is_valid, 'errors': errors } } + +def loadSbml(model_name): + print('load sbml') + is_valid = False + errors = '' + try: cobra.io.read_sbml_model(model_name + '.xml') - print("xml done") - matFile = model_name + '.mat' - if exists(matFile) and getsize(matFile) > 0: - cobra.io.load_matlab_model(matFile) - print("mat done") - jsonFile = model_name + '.json' - if exists(jsonFile) and getsize(jsonFile) > 0: - cobra.io.load_json_model(jsonFile) - print("json done") - is_valid_cobrapy = True + is_valid = True except Exception as e: errors = json.dumps(str(e)) print(e) - return {'cobrapy-load': { cobra.__version__ : is_valid_cobrapy, 'errors': errors } } + return {'cobrapy-load-sbml': { cobra.__version__ : is_valid, 'errors': errors } } + +def loadMatlab(model_name): + print('load matlab') + is_valid = False + errors = '' + try: + cobra.io.load_matlab_model(model_name + '.mat') + is_valid = True + except Exception as e: + errors = json.dumps(str(e)) + print(e) + return {'cobrapy-load-matlab': { cobra.__version__ : is_valid, 'errors': errors } } + +def loadJson(model_name): + print('load json') + is_valid = False + errors = '' + try: + cobra.io.load_json_model(model_name + '.json') + is_valid = True + except Exception as e: + errors = json.dumps(str(e)) + print(e) + return {'cobrapy-load-json': { cobra.__version__ : is_valid, 'errors': errors } } + -def validateSBML(model_name): - print(' validate sbml with cobrapy') +def validateSbml(model_name): + print('validate sbml with cobrapy') is_valid_sbml = False try: _, result = cobra.io.sbml.validate_sbml_model(model_name + '.xml') diff --git a/tests/memote.py b/tests/memote.py index 4f8a2e9..91739dd 100644 --- a/tests/memote.py +++ b/tests/memote.py @@ -3,7 +3,7 @@ import memote def scoreAnnotationAndConsistency(model_name): - print(' memote scoring') + print('memote scoring') memote_score = False errors = '' try: @@ -11,6 +11,7 @@ def scoreAnnotationAndConsistency(model_name): _, results = memote.suite.api.test_model(model=model, results=True, exclusive=['basic', 'annotation', 'consistency']) processed_results = memote.suite.api.snapshot_report(results, None, False) results_json = json.loads(processed_results) + print(results_json['score']) memote_score = results_json['score']['total_score'] except Exception as e: errors = json.dumps(str(e)) diff --git a/tests/yaml.py b/tests/yaml.py index 8a2a4ad..6272266 100644 --- a/tests/yaml.py +++ b/tests/yaml.py @@ -3,18 +3,18 @@ import json def validate(model_name): - print(' validate YAML with yamllint') - is_valid_yaml = False + print('validate YAML with yamllint') + is_valid = False errors = '' try: conf = YamlLintConfig('{extends: default, rules: {line-length: disable}}') with open(model_name + '.yml', 'r') as file: errors = list(map(str, yamllint.linter.run(file, conf))) if len(errors) == 0: - is_valid_yaml = True + is_valid = True else: errors = json.dumps(errors) except Exception as e: errors = json.dumps(str(e)) print(e) - return {'yamllint': { yamllint.__version__ : str(is_valid_yaml), 'errors': errors } } \ No newline at end of file + return {'yamllint': { yamllint.__version__ : is_valid, 'errors': errors } } \ No newline at end of file From ac2a9ab8a5eb66e083a959b113f0c999b8159058 Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Tue, 14 Mar 2023 11:43:05 +0100 Subject: [PATCH 09/10] fix: check if requests succeed --- runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/runner.py b/runner.py index 744dc21..11e6703 100644 --- a/runner.py +++ b/runner.py @@ -87,9 +87,10 @@ def validate(nameWithOwner): if gem_is_standard: for model_format in MODEL_FORMATS: my_model = model + model_format - response = requests.get('https://raw.githubusercontent.com/{}/{}/model/{}'.format(nameWithOwner, model_release, my_model)) - with open(my_model, 'w') as file: - file.write(response.text) + response = requests.get('https://raw.githubusercontent.com/{}/{}/model/{}'.format(nameWithOwner, model_release, my_model), timeout=10) + if response.ok: + with open(my_model, 'w') as file: + file.write(response.text) test_results.update(tests.yaml.validate(model)) test_results.update(tests.cobra.loadYaml(model)) test_results.update(tests.cobra.loadSbml(model)) From be22f8e2ea73fc88a67e2a8f2cc29dfdf86abcef Mon Sep 17 00:00:00 2001 From: Mihail Anton Date: Tue, 14 Mar 2023 11:43:25 +0100 Subject: [PATCH 10/10] fix: memote test listing --- tests/memote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/memote.py b/tests/memote.py index 91739dd..36e5259 100644 --- a/tests/memote.py +++ b/tests/memote.py @@ -8,8 +8,8 @@ def scoreAnnotationAndConsistency(model_name): errors = '' try: model = cobra.io.read_sbml_model(model_name + '.xml') - _, results = memote.suite.api.test_model(model=model, results=True, exclusive=['basic', 'annotation', 'consistency']) - processed_results = memote.suite.api.snapshot_report(results, None, False) + _, results = memote.suite.api.test_model(model=model, results=True, exclusive=['test_stoichiometric_consistency', 'test_reaction_mass_balance', 'test_reaction_charge_balance', 'test_find_disconnected', 'test_find_reactions_unbounded_flux_default_condition', 'test_metabolite_annotation_presence', 'test_metabolite_annotation_overview', 'test_metabolite_annotation_wrong_ids', 'test_metabolite_id_namespace_consistency', 'test_reaction_annotation_presence', 'test_reaction_annotation_overview', 'test_reaction_annotation_wrong_ids', 'test_reaction_id_namespace_consistency', 'test_gene_product_annotation_presence', 'test_gene_product_annotation_overview', 'test_gene_product_annotation_wrong_ids', 'test_model_id_presence', 'test_metabolites_presence', 'test_reactions_presence', 'test_genes_presence', 'test_compartments_presence', 'test_metabolic_coverage', 'test_unconserved_metabolites', 'test_inconsistent_min_stoichiometry', 'test_find_unique_metabolites', 'test_find_duplicate_metabolites_in_compartments', 'test_metabolites_charge_presence', 'test_metabolites_formula_presence', 'test_find_medium_metabolites', 'test_find_pure_metabolic_reactions', 'test_find_constrained_pure_metabolic_reactions', 'test_find_transport_reactions', 'test_find_constrained_transport_reactions', 'test_find_candidate_irreversible_reactions', 'test_find_reactions_with_partially_identical_annotations', 'test_find_duplicate_reactions', 'test_find_reactions_with_identical_genes']) + processed_results = memote.suite.api.snapshot_report(results, config=None, html=False) results_json = json.loads(processed_results) print(results_json['score']) memote_score = results_json['score']['total_score']