Skip to content

Additional reagents associated with doi: 10.1101/2024.02.28.582563 #287

Additional reagents associated with doi: 10.1101/2024.02.28.582563

Additional reagents associated with doi: 10.1101/2024.02.28.582563 #287

Workflow file for this run

name: IBEX Knowledge-Base Data Validation and Update
on:
push:
branches:
- main
paths-ignore: # ignore auto-generated files, only run when source files change
- docs/contrib.md
- docs/fluorescent_probes.md
- docs/index.md
- docs/protocols.md
- docs/publications.md
- docs/reagent_resources.md
- docs/the_who.md
- docs/videos.md
- docs/data_and_software.md
pull_request:
branches:
- main
# Enable manual running of workflow, so that if the validation or
# markdown generation code changes we can force execution
workflow_dispatch:
jobs:
preliminary_test:
# run all pre-commit tests (see .pre-commit-config.yaml)
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Run pre-commit tests
run: |
pip install pre-commit
pre-commit run --all-files
pr_preliminary_test:
# check that no auto-generated file was modified in a PR on main so just check we're dealing with a PR
# (workflow "on" event settings are all on main branch)
needs: preliminary_test
if: github.event_name == 'pull_request'
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
autogenerated_docs:
- docs/contrib.md
- docs/fluorescent_probes.md
- docs/index.md
- docs/protocols.md
- docs/publications.md
- docs/reagent_resources.md
- docs/the_who.md
- docs/videos.md
- docs/data_and_software.md
- name: modified autogenerated docs # succeed if no auto-generated files modified, otherwise fail
if: steps.filter.outputs.autogenerated_docs != 'false'
run: |
echo "The pull request includes changes to auto-generated files, this is not allowed. Please modify the relevant files in the docs_in and data directories."
exit 1
validate_data:
needs: preliminary_test
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install dependencies and validate data
run: | # validate the various data files, update the wheel version here whenever the validation scripts are updated
pip install --upgrade pip
pip install https://github.com/IBEXImagingCommunity/ibex_imaging_knowledge_base_utilities/releases/download/v0.8.2/ibex_imaging_knowledge_base_utilities-0.8.2-py3-none-any.whl
validate_zenodo_json .zenodo.json
validate_bibfile data/publications.bib
validate_basic data/protocols.csv .github/data_validation_configs/protocols.json
validate_basic data/vendor_urls.csv .github/data_validation_configs/vendors.json
validate_basic data/datasets.csv .github/data_validation_configs/datasets.json
validate_basic data/software.csv .github/data_validation_configs/software.json
validate_basic data/fluorescent_probes.csv .github/data_validation_configs/fluorescent_probes.json
validate_videos data/videos.csv .github/data_validation_configs/videos.json .zenodo.json
validate_image_resources data/image_resources.csv .github/data_validation_configs/image_resources.json docs/supporting_material
validate_reagent_resources data/reagent_resources.csv .github/data_validation_configs/reagent_resources.json .zenodo.json data/vendor_urls.csv docs/supporting_material
data_2_md:
# run the data to markdown updates only when pushing to the main branch or forced execution via workflow_dispatch
needs: validate_data
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch'}}
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install dependencies
run: |
brew update
brew install pandoc
pip install --upgrade pip
pip install https://github.com/IBEXImagingCommunity/ibex_imaging_knowledge_base_utilities/releases/download/v0.7.0/ibex_imaging_knowledge_base_utilities-0.7.0-py3-none-any.whl
- name: Fine tune data to markdown conversion
id: ft
shell: bash
run: |
convert_bib=0
convert_reagents=0
update_index=0
convert_fluorescent=0
convert_the_who=0
update_how_to_contrib=0
convert_protocols=0
convert_videos=0
convert_data_software=0
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
convert_bib=1
convert_reagents=1
update_index=1
convert_fluorescent=1
convert_the_who=1
update_how_to_contrib=1
convert_protocols=1
convert_videos=1
convert_data_software=1
else
if ! git diff --quiet HEAD\^ HEAD -- data/publications.bib data/ibex.csl; then
convert_bib=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/reagent_resources.md.in data/reagent_resources.csv data/vendor_urls.csv; then
convert_reagents=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/index.md.in data/reagent_resources.csv; then
update_index=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/fluorescent_probes.md.in data/fluorescent_probes.csv; then
convert_fluorescent=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/the_who.md.in .zenodo.json; then
convert_the_who=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/contrib.md.in data/reagent_data_dict.csv data/reagent_glossary.csv; then
update_how_to_contrib=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/protocols.md.in data/protocols.csv; then
convert_protocols=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/videos.md.in data/videos.csv; then
convert_videos=1
fi
if ! git diff --quiet HEAD\^ HEAD -- docs_in/data_and_software.md.in data/datasets.csv data/software.csv; then
convert_data_software=1
fi
fi
echo "convert_bib=$convert_bib" >> $GITHUB_OUTPUT
echo "convert_reagents=$convert_reagents" >> $GITHUB_OUTPUT
echo "update_index=$update_index" >> $GITHUB_OUTPUT
echo "convert_fluorescent=$convert_fluorescent" >> $GITHUB_OUTPUT
echo "convert_the_who=$convert_the_who" >> $GITHUB_OUTPUT
echo "update_how_to_contrib=$update_how_to_contrib" >> $GITHUB_OUTPUT
echo "convert_protocols=$convert_protocols" >> $GITHUB_OUTPUT
echo "convert_videos=$convert_videos" >> $GITHUB_OUTPUT
echo "convert_data_software=$convert_data_software" >> $GITHUB_OUTPUT
- name: Convert bibliography to markdown
if: ${{ steps.ft.outputs.convert_bib == '1' }}
run: |
pandoc --version
bib2md data/publications.bib data/ibex.csl docs/publications.md
- name: Convert reagent resources csv to markdown
if: ${{ steps.ft.outputs.convert_reagents == '1' }}
run: |
reagent_resources_csv_2_md_url docs_in/reagent_resources.md.in data/reagent_resources.csv data/vendor_urls.csv docs/supporting_material
- name: Update the index.md file statistics
if: ${{ steps.ft.outputs.update_index == '1' }}
run: |
update_index_md_stats docs_in/index.md.in data/reagent_resources.csv docs/
- name: Convert fluorescent probes csv to markdown
if: ${{ steps.ft.outputs.convert_fluorescent == '1' }}
run: |
fluorescent_probes_csv_2_md docs_in/fluorescent_probes.md.in data/fluorescent_probes.csv docs/
- name: Convert Zenodo data to contributors list markdown
if: ${{ steps.ft.outputs.convert_the_who == '1' }}
run: |
zenodo_json_2_thewho_md docs_in/the_who.md.in .zenodo.json docs/
- name: Update how to contribute markdown
if: ${{ steps.ft.outputs.update_how_to_contrib == '1' }}
run: |
data_dict_glossary_2_contrib_md docs_in/contrib.md.in data/reagent_data_dict.csv data/reagent_glossary.csv docs/
- name: Convert protocols csv to markdown
if: ${{ steps.ft.outputs.convert_protocols == '1' }}
run: |
protocols_csv_2_md docs_in/protocols.md.in data/protocols.csv docs/
- name: Convert videos csv to markdown
if: ${{ steps.ft.outputs.convert_videos == '1' }}
run: |
videos_csv_2_md docs_in/videos.md.in data/videos.csv docs/
- name: Convert datasets and software csv files to markdown
if: ${{ steps.ft.outputs.convert_data_software == '1' }}
run: |
data_software_csv_2_md docs_in/data_and_software.md.in data/datasets.csv data/software.csv docs/
- name: Commit and push
shell: bash
run: |
if ! git diff --quiet; then
git config --local user.email "$(git log --format='%ae' HEAD^!)"
git config --local user.name "$(git log --format='%an' HEAD^!)"
git add docs/*.md
git commit -m "Adding converted markdowns."
git push
fi