From 8ff203f8267b08fd581992a5b72bbd8976d89445 Mon Sep 17 00:00:00 2001 From: paulzierep Date: Thu, 11 Jul 2024 17:05:03 +0200 Subject: [PATCH] use checkout twice and sparse checkout --- .github/workflows/fetch_all_tools.yaml | 58 ++++++++++++----------- .github/workflows/filter_communities.yaml | 39 ++++++++++----- .github/workflows/run_tests.yaml | 26 ++++++---- .github/workflows/static.yml | 2 +- 4 files changed, 76 insertions(+), 49 deletions(-) diff --git a/.github/workflows/fetch_all_tools.yaml b/.github/workflows/fetch_all_tools.yaml index f627dfa8..ab8793ea 100644 --- a/.github/workflows/fetch_all_tools.yaml +++ b/.github/workflows/fetch_all_tools.yaml @@ -74,31 +74,33 @@ jobs: needs: fetch-all-tools-stepwise name: Fetch all tools merge steps: - - uses: actions/checkout@v4 - with: - ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439) - name: checkout results - - run: | - git checkout results -- results - - - uses: actions/setup-python@v5 - - name: Install requirement - run: | - python -m pip install -r requirements.txt - sudo apt-get install jq - - name: Merge all tools - run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry) - awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv - jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json - - name: Wordcloud and interactive table - run: | - bash bin/format_tools.sh - - name: Commit all tools - # add or commit any changes in results if there was a change, merge with main and push as bot - uses: s0/git-publish-subdir-action@develop - env: - BRANCH: results - FOLDER: results - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: self - SKIP_EMPTY_COMMITS: true \ No newline at end of file + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439) + - name: Checkout results + uses: actions/checkout@v4 + with: + ref: results + sparse-checkout: results + - uses: actions/setup-python@v5 + - name: Install requirement + run: | + python -m pip install -r requirements.txt + sudo apt-get install jq + - name: Merge all tools + run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry) + awk 'FNR==1 && NR!=1{next;}{print}' results/repositories*.list_tools.tsv > results/all_tools.tsv + jq -s 'map(.[])' results/repositories*.list_tools.json > results/all_tools.json + - name: Wordcloud and interactive table + run: | + bash bin/format_tools.sh + - name: Commit all tools + # add or commit any changes in results if there was a change, merge with main and push as bot + uses: s0/git-publish-subdir-action@develop + env: + BRANCH: results + FOLDER: results + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: self + SKIP_EMPTY_COMMITS: true \ No newline at end of file diff --git a/.github/workflows/filter_communities.yaml b/.github/workflows/filter_communities.yaml index 6ac1c745..1572fc91 100644 --- a/.github/workflows/filter_communities.yaml +++ b/.github/workflows/filter_communities.yaml @@ -28,10 +28,15 @@ jobs: filter-all-tutorials: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 - name: checkout results - run: | - git checkout results -- results + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439) + - name: Checkout results + uses: actions/checkout@v4 + with: + ref: results + sparse-checkout: results - uses: actions/setup-python@v5 with: python-version: '3.8' @@ -53,10 +58,15 @@ jobs: update-tools-to-keep-exclude: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 - name: checkout results - run: | - git checkout results -- results + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439) + - name: Checkout results + uses: actions/checkout@v4 + with: + ref: results + sparse-checkout: results - uses: actions/setup-python@v5 with: python-version: '3.8' @@ -78,10 +88,15 @@ jobs: filter-all-tools: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 - name: checkout results - run: | - git checkout results -- results + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439) + - name: Checkout results + uses: actions/checkout@v4 + with: + ref: results + sparse-checkout: results - uses: actions/setup-python@v5 with: python-version: '3.8' diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 10af6976..e60c4ced 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -6,10 +6,15 @@ jobs: runs-on: ubuntu-20.04 environment: fetch-tools steps: - - uses: actions/checkout@v4 - name: checkout results - run: | - git checkout results -- results + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439) + - name: Checkout results + uses: actions/checkout@v4 + with: + ref: results + sparse-checkout: results - uses: actions/setup-python@v5 with: python-version: '3.11' @@ -29,10 +34,15 @@ jobs: test-tutorials: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v4 - name: checkout results - run: | - git checkout results -- results + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439) + - name: Checkout results + uses: actions/checkout@v4 + with: + ref: results + sparse-checkout: results - uses: actions/setup-python@v5 with: python-version: '3.11' diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml index 6e33297a..afa9a3ab 100644 --- a/.github/workflows/static.yml +++ b/.github/workflows/static.yml @@ -36,7 +36,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - ref: 'results' + ref: results - name: Setup Pages uses: actions/configure-pages@v4 - name: Upload artifact