87 2024 updates #158
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Visual diff for PRs | |
| on: | |
| pull_request: | |
| branches: | |
| - '**' | |
| jobs: | |
| check_possible: | |
| name: Check visual diff is possible | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Python Dependencies (PR) | |
| run: | | |
| pip install -r cicd/compare/requirements-pr.txt | |
| - name: Show Pending on PR | |
| run: | | |
| python -m cicd.compare.pr.check_possible $GITHUB_REPOSITORY $GITHUB_REF | |
| visual_diff: | |
| name: Update PR with visual diff | |
| needs: check_possible | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 # required to check out older versions for comparison | |
| - name: Setup Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Python Dependencies (PR) | |
| run: | | |
| pip install -r cicd/compare/requirements-pr.txt | |
| - name: Show Pending on PR | |
| run: | | |
| python -m cicd.compare.pr.show_pending $GITHUB_REPOSITORY $GITHUB_REF | |
| env: | |
| PAT_GITHUB_API: ${{ secrets.PAT_GITHUB_API }} | |
| - name: Get SHAs | |
| run: | | |
| python -m cicd.compare.pr.get_shas $GITHUB_REPOSITORY $GITHUB_REF | |
| - name: Read to Var (SHAs) | |
| id: read_to_var_shas | |
| run: | | |
| echo "::set-output name=PR_SHA1::$(head -n 1 cicd/compare/pr/shas.txt)" | |
| echo "::set-output name=PR_SHA2::$(head -n 2 cicd/compare/pr/shas.txt | tail -n 1)" | |
| - name: Validate Comparison | |
| run: | | |
| cicd/compare/scripts/check_comparable.sh before=${{ steps.read_to_var_shas.outputs.PR_SHA1 }} after=${{ steps.read_to_var_shas.outputs.PR_SHA2 }} | |
| - name: Create Directories | |
| run: | | |
| . cicd/compare/scripts/create_directories.sh before=${{ steps.read_to_var_shas.outputs.PR_SHA1 }} after=${{ steps.read_to_var_shas.outputs.PR_SHA2 }} | |
| echo $COMPARE_DIR > compare.dir | |
| echo $OUTPUT_BASE > output-base.dir | |
| - name: Read to Var (Dirs) | |
| id: read_to_var_dirs | |
| run: | | |
| echo "::set-output name=COMPARE_DIR::$(head -n 1 compare.dir)" | |
| echo "::set-output name=OUTPUT_BASE::$(head -n 1 output-base.dir)" | |
| - name: Create Directories | |
| run: | | |
| mkdir -p ${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA1 }} | |
| mkdir -p ${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA2 }} | |
| - name: Generate Exports | |
| run: | | |
| cicd/scripts/pull_or_build.sh repo=tomfumb image=qgis-exporter:3 build_dir=cicd/export/docker context_dir=cicd/export upload_if_missing=1 | |
| docker run --rm -e REVISION=${{ steps.read_to_var_shas.outputs.PR_SHA1 }} -v $PWD:/code tomfumb/qgis-exporter:3 /workdir/cicd/export/docker/generate.sh output_base=/code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA1 }} png=1 permit_label_locks=1 | |
| docker run --rm -e REVISION=${{ steps.read_to_var_shas.outputs.PR_SHA2 }} -v $PWD:/code tomfumb/qgis-exporter:3 /workdir/cicd/export/docker/generate.sh output_base=/code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA2 }} png=1 permit_label_locks=1 | |
| env: | |
| PAT_DOCKER_HUB: ${{ secrets.PAT_DOCKER_HUB }} | |
| - name: Generate OSM XML | |
| run: | | |
| cicd/osm/scripts/export.sh local_output_dir=${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA1 }} revision=${{ steps.read_to_var_shas.outputs.PR_SHA1 }} | |
| cicd/osm/scripts/export.sh local_output_dir=${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA2 }} revision=${{ steps.read_to_var_shas.outputs.PR_SHA2 }} | |
| - name: Identify change MBRs | |
| run: | | |
| cicd/scripts/pull_or_build.sh repo=tomfumb image=bvnordic-osm-renderer:3 build_dir=cicd/osm/docker/renderer context_dir=cicd upload_if_missing=1 | |
| docker run --rm -e REVISION=${{ steps.read_to_var_shas.outputs.PR_SHA1 }} -v $PWD:/code tomfumb/bvnordic-osm-renderer:3 ogr2ogr -f "GPKG" /code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/compare.gpkg /workdir/main-data.gpkg -nln before -sql "SELECT * FROM Trails" | |
| docker run --rm -e REVISION=${{ steps.read_to_var_shas.outputs.PR_SHA2 }} -v $PWD:/code tomfumb/bvnordic-osm-renderer:3 ogr2ogr -f "GPKG" /code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/compare.gpkg /workdir/main-data.gpkg -nln after -update -sql "SELECT * FROM Trails" | |
| docker run --rm -v $PWD:/code tomfumb/bvnordic-osm-renderer:3 ogr2ogr -f "GeoJSON" /code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/parts_3857.geojson /code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/compare.gpkg -nln diff -t_srs "EPSG:3857" -dialect sqlite -sql "SELECT geom FROM (SELECT ST_Envelope(ST_Transform(ST_Difference(a.geom, b.geom), 3857)) AS geom FROM after a LEFT JOIN before b ON a.fid = b.fid UNION SELECT ST_Envelope(ST_Transform(ST_Difference(b.geom, a.geom), 3857)) AS geom FROM before b LEFT JOIN after a ON b.fid = a.fid) WHERE geom IS NOT NULL" | |
| docker run --rm -v $PWD:/code tomfumb/bvnordic-osm-renderer:3 ogr2ogr -f "GeoJSON" /code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/diff_3857.geojson /code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/parts_3857.geojson -nln diff -t_srs "EPSG:3857" -dialect sqlite -sql "SELECT ST_Union(Geometry) FROM diff" | |
| - name: Render OSM | |
| run: | | |
| cicd/scripts/pull_or_build.sh repo=tomfumb image=bvnordic-osm-renderer:3 build_dir=cicd/osm/docker/renderer context_dir=cicd upload_if_missing=1 | |
| docker run --rm -e REVISION=${{ steps.read_to_var_shas.outputs.PR_SHA1 }} -v $PWD:/code tomfumb/bvnordic-osm-renderer:3 /workdir/cicd/osm/docker/renderer/render_changes.sh data_dir=/code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA1 }}/main changes_3857=/code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/diff_3857.geojson | |
| docker run --rm -e REVISION=${{ steps.read_to_var_shas.outputs.PR_SHA2 }} -v $PWD:/code tomfumb/bvnordic-osm-renderer:3 /workdir/cicd/osm/docker/renderer/render_changes.sh data_dir=/code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA2 }}/main changes_3857=/code/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/diff_3857.geojson | |
| - name: Python Dependencies (Change Detection) | |
| run: | | |
| pip install -r cicd/compare/requirements-detect.txt | |
| - name: Detect Changes | |
| run: | | |
| python -m cicd.compare.detect_changes ${{ steps.read_to_var_shas.outputs.PR_SHA1 }} ${{ steps.read_to_var_shas.outputs.PR_SHA2 }} $PWD/${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }} --before_after_exclude bvnordic\\.osm\\-.+ | |
| - name: Remove Unnecessary Files | |
| run: | | |
| sudo rm -rf ${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA1 }} | |
| sudo rm -rf ${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/${{ steps.read_to_var_shas.outputs.PR_SHA2 }} | |
| - name: Python Dependencies (upload) | |
| run: | | |
| pip install -r cicd/upload/requirements.txt | |
| - name: Upload to Google Drive | |
| run: | | |
| python -m cicd.upload.upload ${{ steps.read_to_var_dirs.outputs.COMPARE_DIR }} $PWD/output | |
| env: | |
| GDRIVE_UPLOAD_SERVICE_ACCT_INFO: ${{ secrets.GDRIVE_PR_UPLOAD_SERVICE_ACCT_INFO }} | |
| - name: Show Changes on PR | |
| run: | | |
| python -m cicd.compare.pr.show_changes $GITHUB_REPOSITORY $GITHUB_REF ${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/result/changes.json ${{ steps.read_to_var_dirs.outputs.OUTPUT_BASE }}/uploads.json | |
| env: | |
| PAT_GITHUB_API: ${{ secrets.PAT_GITHUB_API }} |