From d60224ed13ff963e20515435fc30e01729611038 Mon Sep 17 00:00:00 2001 From: "kentsang.gds" Date: Wed, 17 Nov 2021 11:52:07 +0000 Subject: [PATCH 1/7] Bump docker image version to 0.0.5 --- static/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/manifest.yml b/static/manifest.yml index 3681ec3..3b3bd99 100644 --- a/static/manifest.yml +++ b/static/manifest.yml @@ -6,4 +6,4 @@ applications: health-check-type: http health-check-http-endpoint: /status docker: - image: govuk/ckan-static-mock-harvest-source:0.0.4 + image: govuk/ckan-static-mock-harvest-source:0.0.5 From 4972286520ffbbd3f7fd347fb47f98f4199ce889 Mon Sep 17 00:00:00 2001 From: "kentsang.gds" Date: Thu, 30 Mar 2023 16:44:12 +0100 Subject: [PATCH 2/7] Update dynamic harvest source processing --- dynamic/main.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/dynamic/main.py b/dynamic/main.py index 1da6b85..3e05e88 100644 --- a/dynamic/main.py +++ b/dynamic/main.py @@ -1,11 +1,10 @@ -import os import time from flask import request, Flask, Response app = Flask(__name__) DELAY_TAG = 'delay-' NUM_LINKS_TAG = 'num_links-' -NUM_DIGITS=3 +NUM_DIGITS = 3 @app.route('/', defaults={'path': '0'}) @@ -34,20 +33,21 @@ def catch_all(path): path_split = path.split('/') url_index = path_split[0] if '/' in path else path if len(path) else '0' + delay = 1 + + if len(path_split) > 2 and path_split[1].startswith(DELAY_TAG): + delay = int(path_split[1][len(DELAY_TAG):]) + if path.endswith('.tmp.xml'): filename_base = path_split[-1] filename_parts = filename_base.split('.') - delay = 1 - - if len(path_split) > 2 and path_split[2].startswith(DELAY_TAG): - delay = int(path_split[2][len(DELAY_TAG):]) filename = '.'.join([filename_parts[-4], filename_parts[-2], filename_parts[-1]]) _id = filename_parts[-3].zfill(NUM_DIGITS) with open(filename, 'rb') as f: content = f.read().decode('utf-8') - content = content.replace('{{url-index}}', url_index.zfill(NUM_DIGITS)) + content = content.replace('{{url-index}}', url_index.zfill(2)) content = content.replace('{{end-guid}}', _id) # delay to simulate slow servers @@ -62,19 +62,22 @@ def catch_all(path): else: return resp else: - delay = 1 num_links = 10 if path_split[1].startswith(NUM_LINKS_TAG): num_links = int(path_split[1][len(NUM_LINKS_TAG):]) - num_links = num_links if num_links < 1000 else 999 + num_links = num_links if num_links < 100 else 99 links = '' for index in range(1, 1 + num_links): links += f'
Harvest source {index}
' content = f'{url_index}{links}' + # delay to simulate slow servers + if delay > 0: + time.sleep(delay) + return content if __name__ == "__main__": - app.run(debug='DEBUG' in os.environ, host='0.0.0.0', port=int(os.getenv("PORT", 8001))) + app.run(debug=True, port=8001) From 07be1bf2e21e8794a7fa71168a63bd4450c832ea Mon Sep 17 00:00:00 2001 From: kentsanggds Date: Tue, 5 Dec 2023 15:41:22 +0000 Subject: [PATCH 3/7] Update dynamic harvest source --- dynamic/Dockerfile | 2 +- dynamic/main.py | 2 +- dynamic/requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dynamic/Dockerfile b/dynamic/Dockerfile index 8897fe2..460bf5d 100644 --- a/dynamic/Dockerfile +++ b/dynamic/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.7 +FROM alpine:3.16 ENV FLASK_APP=main.py diff --git a/dynamic/main.py b/dynamic/main.py index 3e05e88..49ca12d 100644 --- a/dynamic/main.py +++ b/dynamic/main.py @@ -80,4 +80,4 @@ def catch_all(path): if __name__ == "__main__": - app.run(debug=True, port=8001) + app.run(debug=True, port=8001, host="0.0.0.0") diff --git a/dynamic/requirements.txt b/dynamic/requirements.txt index 32e8968..047e950 100644 --- a/dynamic/requirements.txt +++ b/dynamic/requirements.txt @@ -1 +1 @@ -Flask==1.1.1 +Flask==3.0.0 From 85a19e7476b06ad3c215576a3605aed693fc308e Mon Sep 17 00:00:00 2001 From: kentsanggds Date: Tue, 5 Dec 2023 15:43:17 +0000 Subject: [PATCH 4/7] Add dcat to static mock harvest source --- static/responses/dcat/dcat/data.json | 54 ++++++++++++++++++++++++++++ static/sites/mock-ckan.conf | 7 ++++ 2 files changed, 61 insertions(+) create mode 100644 static/responses/dcat/dcat/data.json diff --git a/static/responses/dcat/dcat/data.json b/static/responses/dcat/dcat/data.json new file mode 100644 index 0000000..3c7e977 --- /dev/null +++ b/static/responses/dcat/dcat/data.json @@ -0,0 +1,54 @@ +{ + "@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld", + "@id": "https://open.barnet.gov.uk/data.json?masthead=london-borough-of-barnet", + "@type": "dcat:Catalog", + "conformsTo": "https://project-open-data.cio.gov/v1.1/schema", + "describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json", + "dataset": [ + { + "title": "Brownfield Land Register", + "theme": [ + "Environment, Property & Land" + ], + "publisher": { + "@type": "org:Organization", + "name": "London Borough of Barnet" + }, + "issued": "2017-12-19T09:51:17.000Z", + "modified": "2021-01-05T09:42:56.000Z", + "landingPage": "https://open.barnet.gov.uk/dataset/249xe/brownfield-land-register", + "license": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/", + "distribution": [ + { + "@type": "dcat:Distribution", + "downloadURL": "https://open.barnet.gov.uk/download/249xe/2gv/Brownfield%20Land%20Register%202020%20-%20Barnet.csv", + "title": "Brownfield Land Register - Barnet.csv", + "description": "" + }, + { + "@type": "dcat:Distribution", + "downloadURL": "https://open.barnet.gov.uk/download/249xe/dlr/Brownfield-Land-Register.xml", + "title": "Brownfield Land Register metadata file", + "description": "" + } + ], + "description": "This is a dataset of the London Borough of Barnet's Brownfield Land Register, as referenced in the Housing and Planning Act 2016. Part 1 lists brownfield land sites of at least 0.25ha or capable of providing 5 or more dwellings that are considered suitable for housing led development. Sites listed in Part 2 of the register will have been granted Permission in Principle. Local Planning Authorities are required to review registers at least once a year.\r\nFurther information from the DCLG on Brownfield Land Registers is provided here\r\nThis dataset has been published by the London Borough of Barnet under the Open Government Licence (OGL) (v3).\r\nPlease acknowledge the Information Provider through the following attribution statement:\r\n© London Borough of Barnet, 2017, OGL, v3.0\r\nContains OS data © Crown copyright and database right, 2017", + "accessLevel": "public", + "@type": "dcat:Dataset", + "contactPoint": { + "@type": "vcard:Contact", + "fn": "" + }, + "identifier": "249xe", + "keyword": [ + "Brownfield", + "Development", + "Housing", + "Land", + "Permission in Prin", + "Register", + "Sites" + ] + } + ] +} \ No newline at end of file diff --git a/static/sites/mock-ckan.conf b/static/sites/mock-ckan.conf index 723b29f..16162e8 100644 --- a/static/sites/mock-ckan.conf +++ b/static/sites/mock-ckan.conf @@ -21,6 +21,13 @@ server { try_files $1 =404; } + location /dcat { + root $mock_ckan_responses_root/dcat; + types { } + default_type "application/json"; + try_files $apiless_uri$is_args$args @html; + } + location /api { root $mock_ckan_responses_root/json; types { } From 16a929d9bca22ba63c75f990dbf6087046710608 Mon Sep 17 00:00:00 2001 From: kentsanggds Date: Tue, 5 Dec 2023 16:26:38 +0000 Subject: [PATCH 5/7] Add build image github actions --- .github/workflows/build-image.sh | 28 +++++++++++++++++++ .github/workflows/build-image.yaml | 45 ++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100755 .github/workflows/build-image.sh create mode 100644 .github/workflows/build-image.yaml diff --git a/.github/workflows/build-image.sh b/.github/workflows/build-image.sh new file mode 100755 index 0000000..9a67c4c --- /dev/null +++ b/.github/workflows/build-image.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eux + +build () { + if [ "${ARCH}" = "amd64" ]; then + docker build . -t "ghcr.io/alphagov/dynamic-${APP}:${1}" -f "dynamic/Dockerfile" + docker build . -t "ghcr.io/alphagov/static-${APP}:${1}" -f "static/Dockerfile" + else + docker buildx build --platform "linux/${ARCH}" . -t "ghcr.io/alphagov/dynamic-${APP}:${1}" -f "dynamic/Dockerfile" + docker buildx build --platform "linux/${ARCH}" . -t "ghcr.io/alphagov/static-${APP}:${1}" -f "static/Dockerfile" + fi +} + +DOCKER_TAG="${GITHUB_SHA}" + +if [[ -n ${GH_REF:-} ]]; then + DOCKER_TAG="${GH_REF}" +fi + +build "${DOCKER_TAG}" + +if [[ -n ${DRY_RUN:-} ]]; then + echo "Dry run; not pushing to registry" +else + docker push "ghcr.io/alphagov/dynamic-${APP}:${DOCKER_TAG}" + docker push "ghcr.io/alphagov/static-${APP}:${DOCKER_TAG}" +fi diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml new file mode 100644 index 0000000..b54f8d8 --- /dev/null +++ b/.github/workflows/build-image.yaml @@ -0,0 +1,45 @@ +name: Build and push images + +on: + workflow_dispatch: + inputs: + buildType: + description: Decide on what to build + required: true + type: choice + options: + - build_push + - build_only + push: + branches: + - main + +jobs: + build_and_push: + name: ${{ matrix.app.name }} (${{ matrix.arch }}) + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Login to GHCR + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.gitRef }} + - name: Build images (without pushing to registry) + if: ${{ inputs.buildType == 'build_only' }} + env: + DRY_RUN: "1" + APP: ckan-harvest-source + ARCH: amd64 + run: ./docker/build-image.sh + - name: Build and push images + if: ${{ inputs.buildType == 'build_push' || github.ref == 'refs/heads/main' }} + env: + APP: ckan-harvest-source + ARCH: amd64 + run: ./docker/build-image.sh From df48b75593e5b574ea7840814d125e53f9298df4 Mon Sep 17 00:00:00 2001 From: kentsanggds Date: Tue, 5 Dec 2023 17:37:10 +0000 Subject: [PATCH 6/7] Try build-push-action --- .github/workflows/build-image.yaml | 37 +++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml index b54f8d8..64f5916 100644 --- a/.github/workflows/build-image.yaml +++ b/.github/workflows/build-image.yaml @@ -30,16 +30,31 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ inputs.gitRef }} - - name: Build images (without pushing to registry) + - name: Build dynamic if: ${{ inputs.buildType == 'build_only' }} - env: - DRY_RUN: "1" - APP: ckan-harvest-source - ARCH: amd64 - run: ./docker/build-image.sh - - name: Build and push images + uses: docker/build-push-action@v5 + with: + context: ./dynamic + push: false + tags: alphagov/dynamic-ckan-harvest-source:1.0.0 + - name: Build static + if: ${{ inputs.buildType == 'build_only' }} + uses: docker/build-push-action@v5 + with: + context: ./static + push: false + tags: alphagov/static-ckan-harvest-source:1.0.0 + - name: Build and push dynamic if: ${{ inputs.buildType == 'build_push' || github.ref == 'refs/heads/main' }} - env: - APP: ckan-harvest-source - ARCH: amd64 - run: ./docker/build-image.sh + uses: docker/build-push-action@v5 + with: + context: ./dynamic + push: true + tags: alphagov/dynamic-ckan-harvest-source:1.0.0 + - name: Build and push static + if: ${{ inputs.buildType == 'build_push' || github.ref == 'refs/heads/main' }} + uses: docker/build-push-action@v5 + with: + context: ./static + push: true + tags: alphagov/static-ckan-harvest-source:1.0.0 From b685cd72288f4c6ef0873cda9c44097f3ea0923d Mon Sep 17 00:00:00 2001 From: kentsanggds Date: Tue, 5 Dec 2023 17:42:33 +0000 Subject: [PATCH 7/7] Delete build-image.sh --- .github/workflows/build-image.sh | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100755 .github/workflows/build-image.sh diff --git a/.github/workflows/build-image.sh b/.github/workflows/build-image.sh deleted file mode 100755 index 9a67c4c..0000000 --- a/.github/workflows/build-image.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -eux - -build () { - if [ "${ARCH}" = "amd64" ]; then - docker build . -t "ghcr.io/alphagov/dynamic-${APP}:${1}" -f "dynamic/Dockerfile" - docker build . -t "ghcr.io/alphagov/static-${APP}:${1}" -f "static/Dockerfile" - else - docker buildx build --platform "linux/${ARCH}" . -t "ghcr.io/alphagov/dynamic-${APP}:${1}" -f "dynamic/Dockerfile" - docker buildx build --platform "linux/${ARCH}" . -t "ghcr.io/alphagov/static-${APP}:${1}" -f "static/Dockerfile" - fi -} - -DOCKER_TAG="${GITHUB_SHA}" - -if [[ -n ${GH_REF:-} ]]; then - DOCKER_TAG="${GH_REF}" -fi - -build "${DOCKER_TAG}" - -if [[ -n ${DRY_RUN:-} ]]; then - echo "Dry run; not pushing to registry" -else - docker push "ghcr.io/alphagov/dynamic-${APP}:${DOCKER_TAG}" - docker push "ghcr.io/alphagov/static-${APP}:${DOCKER_TAG}" -fi