Merge pull request #13 from alphagov/add-build-image

Add build image and update dynamic harvest source
alphagov · Dec 5, 2023 · eb73212 · eb73212
2 parents 3fc9eb1 + b685cd7
commit eb73212
Show file tree

Hide file tree

Showing 7 changed files with 137 additions and 13 deletions.
diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml
@@ -0,0 +1,60 @@
+name: Build and push images
+
+on:
+  workflow_dispatch:
+    inputs:
+      buildType:
+        description: Decide on what to build
+        required: true
+        type: choice
+        options:
+          - build_push
+          - build_only
+  push:
+    branches:
+      - main
+
+jobs:
+  build_and_push:
+    name: ${{ matrix.app.name }} (${{ matrix.arch }})
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Login to GHCR
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a  # v2.1.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.gitRef }}
+      - name: Build dynamic
+        if: ${{ inputs.buildType == 'build_only' }}
+        uses: docker/build-push-action@v5
+        with:
+          context: ./dynamic
+          push: false
+          tags: alphagov/dynamic-ckan-harvest-source:1.0.0
+      - name: Build static
+        if: ${{ inputs.buildType == 'build_only' }}
+        uses: docker/build-push-action@v5
+        with:
+          context: ./static
+          push: false
+          tags: alphagov/static-ckan-harvest-source:1.0.0
+      - name: Build and push dynamic
+        if: ${{ inputs.buildType == 'build_push' || github.ref == 'refs/heads/main' }}
+        uses: docker/build-push-action@v5
+        with:
+          context: ./dynamic
+          push: true
+          tags: alphagov/dynamic-ckan-harvest-source:1.0.0
+      - name: Build and push static
+        if: ${{ inputs.buildType == 'build_push' || github.ref == 'refs/heads/main' }}
+        uses: docker/build-push-action@v5
+        with:
+          context: ./static
+          push: true
+          tags: alphagov/static-ckan-harvest-source:1.0.0
diff --git a/dynamic/Dockerfile b/dynamic/Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:3.7
+FROM alpine:3.16
 
 ENV FLASK_APP=main.py
 

diff --git a/dynamic/main.py b/dynamic/main.py
@@ -1,11 +1,10 @@
-import os
 import time
 from flask import request, Flask, Response
 
 app = Flask(__name__)
 DELAY_TAG = 'delay-'
 NUM_LINKS_TAG = 'num_links-'
-NUM_DIGITS=3
+NUM_DIGITS = 3
 
 
 @app.route('/', defaults={'path': '0'})
@@ -34,20 +33,21 @@ def catch_all(path):
     path_split = path.split('/')
     url_index = path_split[0] if '/' in path else path if len(path) else '0'
 
+    delay = 1
+
+    if len(path_split) > 2 and path_split[1].startswith(DELAY_TAG):
+        delay = int(path_split[1][len(DELAY_TAG):])
+
     if path.endswith('.tmp.xml'):
         filename_base = path_split[-1]
         filename_parts = filename_base.split('.')
-        delay = 1
-
-        if len(path_split) > 2 and path_split[2].startswith(DELAY_TAG):
-            delay = int(path_split[2][len(DELAY_TAG):])
 
         filename = '.'.join([filename_parts[-4], filename_parts[-2], filename_parts[-1]])
         _id = filename_parts[-3].zfill(NUM_DIGITS)
 
         with open(filename, 'rb') as f:
             content = f.read().decode('utf-8')
-            content = content.replace('{{url-index}}', url_index.zfill(NUM_DIGITS))
+            content = content.replace('{{url-index}}', url_index.zfill(2))
             content = content.replace('{{end-guid}}', _id)
 
             # delay to simulate slow servers
@@ -62,19 +62,22 @@ def catch_all(path):
             else:
                 return resp
     else:
-        delay = 1
         num_links = 10
         if path_split[1].startswith(NUM_LINKS_TAG):
             num_links = int(path_split[1][len(NUM_LINKS_TAG):])
 
-        num_links = num_links if num_links < 1000 else 999
+        num_links = num_links if num_links < 100 else 99
         links = ''
         for index in range(1, 1 + num_links):
             links += f'<div><a href="test.{index}.tmp.xml">Harvest source {index}</a></div>'
         content = f'<html><body>{url_index}{links}</body></html>'
 
+        # delay to simulate slow servers
+        if delay > 0:
+            time.sleep(delay)
+
         return content
 
 
 if __name__ == "__main__":
-    app.run(debug='DEBUG' in os.environ, host='0.0.0.0', port=int(os.getenv("PORT", 8001)))
+    app.run(debug=True, port=8001, host="0.0.0.0")
diff --git a/dynamic/requirements.txt b/dynamic/requirements.txt
@@ -1 +1 @@
-Flask==1.1.1
+Flask==3.0.0
diff --git a/static/manifest.yml b/static/manifest.yml
@@ -6,4 +6,4 @@ applications:
   health-check-type: http
   health-check-http-endpoint: /status
   docker:
-    image: govuk/ckan-static-mock-harvest-source:0.0.4
+    image: govuk/ckan-static-mock-harvest-source:0.0.5
diff --git a/static/responses/dcat/dcat/data.json b/static/responses/dcat/dcat/data.json
@@ -0,0 +1,54 @@
+{
+    "@context": "https://project-open-data.cio.gov/v1.1/schema/catalog.jsonld",
+    "@id": "https://open.barnet.gov.uk/data.json?masthead=london-borough-of-barnet",
+    "@type": "dcat:Catalog",
+    "conformsTo": "https://project-open-data.cio.gov/v1.1/schema",
+    "describedBy": "https://project-open-data.cio.gov/v1.1/schema/catalog.json",
+    "dataset": [
+        {
+            "title": "Brownfield Land Register",
+            "theme": [
+                "Environment, Property & Land"
+            ],
+            "publisher": {
+                "@type": "org:Organization",
+                "name": "London Borough of Barnet"
+            },
+            "issued": "2017-12-19T09:51:17.000Z",
+            "modified": "2021-01-05T09:42:56.000Z",
+            "landingPage": "https://open.barnet.gov.uk/dataset/249xe/brownfield-land-register",
+            "license": "https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/",
+            "distribution": [
+                {
+                    "@type": "dcat:Distribution",
+                    "downloadURL": "https://open.barnet.gov.uk/download/249xe/2gv/Brownfield%20Land%20Register%202020%20-%20Barnet.csv",
+                    "title": "Brownfield Land Register - Barnet.csv",
+                    "description": ""
+                },
+                {
+                    "@type": "dcat:Distribution",
+                    "downloadURL": "https://open.barnet.gov.uk/download/249xe/dlr/Brownfield-Land-Register.xml",
+                    "title": "Brownfield Land Register metadata file",
+                    "description": ""
+                }
+            ],
+            "description": "This is a dataset of the London Borough of Barnet's Brownfield Land Register, as referenced in the Housing and Planning Act 2016.  Part 1 lists brownfield land sites of at least 0.25ha or capable of providing 5 or more dwellings that are considered suitable for housing led development. Sites listed in Part 2 of the register will have been granted Permission in Principle. Local Planning Authorities are required to review registers at least once a year.\r\nFurther information from the DCLG on Brownfield Land Registers is provided here\r\nThis dataset has been published by the London Borough of Barnet under the Open Government Licence (OGL) (v3).\r\nPlease acknowledge the Information Provider through the following attribution statement:\r\n© London Borough of Barnet, 2017, OGL, v3.0\r\nContains OS data © Crown copyright and database right, 2017",
+            "accessLevel": "public",
+            "@type": "dcat:Dataset",
+            "contactPoint": {
+                "@type": "vcard:Contact",
+                "fn": "<Nobody>"
+            },
+            "identifier": "249xe",
+            "keyword": [
+                "Brownfield",
+                "Development",
+                "Housing",
+                "Land",
+                "Permission in Prin",
+                "Register",
+                "Sites"
+            ]
+        }
+    ]
+}
diff --git a/static/sites/mock-ckan.conf b/static/sites/mock-ckan.conf
@@ -21,6 +21,13 @@ server {
         try_files $1 =404;
     }
 
+    location /dcat {
+        root $mock_ckan_responses_root/dcat;
+        types { }
+        default_type "application/json";
+        try_files $apiless_uri$is_args$args @html;
+    }
+
     location /api {
         root $mock_ckan_responses_root/json;
         types { }