diff --git a/.github/workflows/dependabot.yaml b/.github/workflows/dependabot.yaml new file mode 100644 index 0000000..cf6fc94 --- /dev/null +++ b/.github/workflows/dependabot.yaml @@ -0,0 +1,14 @@ +name: Opdrachten Team Dependabot + +on: + schedule: # Run the script every day at 6am UTC + - cron: "0 6 * * *" + workflow_dispatch: + +jobs: + dependabot: + name: Templates + uses: amsterdam/github-workflows/.github/workflows/dependabot.yml@v1 + secrets: inherit + with: + check_diff: true diff --git a/Dockerfile b/Dockerfile index 232257a..5e9d27d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,24 +1,22 @@ FROM python:3.11-bookworm as base -ENV PYTHONUNBUFFERED=1 \ - PIP_NO_CACHE_DIR=off \ - REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +ENV TZ=Europe/Amsterdam +ENV PYTHONUNBUFFERED=1 +ENV PIP_NO_CACHE_DIR=off WORKDIR /api -COPY ca/* /usr/local/share/ca-certificates/extras/ - RUN apt-get update \ && apt-get dist-upgrade -y \ && apt-get autoremove -y \ - && apt-get install --no-install-recommends -y \ + && apt-get install -y --no-install-recommends \ nano \ + openssh-server \ locales \ - && rm -rf /var/lib/apt/lists/* /var/cache/debconf/*-old \ && pip install --upgrade pip \ - && pip install uwsgi \ - && chmod -R 644 /usr/local/share/ca-certificates/extras/ \ - && update-ca-certificates + && pip install uwsgi + +COPY requirements.txt /api RUN sed -i -e 's/# nl_NL.UTF-8 UTF-8/nl_NL.UTF-8 UTF-8/' /etc/locale.gen && \ locale-gen @@ -26,16 +24,47 @@ ENV LANG nl_NL.UTF-8 ENV LANGUAGE nl_NL:nl ENV LC_ALL nl_NL.UTF-8 -COPY requirements.txt /api - RUN pip install -r requirements.txt COPY ./scripts /api/scripts COPY ./app /api/app -COPY ./uwsgi.ini /api -COPY ./test.sh /api -COPY .flake8 /api + + +FROM base as tests + +COPY conf/test.sh /api/ +COPY .flake8 /api/ RUN chmod u+x /api/test.sh -CMD uwsgi --uid www-data --gid www-data --ini /api/uwsgi.ini +ENTRYPOINT [ "/bin/sh", "/api/test.sh"] + +FROM base as publish + +# ssh ( see also: https://github.com/Azure-Samples/docker-django-webapp-linux ) +ENV SSH_PASSWD "root:Docker!" + +EXPOSE 8000 +ENV PORT 8000 + +ARG MA_OTAP_ENV +ENV MA_OTAP_ENV=$MA_OTAP_ENV + +ARG MA_BUILD_ID +ENV MA_BUILD_ID=$MA_BUILD_ID + +ARG MA_GIT_SHA +ENV MA_GIT_SHA=$MA_GIT_SHA + +COPY conf/uwsgi.ini /api/ +COPY conf/docker-entrypoint.sh /api/ +COPY conf/sshd_config /etc/ssh/ + +RUN chmod u+x /api/docker-entrypoint.sh \ + && echo "$SSH_PASSWD" | chpasswd + +ENTRYPOINT [ "/bin/sh", "/api/docker-entrypoint.sh"] + +FROM publish as publish-final + +COPY /files /app/files \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 02b1ee7..dfc4203 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,6 +11,7 @@ def retagAndPush(String imageName, String currentTag, String newTag) String BRANCH = "${env.BRANCH_NAME}" String IMAGE_NAME = "mijnams/focus" String IMAGE_TAG = "${IMAGE_NAME}:${env.BUILD_NUMBER}" +String IMAGE_TEST = "${IMAGE_NAME}:test-${env.BUILD_NUMBER}" String CMDB_ID = "app_mijn-focus" node { @@ -18,37 +19,34 @@ node { checkout scm } - stage("Build image") { - docker.withRegistry(DOCKER_REGISTRY_HOST, "docker_registry_auth") { - def image = docker.build(IMAGE_TAG) - image.push() - } - } -} - -// Skipping tests for the test branch -if (BRANCH != "test-acc") { - node { + // Skipping tests for the test branch + if (BRANCH != "test-acc") { stage("Test") { docker.withRegistry(DOCKER_REGISTRY_HOST, "docker_registry_auth") { - docker.image(IMAGE_TAG).pull() - sh "docker run --rm ${IMAGE_TAG} /api/test.sh" + sh "docker build -t ${IMAGE_TEST} " + + "--target=tests " + + "--shm-size 1G " + + "." + sh "docker run --rm ${IMAGE_TEST}" } } } -} -if (BRANCH == "test-acc" || BRANCH == "main") { - node { + stage("Build image") { + docker.withRegistry(DOCKER_REGISTRY_HOST, "docker_registry_auth") { + def image = docker.build(IMAGE_TAG, "--target=publish .") + image.push() + } + } + + if (BRANCH == "test-acc" || BRANCH == "main") { stage("Push acceptance image") { docker.withRegistry(DOCKER_REGISTRY_HOST, "docker_registry_auth") { docker.image(IMAGE_TAG).pull() retagAndPush(IMAGE_NAME, env.BUILD_NUMBER, "acceptance") } } - } - node { stage("Deploy to ACC") { build job: "Subtask_Openstack_Playbook", parameters: [ @@ -58,23 +56,19 @@ if (BRANCH == "test-acc" || BRANCH == "main") { ] } } -} -if (BRANCH == "production-release") { - stage("Waiting for approval") { - input "Deploy to Production?" - } + if (BRANCH == "production-release") { + stage("Waiting for approval") { + input "Deploy to Production?" + } - node { stage("Push production image") { docker.withRegistry(DOCKER_REGISTRY_HOST, "docker_registry_auth") { docker.image(IMAGE_TAG).pull() retagAndPush(IMAGE_NAME, env.BUILD_NUMBER, "production") } } - } - node { stage("Deploy") { build job: "Subtask_Openstack_Playbook", parameters: [ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fb46e11 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +# This Makefile is based on the Makefile defined in the Python Best Practices repository: +# https://git.data.amsterdam.nl/Datapunt/python-best-practices/blob/master/dependency_management/ + +PYTHON = python3 + +pip-tools: + pip install pip-tools + +requirements: pip-tools ## Upgrade requirements (in requirements-root.txt) to latest versions and compile requirements.txt + pip-compile --upgrade --output-file requirements.txt requirements-root.txt + +diff: + @python3 ./scripts/diff.py diff --git a/app/auth.py b/app/auth.py index 84605ff..40ccf99 100644 --- a/app/auth.py +++ b/app/auth.py @@ -1,8 +1,12 @@ +import logging import os import unittest from unittest.mock import patch -from flask_httpauth import HTTPTokenAuth + import jwt +from flask_httpauth import HTTPTokenAuth + +from app.config import VERIFY_JWT_SIGNATURE auth = HTTPTokenAuth(scheme="Bearer") @@ -118,7 +122,12 @@ def get_verified_token_data(token): def get_user_profile_from_token(token): - token_data = get_verified_token_data(token) + if VERIFY_JWT_SIGNATURE: + token_data = get_verified_token_data(token) + else: + token_data = jwt.api_jwt.decode(token, options={"verify_signature": False}) + + logging.info(token_data) profile_type = get_profile_type(token_data) profile_id = get_profile_id(token_data) diff --git a/app/config.py b/app/config.py index fee957a..a1719a2 100644 --- a/app/config.py +++ b/app/config.py @@ -1,7 +1,9 @@ +import base64 import locale import logging import os from datetime import date, time, datetime +import tempfile from flask.json.provider import DefaultJSONProvider locale.setlocale(locale.LC_TIME, "nl_NL.UTF-8") @@ -15,27 +17,44 @@ # Environment determination IS_PRODUCTION = SENTRY_ENV == "production" IS_ACCEPTANCE = SENTRY_ENV == "acceptance" -IS_AP = IS_PRODUCTION or IS_ACCEPTANCE -IS_DEV = os.getenv("FLASK_ENV") == "development" and not IS_AP +IS_DEV = SENTRY_ENV == "development" +IS_TEST = SENTRY_ENV == "test" + +IS_TAP = IS_PRODUCTION or IS_ACCEPTANCE or IS_TEST +IS_AP = IS_ACCEPTANCE or IS_PRODUCTION +IS_OT = IS_DEV or IS_TEST + +# App constants +VERIFY_JWT_SIGNATURE = os.getenv("VERIFY_JWT_SIGNATURE", IS_AP) +API_REQUEST_TIMEOUT = 30 +API_BASE_PATH = "/wpi" # Server security / certificates for ZorgNed SERVER_CLIENT_CERT = os.getenv("MIJN_DATA_CLIENT_CERT") SERVER_CLIENT_KEY = os.getenv("MIJN_DATA_CLIENT_KEY") +# TODO: Add other AZ env conditions after migration. +if IS_TEST: + # https://stackoverflow.com/a/46570364/756075 + # Server security / certificates + cert = tempfile.NamedTemporaryFile(delete=False) + cert.write(base64.b64decode(SERVER_CLIENT_CERT)) + cert.close() + + key = tempfile.NamedTemporaryFile(delete=False) + key.write(base64.b64decode(SERVER_CLIENT_KEY)) + key.close() + + SERVER_CLIENT_CERT = cert.name + SERVER_CLIENT_KEY = key.name + # ZorgNed vars ZORGNED_STADSPASSEN_ENABLED = True ZORGNED_API_REQUEST_TIMEOUT_SECONDS = 30 -ZORGNED_API_TOKEN = os.getenv("WMO_NED_API_TOKEN") +ZORGNED_API_TOKEN = os.getenv("ZORGNED_API_TOKEN") ZORGNED_API_URL = os.getenv("ZORGNED_API_URL") ZORGNED_GEMEENTE_CODE = "0363" - -# App constants -ENABLE_OPENAPI_VALIDATION = os.getenv("ENABLE_OPENAPI_VALIDATION", not IS_AP) - -API_REQUEST_TIMEOUT = 30 -API_BASE_PATH = "/wpi" - # Set-up logging LOG_LEVEL = os.getenv("LOG_LEVEL", "ERROR").upper() logging.basicConfig( diff --git a/app/focus_config.py b/app/focus_config.py index 5d7a7dc..98949c8 100644 --- a/app/focus_config.py +++ b/app/focus_config.py @@ -1,7 +1,6 @@ -import json import os -from app.config import API_BASE_PATH, IS_ACCEPTANCE +from app.config import API_BASE_PATH # Focus FOCUS_WSDL = os.getenv("FOCUS_WSDL") @@ -59,12 +58,6 @@ 3557: "kind", } -FOCUS_STADSPAS_ADMIN_NUMBER_CONVERSION_ACC = ( - json.loads(os.getenv("FOCUS_STADSPAS_ADMIN_NUMBER_CONVERSION_ACC", "null")) - if IS_ACCEPTANCE - else None -) - zeep_config = {"wsdl": FOCUS_WSDL, "session_verify": FOCUS_CERTIFICATE} focus_credentials = { diff --git a/app/focus_service_stadspas_admin.py b/app/focus_service_stadspas_admin.py index b9b434d..96f3eca 100644 --- a/app/focus_service_stadspas_admin.py +++ b/app/focus_service_stadspas_admin.py @@ -19,15 +19,6 @@ def has_groene_stip(fondsen): return False -def get_administratienummer(number_from_source): - # Disable ACC dataset coupling, enable if we need it again. - # if FOCUS_STADSPAS_ADMIN_NUMBER_CONVERSION_ACC: - # return FOCUS_STADSPAS_ADMIN_NUMBER_CONVERSION_ACC.get( - # number_from_source, number_from_source - # ) - return number_from_source - - def get_first_pas_type(fondsen): pas_type = None @@ -83,8 +74,6 @@ def get_stadspas_admin_number(bsn): pas_type = get_first_pas_type(fondsen) return { - "admin_number": get_administratienummer( - volledig_administratienummer(admin_number) - ), + "admin_number": volledig_administratienummer(admin_number), "type": pas_type, } diff --git a/app/gpass_config.py b/app/gpass_config.py index 71f4cb7..3d3bf09 100644 --- a/app/gpass_config.py +++ b/app/gpass_config.py @@ -3,7 +3,7 @@ # GPASS GPASS_API_TOKEN = os.getenv("GPASS_TOKEN") GPASS_API_LOCATION = os.getenv("GPASS_API_LOCATION") -GPASS_FERNET_ENCRYPTION_KEY = os.getenv("FERNET_KEY") +GPASS_FERNET_ENCRYPTION_KEY = os.getenv("FERNET_ENCRYPTION_KEY") STADSPAS_TRANSACTIONS_PATH = "/wpi/stadspas/transacties/" diff --git a/app/server.py b/app/server.py index 0eb8502..edfb980 100644 --- a/app/server.py +++ b/app/server.py @@ -17,13 +17,12 @@ decrypt, error_response_json, success_response_json, - validate_openapi, ) from app import auth from app.config import ( API_BASE_PATH, - IS_DEV, + IS_OT, SENTRY_DSN, ZORGNED_STADSPASSEN_ENABLED, UpdatedJSONProvider, @@ -42,6 +41,7 @@ ) +@application.route("/", methods=["GET"]) @application.route("/status/health", methods=["GET"]) def status_health(): return success_response_json("OK") @@ -49,7 +49,6 @@ def status_health(): @application.route(f"{API_BASE_PATH}/uitkering-en-stadspas/aanvragen", methods=["GET"]) @auth.login_required -@validate_openapi def aanvragen(): user = auth.get_current_user() aanvragen = get_aanvragen(user["id"]) @@ -58,7 +57,6 @@ def aanvragen(): @application.route(f"{API_BASE_PATH}/e-aanvragen", methods=["GET"]) @auth.login_required -@validate_openapi def e_aanvragen(): user = auth.get_current_user() aanvragen = get_e_aanvragen(user["id"]) @@ -67,7 +65,6 @@ def e_aanvragen(): @application.route(f"/{FOCUS_DOCUMENT_PATH}", methods=["GET"]) @auth.login_required -@validate_openapi def document(): user = auth.get_current_user() id = request.args.get("id", None) @@ -92,7 +89,6 @@ def document(): f"{API_BASE_PATH}/uitkering/specificaties-en-jaaropgaven", methods=["GET"] ) @auth.login_required -@validate_openapi def specificaties_en_jaaropgaven(): user = auth.get_current_user() jaaropgaven = get_jaaropgaven(user["id"]) @@ -106,7 +102,6 @@ def specificaties_en_jaaropgaven(): @application.route(f"{API_BASE_PATH}/stadspas", methods=["GET"]) @auth.login_required -@validate_openapi def stadspassen(): user = auth.get_current_user() stadspassen = [] @@ -147,7 +142,6 @@ def stadspassen(): methods=["GET"], ) @auth.login_required -@validate_openapi def stadspastransactions(encrypted_admin_pasnummer): budget_code, admin_number, stadspas_number = decrypt(encrypted_admin_pasnummer) stadspas_transations = get_transactions(admin_number, stadspas_number, budget_code) @@ -164,7 +158,7 @@ def handle_error(error): logging.exception(error, extra={"error_message_original": error_message_original}) - if IS_DEV: # pragma: no cover + if IS_OT: # pragma: no cover msg_auth_exception = error_message_original msg_request_http_error = error_message_original msg_server_error = error_message_original diff --git a/app/test_focus_service_specificaties.py b/app/test_focus_service_specificaties.py index e04d408..52e85c0 100644 --- a/app/test_focus_service_specificaties.py +++ b/app/test_focus_service_specificaties.py @@ -88,11 +88,7 @@ def test_get_jaaropgaven_error( get_client_mock.return_value = mock_client bsn = "12312312399" - jaaropgaven = get_jaaropgaven(bsn) - - print(jaaropgaven) - - print("\n\n\n\n\n\n\n\n\n") + get_jaaropgaven(bsn) mock_client.service.getJaaropgaven.assert_called_with(bsn) handle_soap_service_error_mock.assert_called() diff --git a/app/test_focus_service_stadspas_admin.py b/app/test_focus_service_stadspas_admin.py index c1a3870..3d35f77 100644 --- a/app/test_focus_service_stadspas_admin.py +++ b/app/test_focus_service_stadspas_admin.py @@ -47,15 +47,6 @@ def test_get_stadspas_admin_number_none(self, get_client_mock): self.assertEqual(result, None) - # @patch( - # "app.focus_service_stadspas_admin.FOCUS_STADSPAS_ADMIN_NUMBER_CONVERSION_ACC", - # {"xx": "yy"}, - # ) - # def test_admin_number_conversion(self): - # self.assertEqual(get_administratienummer("xx"), "yy") - # self.assertEqual(get_administratienummer("yy"), "yy") - # self.assertEqual(get_administratienummer("zz"), "zz") - example_response = { "administratienummer": 123123123, diff --git a/app/test_server.py b/app/test_server.py index 805ca31..277d5b4 100644 --- a/app/test_server.py +++ b/app/test_server.py @@ -8,7 +8,6 @@ from app.utils import encrypt -@patch("app.utils.ENABLE_OPENAPI_VALIDATION", True) class WPITestServer(WpiApiTestApp): def test_status_health(self): response = self.client.get("/status/health") @@ -109,7 +108,12 @@ def create_specificatie(title, dcteId, date_published): @patch("app.server.get_clientnummer") @patch("app.server.get_stadspas_admin_number") @patch("app.server.get_stadspassen") - def test_stadspassen(self, get_stadspassen_mock, get_stadspas_admin_number_mock, get_clientnummer_mock): + def test_stadspassen( + self, + get_stadspassen_mock, + get_stadspas_admin_number_mock, + get_clientnummer_mock, + ): get_stadspassen_mock.return_value = [GpassServiceGetStadspas.gpass_formatted] get_stadspas_admin_number_mock.return_value = { "admin_number": "abcdefg123", @@ -130,7 +134,12 @@ def test_stadspassen(self, get_stadspassen_mock, get_stadspas_admin_number_mock, @patch("app.server.get_clientnummer") @patch("app.server.get_stadspas_admin_number") @patch("app.server.get_stadspassen") - def test_stadspassen_with_zorgned_result(self, get_stadspassen_mock, get_stadspas_admin_number_mock, get_clientnummer_mock): + def test_stadspassen_with_zorgned_result( + self, + get_stadspassen_mock, + get_stadspas_admin_number_mock, + get_clientnummer_mock, + ): get_stadspassen_mock.return_value = [GpassServiceGetStadspas.gpass_formatted] get_stadspas_admin_number_mock.return_value = { "admin_number": "abcdefg123", diff --git a/app/utils.py b/app/utils.py index 4ee8602..e248ab0 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,59 +1,13 @@ import logging -import os from datetime import date, datetime -from functools import wraps from re import sub -import yaml from cryptography.fernet import Fernet -from flask import request from flask.helpers import make_response -from openapi_core import create_spec -from openapi_core.contrib.flask import FlaskOpenAPIRequest, FlaskOpenAPIResponse -from openapi_core.validation.request.validators import RequestValidator -from openapi_core.validation.response.validators import ResponseValidator -from yaml import load -from app.config import BASE_PATH, CONNECTION_ERRORS, ENABLE_OPENAPI_VALIDATION +from app.config import CONNECTION_ERRORS from app.gpass_config import GPASS_FERNET_ENCRYPTION_KEY -openapi_spec = None - - -def get_openapi_spec(): - global openapi_spec - if not openapi_spec: - with open(os.path.join(BASE_PATH, "openapi.yml"), "r") as spec_file: - spec_dict = load(spec_file, Loader=yaml.Loader) - - openapi_spec = create_spec(spec_dict) - - return openapi_spec - - -def validate_openapi(function): - @wraps(function) - def validate(*args, **kwargs): - - if ENABLE_OPENAPI_VALIDATION: - spec = get_openapi_spec() - openapi_request = FlaskOpenAPIRequest(request) - validator = RequestValidator(spec) - result = validator.validate(openapi_request) - result.raise_for_errors() - - response = function(*args, **kwargs) - - if ENABLE_OPENAPI_VALIDATION: - openapi_response = FlaskOpenAPIResponse(response) - validator = ResponseValidator(spec) - result = validator.validate(openapi_request, openapi_response) - result.raise_for_errors() - - return response - - return validate - def success_response_json(response_content): return make_response({"status": "OK", "content": response_content}, 200) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml new file mode 100644 index 0000000..569a397 --- /dev/null +++ b/azure-pipelines.yaml @@ -0,0 +1,64 @@ +trigger: + batch: true + branches: + include: + - ontwikkelen + - testen +pr: + autoCancel: true + branches: + include: + - main + +resources: + repositories: + - repository: MamsInfra + type: git + name: MijnAmsterdam/mijn-amsterdam-infra + ref: refs/heads/pipeline-fe + +parameters: + - name: btdBuild + type: boolean + default: true + - name: btdTest + type: boolean + default: true + - name: btdDeploy + type: boolean + default: true + - name: dtapName + type: string + default: none + values: + - none + - o + - t + # - a + # - p + + - name: updateAppSettings + type: boolean + default: false + +variables: + - ${{ if eq(variables['Build.SourceBranchName'], 'ontwikkelen') }}: + - name: dtapName + value: o + - ${{ if or(eq(variables['Build.SourceBranchName'], 'testen'), eq(variables['Build.Reason'], 'PullRequest')) }}: + - name: dtapName + value: t + - ${{ if ne(parameters.dtapName, 'none') }}: + - name: dtapName + value: ${{ parameters.dtapName }} + +jobs: + - template: pipelines/jobs/apps/btd-koppel-api.yaml@MamsInfra + parameters: + appServiceShortName: wpi + dtapName: ${{ variables.dtapName }} + btdBuild: ${{ parameters.btdBuild }} + btdTest: ${{ parameters.btdTest }} + btdDeploy: ${{ parameters.btdDeploy }} + updateAppSettings: ${{ parameters.updateAppSettings }} + aquaScan: ${{ eq(variables['Build.Reason'], 'PullRequest') }} diff --git a/conf/docker-entrypoint.sh b/conf/docker-entrypoint.sh new file mode 100644 index 0000000..549dd97 --- /dev/null +++ b/conf/docker-entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +# AZ AppService allows SSH into a App instance. +if [ $MA_OTAP_ENV == "test" ] +then + # echo "Starting SSH ..." +service ssh start +fi + +uwsgi --uid www-data --gid www-data --ini /api/uwsgi.ini \ No newline at end of file diff --git a/conf/sshd_config b/conf/sshd_config new file mode 100644 index 0000000..8838f56 --- /dev/null +++ b/conf/sshd_config @@ -0,0 +1,15 @@ +# +# /etc/ssh/sshd_config +# + +Port 2222 +ListenAddress 0.0.0.0 +LoginGraceTime 180 +X11Forwarding yes +Ciphers aes128-cbc,3des-cbc,aes256-cbc,aes128-ctr,aes192-ctr,aes256-ctr +MACs hmac-sha1,hmac-sha1-96 +StrictModes yes +SyslogFacility DAEMON +PasswordAuthentication yes +PermitEmptyPasswords no +PermitRootLogin yes \ No newline at end of file diff --git a/test.sh b/conf/test.sh similarity index 100% rename from test.sh rename to conf/test.sh diff --git a/uwsgi.ini b/conf/uwsgi.ini similarity index 100% rename from uwsgi.ini rename to conf/uwsgi.ini diff --git a/requirements-root.txt b/requirements-root.txt index 3f908d1..f42f6e9 100644 --- a/requirements-root.txt +++ b/requirements-root.txt @@ -5,9 +5,6 @@ dpath flake8 flask flask_httpauth -openapi-core==0.14.2 -openapi-schema-validator==0.1.5 -openapi-spec-validator==0.3.0 pyjwt requests sentry-sdk[flask] diff --git a/requirements.txt b/requirements.txt index 7b6b9f3..3260212 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,48 +1,106 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements.txt requirements-root.txt +# attrs==23.1.0 -black==23.3.0 -blinker==1.6.2 + # via zeep +black==23.10.1 + # via -r requirements-root.txt +blinker==1.6.3 + # via + # flask + # sentry-sdk certifi==2023.7.22 -cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 -coverage==7.2.5 -cryptography==41.0.4 -dictpath==0.1.3 -dpath==2.1.5 -flake8==6.0.0 -Flask==2.3.2 -Flask-HTTPAuth==4.8.0 + # via + # requests + # sentry-sdk +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.1 + # via requests +click==8.1.7 + # via + # black + # flask +coverage==7.3.2 + # via -r requirements-root.txt +cryptography==41.0.5 + # via -r requirements-root.txt +dpath==2.1.6 + # via -r requirements-root.txt +flake8==6.1.0 + # via -r requirements-root.txt +flask==3.0.0 + # via + # -r requirements-root.txt + # flask-httpauth + # sentry-sdk +flask-httpauth==4.8.0 + # via -r requirements-root.txt idna==3.4 + # via requests isodate==0.6.1 + # via zeep itsdangerous==2.1.2 -Jinja2==3.1.2 -jsonschema==4.17.3 -lazy-object-proxy==1.9.0 -lxml==4.9.2 -MarkupSafe==2.1.2 + # via flask +jinja2==3.1.2 + # via flask +lxml==4.9.3 + # via zeep +markupsafe==2.1.3 + # via + # jinja2 + # sentry-sdk + # werkzeug mccabe==0.7.0 -more-itertools==9.1.0 + # via flake8 mypy-extensions==1.0.0 -openapi-core==0.14.2 -openapi-schema-validator==0.1.5 -openapi-spec-validator==0.3.0 -packaging==23.1 -parse==1.19.0 -pathspec==0.11.1 -platformdirs==3.5.0 -pycodestyle==2.10.0 + # via black +packaging==23.2 + # via black +pathspec==0.11.2 + # via black +platformdirs==3.11.0 + # via + # black + # zeep +pycodestyle==2.11.1 + # via flake8 pycparser==2.21 -pyflakes==3.0.1 -PyJWT==2.6.0 -pyrsistent==0.19.3 -pytz==2023.3 -PyYAML==6.0 + # via cffi +pyflakes==3.1.0 + # via flake8 +pyjwt==2.8.0 + # via -r requirements-root.txt +pytz==2023.3.post1 + # via zeep requests==2.31.0 + # via + # -r requirements-root.txt + # requests-file + # requests-toolbelt + # zeep requests-file==1.5.1 + # via zeep requests-toolbelt==1.0.0 -sentry-sdk==1.22.2 + # via zeep +sentry-sdk[flask]==1.32.0 + # via -r requirements-root.txt six==1.16.0 + # via + # isodate + # requests-file tomli==2.0.1 -urllib3==1.26.17 -Werkzeug==2.3.4 + # via black +typing-extensions==4.8.0 + # via black +urllib3==2.0.7 + # via + # requests + # sentry-sdk +werkzeug==3.0.1 + # via flask zeep==4.2.1 + # via -r requirements-root.txt diff --git a/scripts/diff.py b/scripts/diff.py new file mode 100644 index 0000000..b0e4b37 --- /dev/null +++ b/scripts/diff.py @@ -0,0 +1,125 @@ +# standard library +import json +import urllib.request +from collections import namedtuple +from contextlib import suppress +from distutils.version import StrictVersion +from pathlib import Path +from subprocess import check_output +from typing import Iterable, List, NamedTuple + +import pkg_resources + + +def git_diff(cwd) -> Iterable[str]: + """ + Run git diff in the specified directory. + """ + return ( + check_output(["git", "diff", "requirements.txt"], cwd=cwd).decode().splitlines() + ) + + +class PackageChange(NamedTuple): + package: str + from_version: str + to_version: str + + +def normalise_package_name(package_name): + """ + Ensure we always use the same package names when parsing git diff + sometimes a package is referred to using - and sometimes with . (I + think depending on pip version). + """ + data = json.load( + urllib.request.urlopen(f"https://pypi.org/pypi/{package_name}/json") + ) + return data["info"]["name"] + + +def parse_diff(diff: Iterable[str]) -> List[PackageChange]: + """ + Parse a git diff to find changes to dependencies. + + :param diff: The output from git diff + + :return: Iterable of package changes. + """ + + def get_change_type(line): + for char in "-+": + # filter single line changes + if line.startswith(char) and not line.startswith(char * 3): + return char + + # because we are parsing from a pip-tools generated file the specifier + # should always be of the form ==version, so we strip the first two + # characters + # TODO: assert this assumption + LineChange = namedtuple("LineChange", "change_type package version") + changes = { + LineChange( + change_type, + normalise_package_name(requirement.key), + str(requirement.specifier)[2:], + ) + for line in diff + if (change_type := get_change_type(line)) is not None + if (requirement := next(pkg_resources.parse_requirements(line[1:]), None)) + is not None + } + + def version(package, change_type): + changes_for_this_package_of_type = ( + c.version + for c in changes + if c.package == package + if c.change_type == change_type + ) + return next(changes_for_this_package_of_type, None) + + return [ + PackageChange(package, from_version, to_version) + for package in {c.package for c in changes} + if (from_version := version(package, "-")) + != (to_version := version(package, "+")) + ] + + +def print_package_version_update_message(project_package_changes: List): + """ + Create threads on slack which show the major dependency upgrades + and the projects that are affected. + """ + messages = [] + + for package_change in project_package_changes: + if ( + package_change.to_version is not None + and package_change.from_version is not None + ): + # if one of the versions is not strict, so we can't say anything about if this is a patch + # release or not so we just show the message to be sure + with suppress(ValueError): + strict_from_version = StrictVersion(package_change.from_version) + strict_to_version = StrictVersion(package_change.to_version) + show_message = ( + strict_from_version.version[0] != strict_to_version.version[0] + ) + + if package_change.from_version < package_change.to_version: + message = f"{package_change.from_version} ➩ {package_change.to_version}" + else: + message = f"{package_change.from_version} ➩ {package_change.to_version}" + + if show_message: + messages.append(f"{package_change.package} | {message}") + + print("\n".join(messages)) + + +if __name__ == "__main__": + path = Path(__file__).parent.parent + diff = parse_diff(git_diff(path)) + print_package_version_update_message(diff) diff --git a/scripts/test_zorgned_service.py b/scripts/test_zorgned_service.py index 1dc929d..93e9ce4 100644 --- a/scripts/test_zorgned_service.py +++ b/scripts/test_zorgned_service.py @@ -1,5 +1,6 @@ import sys -from app.zorgned_service import get_clientnummer +from app.gpass_service import get_stadspassen +from app.zorgned_service import get_clientnummer, volledig_clientnummer bsn = None if len(sys.argv) >= 2: @@ -7,4 +8,7 @@ client_nummer = get_clientnummer(bsn) +stadspassen = get_stadspassen(volledig_clientnummer(client_nummer), "M") + print(client_nummer) +print(stadspassen)