Merge pull request #57 from EURODEO/ci-cd-2

Add linters for Docker, Go, Python, and Shell

.github/workflows/ci.yml

@@ -7,7 +7,29 @@ on:
     tags:
       - '[0-9]+.[0-9]+.[0-9]+'
 
+permissions:
+  contents: read
+
 jobs:
+  pre-commit-hook:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+    - name: Install pre-commit
+      run: python -m pip install pre-commit
+      shell: bash
+    - name: Show pre-commit requirements
+      run: python -m pip freeze --local
+      shell: bash
+    - uses: actions/cache@v3
+      with:
+        path: ~/.cache/pre-commit
+        key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+    - name: Run pre-commit for all files
+      run: pre-commit run --config './.pre-commit-config.yaml' --all-files --color=always --show-diff-on-failure
+      shell: bash
+
   test:
     runs-on: ubuntu-latest
     steps:

.pre-commit-config.yaml

@@ -0,0 +1,73 @@
+repos:
+  - repo: local
+    hooks:
+      # go-fmt ~ Enforces Go standard formatting (whitespace, indentation, et cetera)
+      - id: go-fmt
+        name: go-fmt
+        description: "Enforces Go standard formatting (whitespace, indentation, et cetera)."
+        entry: ./ci/go/go-fmt.sh
+        language: script
+        pass_filenames: false
+#      # go-vet ~ Finds subtle issues in Go where your code may not work as intended
+#      - id: go-vet
+#        name: go-vet
+#        description: "Finds subtle issues in Go where your code may not work as intended."
+#        entry: ./ci/go/go-vet.sh
+#        language: script
+#        pass_filenames: false
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      # Formatting
+      - id: end-of-file-fixer # Makes sure files end in a newline and only a newline.
+      - id: pretty-format-json
+        args: ["--autofix", "--indent=4", "--no-ensure-ascii", "--no-sort-keys"] # Formats and sorts your JSON files.
+      - id: trailing-whitespace # Trims trailing whitespace.
+      # Checks
+      - id: check-json # Attempts to load all json files to verify syntax.
+      - id: check-merge-conflict # Check for files that contain merge conflict strings.
+      - id: check-shebang-scripts-are-executable # Checks that scripts with shebangs are executable.
+      - id: check-yaml
+        # only checks syntax not load the yaml:
+        # https://stackoverflow.com/questions/59413979/how-exclude-ref-tag-from-check-yaml-git-hook
+        args: ["--unsafe"] # Parse the yaml files for syntax.
+
+  # reorder-python-imports ~ sort python imports
+  - repo: https://github.com/asottile/reorder_python_imports
+    rev: v3.12.0
+    hooks:
+      - id: reorder-python-imports
+
+  # black ~ Formats Python code
+  - repo: https://github.com/psf/black
+    rev: 23.10.0
+    hooks:
+      - id: black
+        args: ["--line-length=120"]
+
+  # flake8 ~ Enforces the Python PEP8 style guide
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+      - id: flake8
+        args:
+          [
+            "--ignore=W503",
+            "--max-line-length=120",
+          ]
+
+  # hadolint ~ Docker linter
+  - repo: https://github.com/hadolint/hadolint
+    rev: v2.12.0
+    hooks:
+      - id: hadolint-docker
+        args: [
+            "--ignore=DL3008", # Pin versions in apt get install.
+          ]
+
+  # ShellCheck ~ Gives warnings and suggestions for bash/sh shell scripts
+  - repo: https://github.com/koalaman/shellcheck-precommit
+    rev: v0.9.0
+    hooks:
+      - id: shellcheck

README.md

@@ -1,2 +1,15 @@
 # e-soh-datastore-poc
 E-SOH datastore PoCs
+
+
+## Pre-commit
+### Setup
+1. Go to the root of the repository.
+2. Install the python pre-commit package with `pip install pre-commit`.
+3. Reinitialize there repository with `git init`.
+4. Install the hooks defined in `.pre-commit-config.yaml` with `pre-commit install`.
+
+### Useful Commands
+- To update the pre-commit hooks in `.pre-commit-config.yaml` run: `pre-commit autoupdate`.
+- To run the pre-commit for every file in the repository run `pre-commit run --config './.pre-commit-config.yaml' --all-files`.
+- To commit without the pre-commit hook `git commit -m "Some message" --no-verify`

api/Dockerfile

@@ -6,7 +6,6 @@ ENV PROJECT_DATASTORE_PATH="datastore"
 ENV PROJECT_PYTHON_PATH="api"
 ENV DOCKER_PATH="/app"
 
-# hadolint ignore=DL3013
 RUN apt-get update \
     && apt-get -y upgrade \
     && apt-get install -y --no-install-recommends git \
@@ -19,6 +18,7 @@ RUN apt-get update \
 COPY "${PROJECT_DATASTORE_PATH}/protobuf/datastore.proto" "/protobuf/datastore.proto"
 COPY "${PROJECT_PYTHON_PATH}/requirements.txt" "${DOCKER_PATH}/requirements.txt"
 
+# hadolint ignore=DL3013
 RUN pip install --no-cache-dir --upgrade pip \
     && pip install --no-cache-dir --upgrade -r "${DOCKER_PATH}/requirements.txt"
 

api/locustfile.py

@@ -1,11 +1,18 @@
 import random
 
-from locust import HttpUser, between, task
+from locust import HttpUser
+from locust import task
 
 
 parameters = ["ff", "dd", "rh", "pp", "tn"]
-stations = ["06203", "06204", "06205", "06207", "06208", "06211", "06214", "06215", "06235", "06239", "06242", "06251", "06260", "06269", "06270", "06275", "06279", "06280", "06290", "06310", "06317", "06319", "06323", "06330", "06340", "06344", "06348", "06350", "06356", "06370", "06375", "06380", "78871", "78873"]
-
+# fmt: off
+stations = [
+    "06203", "06204", "06205", "06207", "06208", "06211", "06214", "06215", "06235", "06239",
+    "06242", "06251", "06260", "06269", "06270", "06275", "06279", "06280", "06290", "06310",
+    "06317", "06319", "06323", "06330", "06340", "06344", "06348", "06350", "06356", "06370",
+    "06375", "06380", "78871", "78873",
+]
+# fmt: on
 headers = {"Accept-Encoding": "br"}
 
 
@@ -14,8 +21,17 @@ class WebsiteUser(HttpUser):
     def get_data_single_station_single_parameter(self):
         parameter = random.choice(parameters)
         station_id = random.choice(stations)
-        self.client.get(f"/collections/observations/locations/{station_id}?parameter-name={parameter}", name=f"single station {parameter}", headers=headers)
+        self.client.get(
+            f"/collections/observations/locations/{station_id}?parameter-name={parameter}",
+            name=f"single station {parameter}",
+            headers=headers,
+        )
 
     @task
     def get_data_bbox_three_parameters(self):
-        self.client.get(f"/collections/observations/area?parameter-name=dd,ff,rh&coords=POLYGON((5.0 52.0,6.0 52.0,6.0 52.1,5.0 52.1,5.0 52.0))", name=f"bbox", headers=headers)
+        self.client.get(
+            "/collections/observations/area?parameter-name=dd,ff,rh&"
+            "coords=POLYGON((5.0 52.0,6.0 52.0,6.0 52.1,5.0 52.1,5.0 52.0))",
+            name="bbox",
+            headers=headers,
+        )