Switch to using pyproject for dependencies

.github/workflows/run-tests.yml

@@ -42,7 +42,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.11]
+        python-version: [3.11, 3.12]
 
     steps:
       - name: Eco CI Energy Estimation - Initialize
@@ -97,7 +97,7 @@ jobs:
         # if: steps.cache-pipenv.outputs.cache-hit != 'true'
         run: |
           uv venv
-          uv pip install -r requirements/requirements.linux.generated.txt
+          uv sync
       - name: Eco CI Energy Estimation - Get Measurement
         uses: green-coding-solutions/[email protected]
         with:

.gitpod.yml

@@ -6,11 +6,12 @@ tasks:
   # set up python dependencies and database
   - name: main terminal
     init: |
+      curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /home/gitpod/.local/bin/
       cp ./.env.gitpod ./.env
       python -m venv .venv
       source .venv/bin/activate
       python -m pip install --upgrade pip uv
-      uv pip install -r requirements/requirements.linux.generated.txt
+      uv sync
       dotenv run -- ./manage.py migrate
       dotenv run -- ./manage.py tailwind install
       dotenv run -- ./manage.py tailwind build

ansible/_add_system_dependencies.yml

@@ -6,6 +6,19 @@
   become: true
   tags: [python]
 
+- name: Install python libraries
+  ansible.builtin.apt:
+    pkg:
+      - python3.12
+      - python3.12-venv
+      - python3.12-dev
+      - python-is-python3
+    state: present
+    update_cache: true
+  become: true
+  when: install_system_reqs is true
+  tags: [python]
+
 - name: Install mariadb libraries for python
   ansible.builtin.apt:
     pkg:
@@ -19,42 +32,65 @@
     update_cache: true
   become: true
   when: install_system_reqs is true
-  tags: [wip]
+  tags: [mariadb]
+
+- name: Check if FNM is already installed
+  ansible.builtin.stat:
+    path: "/home/deploy/.local/share/fnm/fnm"
+  register: fnm_installed
+  when: install_node is true
+  tags: [fnm-install]
+
+# - name:
+#   ansible.builtin.debug:
+#     var: fnm_installed
+#   when: install_node is true
+#   tags: [fnm-install]
 
-- name: Download node source script
+- name: Download FNM install (Fast Node Manager) script
   ansible.builtin.get_url:
-    url: "https://deb.nodesource.com/setup_18.x"
-    dest: "/tmp/setup_18.x.sh"
+    url: "https://fnm.vercel.app/install"
+    dest: "/tmp/fnm.install.sh"
     mode: "0755"
-  when: install_node is true
+  when: install_node is true and fnm_installed.stat.exists is false
+  tags: [fnm-install]
 
-- name: Run node installer as root
-  ansible.builtin.command: "bash /tmp/setup_18.x.sh"
-  when: install_node is true
-  become: true
+- name: Run FNM install script
+  ansible.builtin.command: "bash /tmp/fnm.install.sh"
+  when: install_node is true and fnm_installed.stat.exists is false
+  tags: [fnm-install]
 
+  # FNM relies on a set of environment variables to be set in the shell.
+  # So we call fnm
 - name: Install node and npm
-  ansible.builtin.apt:
-    pkg:
-      - nodejs
-    state: present
-    update_cache: true
-  become: true
-  when: install_node is true
+  ansible.builtin.shell: |
+    set -o pipefail
+    eval "$(fnm env)"
+    fnm use --install-if-missing 22
+  args:
+    executable: "/usr/bin/bash"
+  # set the PATH to include the fnm binary
+  environment:
+    PATH: "/home/deploy/.local/share/fnm/:{{  ansible_env.PATH }}"
+  when: install_node is true and fnm_installed.stat.exists is true
+  tags: [fnm-install]
 
-- name: Install python libraries
-  ansible.builtin.apt:
-    pkg:
-      - python3.11
-      - python3.11-venv
-      - python3.11-dev
-      - python-is-python3
-    state: present
-    update_cache: true
-  become: true
-  when: install_system_reqs is true
-  tags: [python]
+- name: Download uv install script
+  ansible.builtin.get_url:
+    url: "https://astral.sh/uv/install.sh"
+    dest: "/tmp/uv.install.sh"
+    mode: "0755"
+  when: install_uv is true
+  tags: [uv-install]
+
+- name: Check if uv is already installed
+  ansible.builtin.stat:
+    path: "/home/deploy/.cargo/bin/uv"
+  register: uv_installed
+  when: install_uv is true
+  tags: [uv-install]
 
-- name: Install libraries for handling dependencies
-  ansible.builtin.command: "python3.11 -m pip install uv wheel"
+- name: Install uv for managing python dependencies
+  ansible.builtin.shell: "bash /tmp/uv.install.sh"
   when: install_system_reqs is true
+  tags: [uv-install]

ansible/_create_deployment_directories.yml

@@ -29,7 +29,7 @@
     group: deploy
   become: true
 
-- name: Move .aws.credentials file to the deploy directory for using with aws cli
+- name: Move .aws.credentials file to shared directory
   ansible.builtin.template:
     src: "templates/dotaws.credentials.j2"
     dest: "{{ project_root }}/shared/.aws.credentials"

ansible/_install_deploy_dependencies.yml

@@ -1,33 +1,32 @@
 ---
 - name: Set up a virtual environment for this project
-  ansible.builtin.command: "python3.11 -m venv .venv"
+  ansible.builtin.command: /home/deploy/.cargo/bin/uv venv
   args:
     chdir: "{{ project_root }}/current"
   changed_when: false
+  tags: [uv]
 
 - name: Install python dependencies with uv
-  ansible.builtin.shell: |
-    source .venv/bin/activate
-    python3.11 -m pip install uv 
-    uv pip install -r requirements/requirements.linux.generated.txt
+  ansible.builtin.shell: /home/deploy/.cargo/bin/uv sync
   args:
     chdir: "{{ project_root }}/current"
     executable: "/usr/bin/bash"
+  tags: [uv]
 
 - name: Update node deps for building tailwind
   ansible.builtin.shell: |
-    source .venv/bin/activate
-    dotenv run -- python3.11 ./manage.py tailwind update
+    /home/deploy/.cargo/bin/uv run dotenv run -- ./manage.py tailwind update
   args:
     chdir: "{{ project_root }}/current"
     executable: "/usr/bin/bash"
   when: update_front_end_deps is true
+  tags: [uv]
 
 - name: Install node deps for building tailwind
   ansible.builtin.shell: |
-    source .venv/bin/activate
-    dotenv run -- python3.11 ./manage.py tailwind install
+    /home/deploy/.cargo/bin/uv run dotenv run -- ./manage.py tailwind install
   args:
     executable: "/usr/bin/bash"
     chdir: "{{ project_root }}/current"
   changed_when: false
+  tags: [uv]

ansible/deploy.yml

@@ -54,6 +54,7 @@
 
     - name: Install dependencies for python and js
       ansible.builtin.include_tasks: "_install_deploy_dependencies.yml"
+      tags: [uv, uv-install]
 
     - name: Assemble assets and related files
       ansible.builtin.include_tasks: "_assemble_deploy_assets.yml"

ansible/provision_app.yml

@@ -19,20 +19,66 @@
     install_system_reqs: true
     # decide about installing a recent version of node
     install_node: true
+    install_uv: true
     # run an update of our front dependencies as part of the deploy. This is better
     # handled in source control, but if the front end will not build in deployment, this
     # is a workaround
     update_front_end_deps: true
 
+    project_git_repo: "[email protected]:thegreenwebfoundation/admin-portal.git"
+    project_deploy_strategy: git
+    project_version: "{{ project_deploy_branch }}"
+    project_local_path: "../"
+    project_unwanted_items:
+      - .git
+    project_finalize: true
+
+    # set these to decide about setting up a folder structure as if we are deploying
+    deploy_project: true
+    update_symlinks: true
+
   tasks:
     - name: Set up most recent timestamps directories
       ansible.builtin.include_tasks: "_create_deployment_directories.yml"
       when: create_deployment_dirs is true
 
+    - name: Run the deploy steps to check out the project repo
+      ansible.builtin.include_role:
+        name: f500.project_deploy
+      when: deploy_project is defined and deploy_project is true
+
+    - name: Link .env to shared directory
+      ansible.builtin.file:
+        src: "{{ project_root }}/shared/.env"
+        dest: "{{ project_root }}/current/.env"
+        state: link
+      when: update_symlinks is true
+
+    - name: Link ./media to shared ./media directory
+      ansible.builtin.file:
+        src: "{{ project_root }}/shared/media"
+        dest: "{{ project_root }}/current/media"
+        state: link
+      when: update_symlinks is true
+
+    - name: Link ./data to shared ./data directory
+      ansible.builtin.file:
+        src: "{{ project_root }}/shared/data"
+        dest: "{{ project_root }}/current/data"
+        state: link
+      when: update_symlinks is true
+
+    - name: Link to .aws.credentials in shared directory
+      ansible.builtin.file:
+        src: "{{ project_root }}/shared/.aws.credentials"
+        dest: "{{ project_root }}/current/.aws.credentials"
+        state: link
+      when: update_symlinks is true
+
     - name: Install all system dependencies
       ansible.builtin.include_tasks: "_add_system_dependencies.yml"
       when: install_system_reqs is true
-      tags: [python]
+      tags: [python, fnm-install, uv-install]
 
     - name: Install all deploy dependencies
       ansible.builtin.include_tasks: "_install_deploy_dependencies.yml"

apps/accounts/tests/test_provider_request.py

@@ -190,13 +190,6 @@ def wizard_form_preview():
     }
 
 
-@pytest.fixture()
-def mock_open(mocker):
-    file_mock = mocker.patch("builtins.open")
-    file_mock.return_value = io.StringIO("file contents")
-    return file_mock
-
-
 @pytest.fixture()
 def fake_evidence():
     """
@@ -214,7 +207,7 @@ def fake_evidence():
         {
             "title": "this one has both: link and a file",
             "link": "www.example.com",
-            "file": File(file="cert.pdf"),
+            "file": File(file=io.BytesIO(faker.text().encode()), name=faker.file_name()),
             "type": models.EvidenceType.CERTIFICATE,
         },
         {
@@ -224,9 +217,10 @@ def fake_evidence():
     ],
     ids=["both_file_and_link", "neither_file_nor_link"],
 )
-def test_evidence_validation_fails(evidence_data, mock_open):
-    evidence_data["request"] = ProviderRequestFactory.create()
+def test_evidence_validation_fails(evidence_data):
 
+    evidence_data["request"] = ProviderRequestFactory.create()
+
     evidence = models.ProviderRequestEvidence.objects.create(**evidence_data)
 
     with pytest.raises(ValidationError):

apps/greencheck/api/views.py

@@ -1,8 +1,7 @@
 import logging
 
 from django.conf import settings
-from django.contrib.gis.geoip2 import GeoIP2
-from django.contrib.gis.geoip2.base import GeoIP2Exception
+from django.contrib.gis.geoip2 import GeoIP2, GeoIP2Exception
 from drf_yasg.utils import swagger_auto_schema
 from geoip2 import errors
 from rest_framework import permissions, views

justfile

@@ -0,0 +1,78 @@
+# set dotenv-load := true
+
+default:
+    just --list
+
+# used for tagging our docker images.
+TAG := `echo $$APP_RELEASE-$(git log -n 1 --format=%h)`
+
+## Deploy a new release to production. This will not work in Gitpod, as it relies on staff SSH keys for deployment. 
+release:
+    uv run dotenv -f env.prod run -- sentry-cli releases new -p admin-portal `sentry-cli releases propose-version`
+    uv run dotenv -f env.prod run -- sentry-cli releases set-commits --auto `sentry-cli releases propose-version`
+    uv run dotenv -f env.prod run -- ansible-playbook ansible/deploy.yml -i ansible/inventories/prod.yml
+    uv run dotenv -f env.prod run -- sentry-cli releases finalize `sentry-cli releases propose-version`
+
+# Create a super user for local development using the basic django `createsuperuser` command.
+dev_createsuperuser:
+    uv run dotenv run -- python ./manage.py createsuperuser --username admin --email [email protected] --noinput
+    uv run dotenv run -- python ./manage.py set_fake_passwords
+
+# Run a django development server that reloads when codes is changed.
+dev_runserver:
+    uv run dotenv run -- python manage.py runserver
+
+# Start the tailwind watcher - this will re-run tailwind to generate css as code is changed.
+dev_tailwind_start:
+    uv run dotenv run -- python manage.py tailwind start
+
+# Install the front end dependencies.
+dev_tailwind_install:
+    uv run dotenv run -- python manage.py tailwind install
+
+# Run the django tests on a loop with with pytest, and re-running them when code is changed.
+dev_test:
+    uv run uv dotenv run -- pytest -s --create-db --looponfail --ds=greenweb.settings.testing
+
+# Run the django tests on a loop with pytest, but only ones marked with `only`.
+dev_test_only:
+    uv run dotenv run -- pytest -s --create-db --looponfail -m only -v --ds=greenweb.settings.testing
+
+# # Set up the github repo for data analysis against the Green Web Platform database.
+data_analysis_repo:
+    #!/usr/bin/env bash
+    if test -d data-analysis; then
+        echo "data-analysis repo already checked out"
+    else
+        git clone https://github.com/thegreenwebfoundation/data-analysis.git
+    fi
+
+# Start a Marimo notebook session.
+notebook_gitpod: data_analysis_repo
+	# set up our start notebook with django initialised ready for queries
+	uv run dotenv run -- marimo edit data-analysis/starter-notebook.py
+
+# Run the django tests (with pytest), creating a test database using the `testing` settings.
+test *options:
+    uv run dotenv run -- pytest -s --create-db --ds=greenweb.settings.testing {{ options }}
+
+# As above, but only the tests marked 'only'.
+test_only:
+    uv run dotenv run -- pytest -s --create-db -m only -v --ds=greenweb.settings.testing
+
+# Build the documentation using Sphinx.
+docs:
+    uv run dotenv run -- sphinx-build ./docs _build/
+
+# Build the documentation using Sphinx and keep updating it on every change.
+docs_watch:
+    uv run dotenv run -- sphinx-autobuild ./docs _build/
+
+# Make a docker image for publishing to our registry.
+docker_build:
+    docker build -t $(APP_NAME)
+
+# Push the current tagged image to our registry.
+docker_release:
+    docker tag $(APP_NAME) $(DOCKER_REGISTRY):$(TAG)
+    docker push $(DOCKER_REGISTRY)/$(APP_NAME):$(TAG)