From 90148d973220d457b4cf1fb795dee72e7b8981e9 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Fri, 7 May 2021 19:57:27 +0200 Subject: [PATCH 01/13] v1.2.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 48ae7bac..97a8e462 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='openimis-be-core', - version='1.2.1rc6', + version='1.2.1', packages=find_packages(), include_package_data=True, license='GNU AGPL v3', From d02ac165728de6b79f2dc71bfd0c60cfd82c0185 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Mon, 10 May 2021 12:26:12 +0200 Subject: [PATCH 02/13] Missing __init__ in gql folder for setup --- core/gql/__init__.py | 0 setup.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 core/gql/__init__.py diff --git a/core/gql/__init__.py b/core/gql/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/setup.py b/setup.py index 97a8e462..443633d1 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='openimis-be-core', - version='1.2.1', + version='1.2.1.1', packages=find_packages(), include_package_data=True, license='GNU AGPL v3', From e2877a15e657f10456800429471c781b3a6ec8f8 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Fri, 29 Oct 2021 00:59:39 +0200 Subject: [PATCH 03/13] v1.3.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 443633d1..6134a045 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='openimis-be-core', - version='1.2.1.1', + version='1.3.0', packages=find_packages(), include_package_data=True, license='GNU AGPL v3', From 2dd3c4df1d4536fa916d3397bbb55291f06e4d08 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Mon, 15 Nov 2021 12:10:31 +0100 Subject: [PATCH 04/13] Cannot connect with non-admin user OMT-281 This is not exactly OMT-281 but being unable to connect with a claim administrator prevents the test at all. As pointed out in the comment, by default, the rest framework will look for core.user_view permission among the numeric permissions of openIMIS and obviously deny access. Explicitly setting the permission on that specific view allows any authenticated user to view their own data, as intended. Unauthenticated users will still get a 403 response. --- core/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/views.py b/core/views.py index 2e2db6aa..a765f8d1 100644 --- a/core/views.py +++ b/core/views.py @@ -1,6 +1,7 @@ from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.response import Response +from rest_framework.permissions import IsAuthenticated from .models import User from .serializers import UserSerializer @@ -8,6 +9,9 @@ class UserViewSet(viewsets.ModelViewSet): queryset = User.objects.all() serializer_class = UserSerializer + # If we don't specify the IsAuthenticated, the framework will look for the core.user_view permission and prevent + # any access from non-admin users + permission_classes = [IsAuthenticated] @action(detail=False) def current_user(self, request): From 23a9838810b41dd24b307d3a65cd75b36d36caf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20G=C3=A9r=C3=B4me?= Date: Wed, 3 Nov 2021 15:28:29 +0000 Subject: [PATCH 05/13] fix(Performance): Use legacy_id instead of validity_to to filter historical data MS SQL Server is super slow and does not use correctly the indexes when using a filter with a date --- core/utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/utils.py b/core/utils.py index 64890c88..10191491 100644 --- a/core/utils.py +++ b/core/utils.py @@ -53,13 +53,16 @@ def __ne__(self, other): return cls -def filter_validity(arg='validity', **kwargs): +def filter_validity(arg="validity", **kwargs): validity = kwargs.get(arg) if validity is None: - validity = core.datetime.datetime.now() + return ( + Q(validity_from__lte=core.datetime.datetime.now()), + Q(legacy_id__isnull=True), + ) return ( Q(validity_from__lte=validity), - Q(validity_to__isnull=True) | Q(validity_to__gte=validity) + Q(validity_to__isnull=True) | Q(validity_to__gte=validity), ) From ab98b7d83d3ebee0becac3498d21ff466650002a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20G=C3=A9r=C3=B4me?= Date: Thu, 4 Nov 2021 21:01:04 +0000 Subject: [PATCH 06/13] feat(Sentry): Better logging of errors in mutations --- core/schema.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/schema.py b/core/schema.py index 8905eab2..a55dbed7 100644 --- a/core/schema.py +++ b/core/schema.py @@ -155,7 +155,7 @@ def mutate_and_get_payload(cls, root, info, **data): client_mutation_details=json.dumps(data.get( "client_mutation_details"), cls=OpenIMISJSONEncoder) if data.get("client_mutation_details") else None ) - logging.debug("OpenIMISMutation: saved as %s, label: %s", mutation_log.id, mutation_log.client_mutation_label) + logger.debug("OpenIMISMutation: saved as %s, label: %s", mutation_log.id, mutation_log.client_mutation_label) if info and info.context and info.context.user and info.context.user.language: lang = info.context.user.language if isinstance(lang, Language): @@ -164,7 +164,7 @@ def mutate_and_get_payload(cls, root, info, **data): translation.activate(lang) try: - logging.debug("[OpenIMISMutation %s] Sending signals", mutation_log.id) + logger.debug("[OpenIMISMutation %s] Sending signals", mutation_log.id) results = signal_mutation.send( sender=cls, mutation_log_id=mutation_log.id, data=data, user=info.context.user, mutation_module=cls._mutation_module, mutation_class=cls._mutation_class) @@ -173,7 +173,7 @@ def mutate_and_get_payload(cls, root, info, **data): mutation_module=cls._mutation_module, mutation_class=cls._mutation_class )) errors = [err for r in results for err in r[1]] - logging.debug("[OpenIMISMutation %s] signals sent, got errors back: ", mutation_log.id, len(errors)) + logger.debug("[OpenIMISMutation %s] signals sent, got errors back: ", mutation_log.id, len(errors)) if errors: mutation_log.mark_as_failed(json.dumps(errors)) return cls(internal_id=mutation_log.id) @@ -182,37 +182,37 @@ def mutate_and_get_payload(cls, root, info, **data): sender=cls, mutation_log_id=mutation_log.id, data=data, user=info.context.user, mutation_module=cls._mutation_module, mutation_class=cls._mutation_class ) - logging.debug("[OpenIMISMutation %s] before mutate signal sent", mutation_log.id) + logger.debug("[OpenIMISMutation %s] before mutate signal sent", mutation_log.id) if core.async_mutations: - logging.debug("[OpenIMISMutation %s] Sending async mutation", mutation_log.id) + logger.debug("[OpenIMISMutation %s] Sending async mutation", mutation_log.id) openimis_mutation_async.delay( mutation_log.id, cls._mutation_module, cls._mutation_class) else: - logging.debug("[OpenIMISMutation %s] mutating...", mutation_log.id) + logger.debug("[OpenIMISMutation %s] mutating...", mutation_log.id) try: error_messages = cls.async_mutate( info.context.user if info.context and info.context.user else None, **data) if not error_messages: - logging.debug("[OpenIMISMutation %s] marked as successful", mutation_log.id) + logger.debug("[OpenIMISMutation %s] marked as successful", mutation_log.id) mutation_log.mark_as_successful() else: errors_json = json.dumps(error_messages) - logging.debug("[OpenIMISMutation %s] marked as failed: %s", mutation_log.id, errors_json) + logger.error("[OpenIMISMutation %s] marked as failed: %s", mutation_log.id, errors_json) mutation_log.mark_as_failed(errors_json) except BaseException as exc: error_messages = exc logger.error("async_mutate threw an exception. It should have gotten this far.", exc_info=exc) # Record the failure of the mutation but don't include details for security reasons mutation_log.mark_as_failed(f"The mutation threw a {type(exc)}, check logs for details") - logging.debug("[OpenIMISMutation %s] send post mutation signal", mutation_log.id) + logger.debug("[OpenIMISMutation %s] send post mutation signal", mutation_log.id) signal_mutation_module_after_mutating[cls._mutation_module].send( sender=cls, mutation_log_id=mutation_log.id, data=data, user=info.context.user, mutation_module=cls._mutation_module, mutation_class=cls._mutation_class, error_messages=error_messages ) except Exception as exc: - logger.warning(f"Exception while processing mutation id {mutation_log.id}", exc_info=True) + logger.error(f"Exception while processing mutation id {mutation_log.id}", exc_info=exc) mutation_log.mark_as_failed(exc) return cls(internal_id=mutation_log.id) From 8ff81a8884c42c314a2b7bc2f1dcd9f52b62ff24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20G=C3=A9r=C3=B4me?= Date: Fri, 5 Nov 2021 10:45:10 +0000 Subject: [PATCH 07/13] hotfix(User): Keep user roles history --- core/services.py | 8 ++++---- core/test_helpers.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/services.py b/core/services.py index 4d35c82f..3de30eb3 100644 --- a/core/services.py +++ b/core/services.py @@ -43,7 +43,7 @@ def create_or_update_interactive_user(user_id, data, audit_user_id, connected): created = True i_user.save() - create_or_update_user_roles(i_user, data["roles"]) + create_or_update_user_roles(i_user, data["roles"], audit_user_id) if "districts" in data: create_or_update_user_districts( i_user, data["districts"], data_subset["audit_user_id"] @@ -51,7 +51,7 @@ def create_or_update_interactive_user(user_id, data, audit_user_id, connected): return i_user, created -def create_or_update_user_roles(i_user, role_ids): +def create_or_update_user_roles(i_user, role_ids, audit_user_id): from core import datetime now = datetime.datetime.now() @@ -59,8 +59,8 @@ def create_or_update_user_roles(i_user, role_ids): validity_to=now ) for role_id in role_ids: - UserRole.objects.update_or_create( - user=i_user, role_id=role_id, defaults={"validity_to": None} + UserRole.objects.create( + user=i_user, role_id=role_id, audit_user_id=audit_user_id ) diff --git a/core/test_helpers.py b/core/test_helpers.py index 285ce6a3..40063075 100644 --- a/core/test_helpers.py +++ b/core/test_helpers.py @@ -32,7 +32,7 @@ def create_test_interactive_user(username, password="Test1234", roles=None, cust ) i_user.set_password(password) i_user.save() - create_or_update_user_roles(i_user, roles) + create_or_update_user_roles(i_user, roles, None) return User.objects.create( username=username, i_user=i_user, From e5386605f3faaf3c117c665e858b51bc8e8c040e Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Wed, 17 Nov 2021 17:24:19 +0100 Subject: [PATCH 08/13] Facilitate access to health facility The interactive user has a health facility without foreign key but the claim admin has a foreign key. This method encapsulates this and is needed for OMT-281 so that price lists are available. --- core/models.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/core/models.py b/core/models.py index 30841f11..0c8da78b 100644 --- a/core/models.py +++ b/core/models.py @@ -6,6 +6,7 @@ from copy import copy import core +from django.apps import apps from django.db import models from django.db.models import Q, DO_NOTHING, F from django.conf import settings @@ -369,6 +370,13 @@ def rights(self): def rights_str(self): return [str(r) for r in self.rights] + @cached_property + def get_health_facility(self): + if self.health_facility_id: + hf_model = apps.get_model("location", "HealthFacility") + if hf_model: + return hf_model.objects.filter(pk=self.health_facility_id).first() + def set_password(self, raw_password): from hashlib import sha256 from secrets import token_hex @@ -495,6 +503,13 @@ def get_session_auth_hash(self): key_salt = "core.User.get_session_auth_hash" return salted_hmac(key_salt, self.username).hexdigest() + def get_health_facility(self): + if self.claim_admin: + return self.claim_admin.health_facility + if self.i_user: + return self.i_user.get_health_facility() + return None + def __getattr__(self, name): if name == '_u': raise ValueError('wrapper has not been initialised') From a9ed5e7b16d7d3415ec43bb45817c7f3839013a3 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Thu, 18 Nov 2021 23:56:42 +0100 Subject: [PATCH 09/13] v1.3.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6134a045..a2c6d8a9 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='openimis-be-core', - version='1.3.0', + version='1.3.1', packages=find_packages(), include_package_data=True, license='GNU AGPL v3', From f42170a3499cc5d97ef523415f3b68868b14083f Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Wed, 22 Dec 2021 15:04:05 +0100 Subject: [PATCH 10/13] OMT-281 follow-up when HFID is 0 The cached_property would fail when called with HFID==0 (as happens in tblUsers that has no foreign key on this one) --- core/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/models.py b/core/models.py index 0c8da78b..e34bf3bb 100644 --- a/core/models.py +++ b/core/models.py @@ -371,11 +371,12 @@ def rights_str(self): return [str(r) for r in self.rights] @cached_property - def get_health_facility(self): + def health_facility(self): if self.health_facility_id: hf_model = apps.get_model("location", "HealthFacility") if hf_model: return hf_model.objects.filter(pk=self.health_facility_id).first() + return None def set_password(self, raw_password): from hashlib import sha256 @@ -507,7 +508,7 @@ def get_health_facility(self): if self.claim_admin: return self.claim_admin.health_facility if self.i_user: - return self.i_user.get_health_facility() + return self.i_user.health_facility return None def __getattr__(self, name): From 90624c88a7dd5839013345d1928101e86a0a6936 Mon Sep 17 00:00:00 2001 From: Eric Darchis Date: Wed, 22 Dec 2021 15:07:40 +0100 Subject: [PATCH 11/13] v.1.3.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a2c6d8a9..7a84d646 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='openimis-be-core', - version='1.3.1', + version='1.3.3', packages=find_packages(), include_package_data=True, license='GNU AGPL v3', From 061e2a00cdf12b9ccd230af172135cedfbe789fb Mon Sep 17 00:00:00 2001 From: maxime ngoe Date: Mon, 24 Jul 2023 14:22:45 +0100 Subject: [PATCH 12/13] Add Migration on JSONExt column already fixed on this PR : https://github.com/openimis/openimis-be-core_py/pull/186 Need to be merged as an hotfix to solve the policy issue on main release --- ...0023_alter_jsonext_column_in_tblOfficer.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 core/migrations/0023_alter_jsonext_column_in_tblOfficer.py diff --git a/core/migrations/0023_alter_jsonext_column_in_tblOfficer.py b/core/migrations/0023_alter_jsonext_column_in_tblOfficer.py new file mode 100644 index 00000000..e26c02af --- /dev/null +++ b/core/migrations/0023_alter_jsonext_column_in_tblOfficer.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.19 on 2023-06-01 11:45 +from django.conf import settings +from django.db import migrations + + +class Migration(migrations.Migration): + + psql_code = 'select 1' + mssql_code = 'ALTER TABLE [tblOfficer] ALTER COLUMN [JsonExt]' + + dependencies = [ + ('core', '0021_set_managed_to_true'), + ] + + operations = [ + migrations.RunSQL(f'{mssql_code} NVARCHAR(MAX)' + if settings.MSSQL else psql_code, + reverse_sql=f'{mssql_code} TEXT' + if settings.MSSQL else psql_code), + ] \ No newline at end of file From 2cedf94e8d11ccf5d3d34a5f9863a2dc82e56d00 Mon Sep 17 00:00:00 2001 From: Damian Borowiecki Date: Tue, 19 Sep 2023 10:47:49 +0200 Subject: [PATCH 13/13] CI update 2 --- .github/workflows/ci.yml | 30 +++++++ .github/workflows/openmis-module-test.yml | 99 ----------------------- 2 files changed, 30 insertions(+), 99 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/openmis-module-test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..a201065f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,30 @@ +name: Module CI +on: + pull_request: + types: [opened, synchronize, reopened] + push: + branches: + - main + - 'release/**' + - develop + - 'feature/**' + workflow_dispatch: + inputs: + comment: + description: Just a simple comment to know the purpose of the manual build + required: false + +jobs: + call: + name: Default CI Flow + uses: openimis/openimis-be_py/.github/workflows/ci_module.yml@develop + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + with: + SONAR_PROJECT_KEY: openimis_openimis-be-core_py + SONAR_ORGANIZATION: openimis-1 + SONAR_PROJECT_NAME: openimis-be-core_py + SONAR_PROJECT_VERSION: 1.0 + SONAR_SOURCES: core + SONAR_EXCLUSIONS: "**/migrations/**,**/static/**,**/media/**,**/tests/**" + diff --git a/.github/workflows/openmis-module-test.yml b/.github/workflows/openmis-module-test.yml deleted file mode 100644 index 81f387dd..00000000 --- a/.github/workflows/openmis-module-test.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: Automated CI testing -# This workflow run automatically for every commit on github it checks the syntax and launch the tests. -# | grep . | uniq -c filters out empty lines and then groups consecutive lines together with the number of occurrences -on: - pull_request: - workflow_dispatch: - inputs: - comment: - description: Just a simple comment to know the purpose of the manual build - required: false - -jobs: - run_test: - runs-on: ubuntu-20.04 - services: - mssql: - image: mcr.microsoft.com/mssql/server:2017-latest - env: - ACCEPT_EULA: Y - SA_PASSWORD: GitHub999 - ports: - - 1433:1433 - # needed because the mssql container does not provide a health check - options: --health-interval=10s --health-timeout=3s --health-start-period=10s --health-retries=10 --health-cmd="/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P ${SA_PASSWORD} -Q 'SELECT 1' || exit 1" - - steps: - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: install linux packages - run: | - git clone --depth 1 --branch develop https://github.com/openimis/database_ms_sqlserver.git ./sql - cd sql/ && bash concatenate_files.sh && cd .. - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y mssql-tools build-essential dialog apt-utils unixodbc-dev -y - python -m pip install --upgrade pip - - name: pull openimis backend - run: | - rm ./openimis -rf - git clone --depth 1 --branch develop https://github.com/openimis/openimis-be_py.git ./openimis - - name: copy current branch - uses: actions/checkout@v2 - with: - path: './current-module' - - name: Update the configuration - working-directory: ./openimis - run: | - export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')" - echo "the local module called $MODULE_NAME will be injected in openIMIS .json" - jq --arg name "$MODULE_NAME" 'if [.modules[].name == ($name)]| max then (.modules[] | select(.name == ($name)) | .pip)|="../current-module" else .modules |= .+ [{name:($name), pip:"../current-module"}] end' openimis.json - echo $(jq --arg name "$MODULE_NAME" 'if [.modules[].name == ($name)]| max then (.modules[] | select(.name == ($name)) | .pip)|="../current-module" else .modules |= .+ [{name:($name), pip:"../current-module"}] end' openimis.json) > openimis.json - - name: Install openIMIS Python dependencies - working-directory: ./openimis - run: | - pip install -r requirements.txt - python modules-requirements.py openimis.json > modules-requirements.txt - cat modules-requirements.txt - pip install -r modules-requirements.txt - - name: Environment info - working-directory: ./openimis - run: | - pip list - - name: Initialize DB - run: | - /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U SA -P $SA_PASSWORD -Q 'DROP DATABASE IF EXISTS imis' - /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U SA -P $SA_PASSWORD -Q 'CREATE DATABASE imis' - /opt/mssql-tools/bin/sqlcmd -S localhost,1433 -U SA -P $SA_PASSWORD -d imis -i sql/output/fullDemoDatabase.sql | grep . | uniq -c - env: - SA_PASSWORD: GitHub999 - ACCEPT_EULA: Y - -# - name: Check formatting with black -# run: | -# black --check . - - - name: Django tests - working-directory: ./openimis/openIMIS - run: | - export MODULE_NAME="$(echo $GITHUB_REPOSITORY | sed 's#^openimis/openimis-be-\(.*\)_py$#\1#')" - python -V - ls -l - python manage.py migrate - python init_test_db.py | grep . | uniq -c - python manage.py test --keepdb $MODULE_NAME - env: - SECRET_KEY: secret - DEBUG: true - #DJANGO_SETTINGS_MODULE: hat.settings - DB_HOST: localhost - DB_PORT: 1433 - DB_NAME: imis - DB_USER: sa - DB_PASSWORD: GitHub999 - #DEV_SERVER: true - SITE_ROOT: api -