-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
181 changed files
with
446 additions
and
677 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,154 +1,77 @@ | ||
| SHELL=/bin/bash -o pipefail | ||
| BUILD_PRINT = \e[1;34mSTEP: | ||
| END_BUILD_PRINT = \e[0m | ||
|
|
||
| CURRENT_UID := $(shell id -u) | ||
| export CURRENT_UID | ||
| # These are constants used for make targets so we can start prod and staging services on the same machine | ||
| ENV_FILE := .env | ||
|
|
||
| # include .env files if they exist | ||
| -include .env | ||
|
|
||
| PROJECT_PATH = $(shell pwd) | ||
| AIRFLOW_INFRA_FOLDER ?= ${PROJECT_PATH}/.airflow | ||
| RML_MAPPER_PATH = ${PROJECT_PATH}/.rmlmapper/rmlmapper.jar | ||
| XML_PROCESSOR_PATH = ${PROJECT_PATH}/.saxon/saxon-he-10.6.jar | ||
| LIMES_ALIGNMENT_PATH = $(PROJECT_PATH)/.limes/limes.jar | ||
| HOSTNAME = $(shell hostname) | ||
| CAROOT = $(shell pwd)/infra/traefik/certs | ||
|
|
||
| #----------------------------------------------------------------------------- | ||
| # Dev commands | ||
| #----------------------------------------------------------------------------- | ||
| install: | ||
| @ echo -e "$(BUILD_PRINT)Installing the requirements$(END_BUILD_PRINT)" | ||
| PACKAGE_PATH = $(PROJECT_PATH)/mapping_workbench | ||
|
|
||
| install: install-toolchain | ||
|
|
||
| install-dev: install-dev-toolchain | ||
|
|
||
| install-toolchain: | ||
| @ echo -e "$(BUILD_PRINT)Installing the TOOLCHAIN requirements$(END_BUILD_PRINT)" | ||
| @ pip install --upgrade pip | ||
| @ pip install --no-cache-dir -r requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt" | ||
| @ pip install --no-cache-dir -r $(PACKAGE_PATH)/toolchain/requirements.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt" | ||
| @ pip install $(PACKAGE_PATH)/toolchain | ||
|
|
||
| install-dev: | ||
| @ echo -e "$(BUILD_PRINT)Installing the dev requirements$(END_BUILD_PRINT)" | ||
| install-dev-toolchain: | ||
| @ echo -e "$(BUILD_PRINT)Installing the dev TOOLCHAIN requirements$(END_BUILD_PRINT)" | ||
| @ pip install --upgrade pip | ||
| @ pip install --no-cache-dir -r requirements.dev.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt" | ||
| @ pip install --no-cache-dir -r $(PACKAGE_PATH)/toolchain/requirements.dev.txt --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.5.1/constraints-no-providers-3.8.txt" | ||
|
|
||
| test-all: test-app test-core test-toolchain | ||
| test: test-unit | ||
| test-unit: test-app-unit test-core-unit test-toolchain-unit | ||
| test-e2e: test-app-e2e test-core-e2e test-toolchain-e2e | ||
| test-features: test-toolchain-features | ||
|
|
||
|
|
||
| test-app-unit: test-app-backend-unit test-app-frontend-unit | ||
|
|
||
| test-app-backend-unit: | ||
| @ echo "$(BUILD_PRINT)Unit Testing App Backend ... $(END_BUILD_PRINT)" | ||
|
|
||
| test-app-frontend-unit: | ||
| @ echo "$(BUILD_PRINT)Unit Testing App Frontend ... $(END_BUILD_PRINT)" | ||
|
|
||
| test-app-e2e: test-app-backend-e2e test-app-frontend-e2e | ||
|
|
||
| test-app-backend-e2e: | ||
| @ echo "$(BUILD_PRINT)E2E Testing App Backend ... $(END_BUILD_PRINT)" | ||
|
|
||
| test-app-frontend-e2e: | ||
| @ echo "$(BUILD_PRINT)E2E Testing App Frontend ... $(END_BUILD_PRINT)" | ||
|
|
||
| test-app: test-app-unit test-app-e2e | ||
|
|
||
|
|
||
| test-core-unit: | ||
| @ echo "$(BUILD_PRINT)Unit Testing Core ... $(END_BUILD_PRINT)" | ||
| @ tox -e unit-core | ||
|
|
||
| test-core-e2e: | ||
| @ echo "$(BUILD_PRINT)E2E Testing Core ... $(END_BUILD_PRINT)" | ||
| @ tox -e e2e-core | ||
|
|
||
| test-core: | ||
| @ echo "$(BUILD_PRINT)Testing Core ... $(END_BUILD_PRINT)" | ||
| @ tox -e core | ||
|
|
||
|
|
||
| test-toolchain-unit: | ||
| @ echo "$(BUILD_PRINT)Unit Testing Toolchain ... $(END_BUILD_PRINT)" | ||
| @ tox -e unit-toolchain | ||
|
|
||
| test-unit: | ||
| @ echo -e "$(BUILD_PRINT)Unit Testing ...$(END_BUILD_PRINT)" | ||
| @ tox -e unit | ||
|
|
||
| test-features: | ||
| @ echo -e "$(BUILD_PRINT)Gherkin Features Testing ...$(END_BUILD_PRINT)" | ||
| @ tox -e features | ||
|
|
||
| test-e2e: | ||
| @ echo -e "$(BUILD_PRINT)End to End Testing ...$(END_BUILD_PRINT)" | ||
| @ tox -e e2e | ||
|
|
||
| test-all-parallel: | ||
| @ echo -e "$(BUILD_PRINT)Complete Testing ...$(END_BUILD_PRINT)" | ||
| @ tox -p | ||
|
|
||
| test-all: | ||
| @ echo -e "$(BUILD_PRINT)Complete Testing ...$(END_BUILD_PRINT)" | ||
| @ tox | ||
|
|
||
|
|
||
| #----------------------------------------------------------------------------- | ||
| # VAULT SERVICES | ||
| #----------------------------------------------------------------------------- | ||
| # Testing whether an env variable is set or not | ||
| guard-%: | ||
| @ if [ "${${*}}" = "" ]; then \ | ||
| echo -e "$(BUILD_PRINT)Environment variable $* not set $(END_BUILD_PRINT)"; \ | ||
| exit 1; \ | ||
| fi | ||
|
|
||
| # Testing that vault is installed | ||
| vault-installed: #; @which vault1 > /dev/null | ||
| @ if ! hash vault 2>/dev/null; then \ | ||
| echo -e "$(BUILD_PRINT)Vault is not installed, refer to https://www.vaultproject.io/downloads $(END_BUILD_PRINT)"; \ | ||
| exit 1; \ | ||
| fi | ||
| # Get secrets in dotenv format | ||
| staging-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed | ||
| @ echo -e "$(BUILD_PRINT)Creating .env.staging file $(END_BUILD_PRINT)" | ||
| @ echo VAULT_ADDR=${VAULT_ADDR} > .env | ||
| @ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env | ||
| @ echo DOMAIN=ontobuilder.com >> .env | ||
| @ echo ENVIRONMENT=staging >> .env | ||
| @ echo SUBDOMAIN=staging. >> .env | ||
| @ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env | ||
| @ echo LIMES_ALIGNMENT_PATH=${LIMES_ALIGNMENT_PATH} >> .env | ||
| @ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env | ||
| @ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/staging >> .env | ||
| @ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env | ||
| @ vault kv get -format="json" ted-staging/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-staging/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-staging/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-staging/ted-sws | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-staging/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-staging/fuseki | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-staging/github | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-staging/minio | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
|
|
||
|
|
||
| dev-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed | ||
| @ echo -e "$(BUILD_PRINT)Create .env file $(END_BUILD_PRINT)" | ||
| @ echo VAULT_ADDR=${VAULT_ADDR} > .env | ||
| @ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env | ||
| @ echo DOMAIN=localhost >> .env | ||
| @ echo ENVIRONMENT=dev >> .env | ||
| @ echo SUBDOMAIN= >> .env | ||
| @ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env | ||
| @ echo LIMES_ALIGNMENT_PATH=${LIMES_ALIGNMENT_PATH} >> .env | ||
| @ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env | ||
| @ echo AIRFLOW_INFRA_FOLDER=${AIRFLOW_INFRA_FOLDER} >> .env | ||
| @ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env | ||
| @ vault kv get -format="json" ted-dev/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-dev/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-dev/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-dev/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-dev/fuseki | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-dev/ted-sws | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-dev/github | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-dev/minio | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
|
|
||
|
|
||
| prod-dotenv-file: guard-VAULT_ADDR guard-VAULT_TOKEN vault-installed | ||
| @ echo -e "$(BUILD_PRINT)Create .env file $(END_BUILD_PRINT)" | ||
| @ echo VAULT_ADDR=${VAULT_ADDR} > .env | ||
| @ echo VAULT_TOKEN=${VAULT_TOKEN} >> .env | ||
| @ echo DOMAIN=ted-data.eu >> .env | ||
| @ echo ENVIRONMENT=prod >> .env | ||
| @ echo SUBDOMAIN= >> .env | ||
| @ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env | ||
| @ echo LIMES_ALIGNMENT_PATH=${LIMES_ALIGNMENT_PATH} >> .env | ||
| @ echo XML_PROCESSOR_PATH=${XML_PROCESSOR_PATH} >> .env | ||
| @ echo AIRFLOW_INFRA_FOLDER=~/airflow-infra/prod >> .env | ||
| @ echo AIRFLOW_WORKER_HOSTNAME=${HOSTNAME} >> .env | ||
| @ echo AIRFLOW_CELERY_WORKER_CONCURRENCY=32 >> .env | ||
| @ vault kv get -format="json" ted-prod/airflow | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-prod/mongo-db | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-prod/metabase | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-prod/agraph | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-prod/fuseki | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-prod/ted-sws | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-prod/github | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
| @ vault kv get -format="json" ted-prod/minio | jq -r ".data.data | keys[] as \$$k | \"\(\$$k)=\(.[\$$k])\"" >> .env | ||
|
|
||
| local-dotenv-file: rml-mapper-path-add-dotenv-file | ||
|
|
||
| rml-mapper-path-add-dotenv-file: | ||
| @ echo -e "$(BUILD_PRINT)Add rml-mapper path to local .env file $(END_BUILD_PRINT)" | ||
| @ sed -i '/^RML_MAPPER_PATH/d' .env | ||
| @ echo RML_MAPPER_PATH=${RML_MAPPER_PATH} >> .env | ||
|
|
||
| refresh-mapping-files: | ||
| @ python -m ted_sws.data_manager.entrypoints.cli.cmd_generate_mapping_resources | ||
|
|
||
| install-local: | ||
| @ echo -e "$(BUILD_PRINT)Installing the local requirements$(END_BUILD_PRINT)" | ||
| @ pip install --upgrade --force-reinstall --no-deps ../ted-rdf-conversion-pipeline | ||
| test-toolchain-e2e: | ||
| @ echo "$(BUILD_PRINT)E2E Testing Toolchain ... $(END_BUILD_PRINT)" | ||
| @ tox -e e2e-toolchain | ||
|
|
||
| test-toolchain-features: | ||
| @ echo -e "$(BUILD_PRINT)Gherkin Features Testing Toolchain ...$(END_BUILD_PRINT)" | ||
| @ tox -e features-toolchain | ||
|
|
||
| test-toolchain: | ||
| @ echo "$(BUILD_PRINT)Testing Toolchain ... $(END_BUILD_PRINT)" | ||
| @ tox -e toolchain |
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| from distutils.core import setup | ||
|
|
||
| PKG_NAMESPACE = "mapping_workbench.app" | ||
| PKG_NAME = "mapping_workbench.app" | ||
| PKG_VERSION = "0.0.1" | ||
| PKG_DESCRIPTION = "Mapping Workbench Toolchain" | ||
|
|
||
| setup(name=PKG_NAME, | ||
| description='Mapping Workbench App', | ||
| author='Meaningfy', | ||
| author_email='[email protected]', | ||
| url='https://github.com/meaningfy-ws/mapping-workbench/tree/main/mapping_workbench/app', | ||
| packages=[ | ||
| 'mapping_workbench.app' | ||
| ], | ||
| package_dir={ | ||
| 'mapping_workbench.app': '.' | ||
| }, | ||
| namespace_packages=[ | ||
| 'mapping_workbench.app' | ||
| ], | ||
| ) |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| # mapping-workbench | ||
| RML mapping workbench core |
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| #!/usr/bin/python3 | ||
|
|
||
| """ """ | ||
|
|
||
| __version__ = "0.0.1" | ||
|
|
||
| import pathlib | ||
| import re | ||
|
|
||
| from pkg_resources import parse_requirements | ||
| from setuptools import setup, find_packages | ||
|
|
||
| kwargs = {} | ||
|
|
||
| with pathlib.Path('requirements.txt').open() as requirements: | ||
| kwargs["install_requires"] = [str(requirement) for requirement in parse_requirements(requirements)] | ||
|
|
||
| kwargs["extras_require"] = {} | ||
|
|
||
| PACKAGE_DIR = 'src' | ||
| NAME = "mapping_workbench.core" | ||
| DESCRIPTION = "Mapping Workbench Core" | ||
| LONG_DESCRIPTION = pathlib.Path("README.md").read_text(encoding="utf-8") | ||
| URL = "https://github.com/meaningfy-ws/mapping-workbench/tree/main/mapping_workbench/core" | ||
|
|
||
| SOURCE_PACKAGE_REGEX = re.compile(rf'^{PACKAGE_DIR}') | ||
| source_packages = find_packages(include=[PACKAGE_DIR, f'{PACKAGE_DIR}.*']) | ||
| PACKAGES = [SOURCE_PACKAGE_REGEX.sub(NAME, name) for name in source_packages] | ||
|
|
||
| setup( | ||
| name=NAME, | ||
| version=__version__, | ||
| description=DESCRIPTION, | ||
| author="Meaningfy", | ||
| author_email="[email protected]", | ||
| maintainer="Meaningfy Team", | ||
| maintainer_email="[email protected]", | ||
| url=URL, | ||
| license="Apache License 2.0", | ||
| platforms=["any"], | ||
| python_requires=">=3.7", | ||
| classifiers=[ | ||
| "Intended Audience :: Developers", | ||
| "Intended Audience :: System Administrators", | ||
| "Programming Language :: Python", | ||
| "Programming Language :: Python :: 3", | ||
| "Programming Language :: Python :: 3.7", | ||
| "Programming Language :: Python :: 3.8", | ||
| "Programming Language :: Python :: 3.9", | ||
| "Programming Language :: Python :: 3.10", | ||
| "License :: OSI Approved :: Apache Software License", | ||
| "Operating System :: OS Independent", | ||
| "Natural Language :: English", | ||
| ], | ||
| long_description=LONG_DESCRIPTION, | ||
| long_description_content_type="text/markdown", | ||
| packages=PACKAGES, | ||
| package_dir={NAME: PACKAGE_DIR}, | ||
| include_package_data=True, | ||
| tests_require=['pytest'], | ||
| **kwargs, | ||
| ) |
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| # mapping-workbench | ||
| RML mapping workbench toolchain |
4 changes: 1 addition & 3 deletions
4
mapping_workbench/__init__.py → mapping_workbench/toolchain/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,6 @@ | ||
| __version__ = "0.0.1" | ||
|
|
||
| import os | ||
|
|
||
| RUN_ENV_NAME = "__MAPPING_WORKBENCH_RUN_ENV__" | ||
| RUN_ENV_VAL = "mapping-workbench" | ||
| RUN_TEST_ENV_VAL = "test" | ||
| os.environ[RUN_ENV_NAME] = RUN_ENV_VAL | ||
| os.environ[RUN_ENV_NAME] = RUN_ENV_VAL |
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.