From 5226b5e4400487f8eed53fb72bf15646709ba9b8 Mon Sep 17 00:00:00 2001 From: Jiahui Yang <“jyang753@wisc.edu”> Date: Thu, 12 Sep 2024 11:11:17 -0500 Subject: [PATCH 1/9] yaml converter all done --- docker-compose.yml | 406 ++++++++++---------- services/app/__init__.py | 8 +- services/app/yaml_converter/__init__.py | 0 services/app/yaml_converter/fetch_routes.py | 37 ++ services/app/yaml_converter/routes.py | 123 ++++++ services/requirements/prod_requirements.txt | 6 +- services/requirements/requirements.txt | 1 + 7 files changed, 375 insertions(+), 206 deletions(-) create mode 100644 services/app/yaml_converter/__init__.py create mode 100644 services/app/yaml_converter/fetch_routes.py create mode 100644 services/app/yaml_converter/routes.py diff --git a/docker-compose.yml b/docker-compose.yml index c8b809e9..da4f0027 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,140 +1,140 @@ version: '3' services: - mongo: - image: 'mongo:4.4' - restart: always - container_name: mongo - environment: - - MONGO_INITDB_ROOT_USERNAME=${MM_MONGO_USER} - - MONGO_INITDB_ROOT_PASSWORD=${MM_MONGO_PWD} - - MONGO_DEVS=${MONGO_DEVS} - - MONGO_DEV_PWD=${MONGO_DEV_PWD} - - MONGO_PORT=${MONGO_PORT} - - MM_DB=${MM_DB} - ports: - - '27017:27017' - volumes: - - ./mockDB/db:/data/db - - ./mockDB/restore:/restore - - ./mockDB/log:/var/log/mongodb - - ./mockDB/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh - es: - image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.1' - container_name: es - restart: always - environment: - - node.name=es - - bootstrap.memory_lock=true - - cluster.initial_master_nodes=es - - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' - - xpack.security.enabled=false - ulimits: - memlock: - soft: -1 - hard: -1 - ports: - - '9200:9200' - volumes: - - ./mockDB/es:/usr/share/elasticsearch/data - minio: - container_name: minio - image: minio/minio - command: server --console-address ":9001" /data - environment: - - MINIO_ROOT_USER=${MINIO_ROOT_USER} - - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - ports: - - '9000:9000' - - '9001:9001' - volumes: - - ./mockDB/minio:/data - proxy: - container_name: proxy - depends_on: - - api - - client - restart: always - build: - dockerfile: Dockerfile.dev - context: ./nginx - ports: - - '80:80' - api: - depends_on: - - es - - mongo - - minio - container_name: restful - restart: always - build: - dockerfile: Dockerfile.dev - context: ./resfulservice - volumes: - - /app/node_modules - - ./resfulservice:/app - environment: - - DB_USERNAME=${MONGO_DEVS} - - DB_PASSWORD=${MONGO_DEV_PWD} - - MM_DB=${MM_DB} - - MONGO_ADDRESS=${MONGO_ADDRESS} - - MONGO_PORT=${MONGO_PORT} - - TKNS=${TKNS} - - FILES_DIRECTORY=${FILES_DIRECTORY} - - ESADDRESS=${ESADDRESS} - - PORT=${PORT} - - WORKER_PORT=${WORKER_PORT} - - MINIO_PORT=${MINIO_PORT} - - MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT} - - MINIO_BUCKET=${MINIO_BUCKET} - - MINIO_ROOT_USER=${MINIO_ROOT_USER} - - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - - ROUTER=${HOST_PORT} - - MM_RUNTIME_ENV=${MM_RUNTIME_ENV} - - MM_USER=${MM_USER} - - MM_USER_EMAIL=${MM_USER_EMAIL} - - MM_AUTH_GIVEN_NAME_HEADER=${MM_AUTH_GIVEN_NAME_HEADER} - - MM_AUTH_DISPLAYNAME_HEADER=${MM_AUTH_DISPLAYNAME_HEADER} - - MM_AUTH_SURNAME_HEADER=${MM_AUTH_SURNAME_HEADER} - - MM_AUTH_EMAIL_HEADER=${MM_AUTH_EMAIL_HEADER} - - MM_AUTH_USER_HEADER=${MM_AUTH_USER_HEADER} - - KNOWLEDGE_ADDRESS=${KNOWLEDGE_ADDRESS} - - MAIL_SERVER_HOST=${MAIL_SERVER_HOST} - - MAIL_SERVER_PORT=${MAIL_SERVER_PORT} - - MAIL_USER_ADDRESS=${MAIL_USER_ADDRESS} - - MAIL_USER_PASSWORD=${MAIL_USER_PASSWORD} - - TERM_OF_SERVICE=${TERM_OF_SERVICE} - - SWAGGER_CONTACT_NAME=${SWAGGER_CONTACT_NAME} - - SWAGGER_CONTACT_EMAIL=${SWAGGER_CONTACT_EMAIL} - - DEV_ENDPOINT=${DEV_ENDPOINT} - - QA_SERVER_ENDPOINT=${QA_SERVER_ENDPOINT} - - PROD_SERVER_ENDPOINT=${PROD_SERVER_ENDPOINT} - - MANAGED_SERVICE_ADDRESS=${MANAGED_SERVICE_ADDRESS} - - SYSTEM_EMAIL=${SYSTEM_EMAIL} - - DEPLOYMENT_ADDRESS=${DEPLOYMENT_ADDRESS} - - DOCKER_HUB_ADDRESS=${DOCKER_HUB_ADDRESS} - - DEPLOYMENT_SECRET=${DEPLOYMENT_SECRET} - - DYNAMFIT_TEST_FILE=${DYNAMFIT_TEST_FILE} - ports: - - '3001:3001' - - '3002:3002' - client: - stdin_open: true - container_name: frontend - environment: - - CHOKIDAR_USEPOLLING=true - - API_PORT=${PORT} - - MINIO_PORT=${MINIO_PORT} - - SERVICE_PORT=${HOST_PORT} - build: - dockerfile: Dockerfile.dev - context: ./app - volumes: - - /app/node_modules - - ./app:/app +# mongo: +# image: 'mongo:4.4' +# restart: always +# container_name: mongo +# environment: +# - MONGO_INITDB_ROOT_USERNAME=${MM_MONGO_USER} +# - MONGO_INITDB_ROOT_PASSWORD=${MM_MONGO_PWD} +# - MONGO_DEVS=${MONGO_DEVS} +# - MONGO_DEV_PWD=${MONGO_DEV_PWD} +# - MONGO_PORT=${MONGO_PORT} +# - MM_DB=${MM_DB} +# ports: +# - '27017:27017' +# volumes: +# - ./mockDB/db:/data/db +# - ./mockDB/restore:/restore +# - ./mockDB/log:/var/log/mongodb +# - ./mockDB/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh +# es: +# image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.1' +# container_name: es +# restart: always +# environment: +# - node.name=es +# - bootstrap.memory_lock=true +# - cluster.initial_master_nodes=es +# - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' +# - xpack.security.enabled=false +# ulimits: +# memlock: +# soft: -1 +# hard: -1 +# ports: +# - '9200:9200' +# volumes: +# - ./mockDB/es:/usr/share/elasticsearch/data +# minio: +# container_name: minio +# image: minio/minio +# command: server --console-address ":9001" /data +# environment: +# - MINIO_ROOT_USER=${MINIO_ROOT_USER} +# - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} +# ports: +# - '9000:9000' +# - '9001:9001' +# volumes: +# - ./mockDB/minio:/data +# proxy: +# container_name: proxy +# depends_on: +# - api +# - client +# restart: always +# build: +# dockerfile: Dockerfile.dev +# context: ./nginx +# ports: +# - '80:80' +# api: +# depends_on: +# - es +# - mongo +# - minio +# container_name: restful +# restart: always +# build: +# dockerfile: Dockerfile.dev +# context: ./resfulservice +# volumes: +# - /app/node_modules +# - ./resfulservice:/app +# environment: +# - DB_USERNAME=${MONGO_DEVS} +# - DB_PASSWORD=${MONGO_DEV_PWD} +# - MM_DB=${MM_DB} +# - MONGO_ADDRESS=${MONGO_ADDRESS} +# - MONGO_PORT=${MONGO_PORT} +# - TKNS=${TKNS} +# - FILES_DIRECTORY=${FILES_DIRECTORY} +# - ESADDRESS=${ESADDRESS} +# - PORT=${PORT} +# - WORKER_PORT=${WORKER_PORT} +# - MINIO_PORT=${MINIO_PORT} +# - MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT} +# - MINIO_BUCKET=${MINIO_BUCKET} +# - MINIO_ROOT_USER=${MINIO_ROOT_USER} +# - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} +# - ROUTER=${HOST_PORT} +# - MM_RUNTIME_ENV=${MM_RUNTIME_ENV} +# - MM_USER=${MM_USER} +# - MM_USER_EMAIL=${MM_USER_EMAIL} +# - MM_AUTH_GIVEN_NAME_HEADER=${MM_AUTH_GIVEN_NAME_HEADER} +# - MM_AUTH_DISPLAYNAME_HEADER=${MM_AUTH_DISPLAYNAME_HEADER} +# - MM_AUTH_SURNAME_HEADER=${MM_AUTH_SURNAME_HEADER} +# - MM_AUTH_EMAIL_HEADER=${MM_AUTH_EMAIL_HEADER} +# - MM_AUTH_USER_HEADER=${MM_AUTH_USER_HEADER} +# - KNOWLEDGE_ADDRESS=${KNOWLEDGE_ADDRESS} +# - MAIL_SERVER_HOST=${MAIL_SERVER_HOST} +# - MAIL_SERVER_PORT=${MAIL_SERVER_PORT} +# - MAIL_USER_ADDRESS=${MAIL_USER_ADDRESS} +# - MAIL_USER_PASSWORD=${MAIL_USER_PASSWORD} +# - TERM_OF_SERVICE=${TERM_OF_SERVICE} +# - SWAGGER_CONTACT_NAME=${SWAGGER_CONTACT_NAME} +# - SWAGGER_CONTACT_EMAIL=${SWAGGER_CONTACT_EMAIL} +# - DEV_ENDPOINT=${DEV_ENDPOINT} +# - QA_SERVER_ENDPOINT=${QA_SERVER_ENDPOINT} +# - PROD_SERVER_ENDPOINT=${PROD_SERVER_ENDPOINT} +# - MANAGED_SERVICE_ADDRESS=${MANAGED_SERVICE_ADDRESS} +# - SYSTEM_EMAIL=${SYSTEM_EMAIL} +# - DEPLOYMENT_ADDRESS=${DEPLOYMENT_ADDRESS} +# - DOCKER_HUB_ADDRESS=${DOCKER_HUB_ADDRESS} +# - DEPLOYMENT_SECRET=${DEPLOYMENT_SECRET} +# - DYNAMFIT_TEST_FILE=${DYNAMFIT_TEST_FILE} +# ports: +# - '3001:3001' +# - '3002:3002' +# client: +# stdin_open: true +# container_name: frontend +# environment: +# - CHOKIDAR_USEPOLLING=true +# - API_PORT=${PORT} +# - MINIO_PORT=${MINIO_PORT} +# - SERVICE_PORT=${HOST_PORT} +# build: +# dockerfile: Dockerfile.dev +# context: ./app +# volumes: +# - /app/node_modules +# - ./app:/app managedservices: container_name: managedservices - depends_on: - - mongo + # depends_on: + # - mongo restart: always build: dockerfile: Dockerfile @@ -151,76 +151,76 @@ services: - AUTH_API_TOKEN_EMAIL=${MANAGED_SERVICES_AUTH_API_TOKEN_EMAIL} - AUTH_API_REFRESH_EMAIL=${MANAGED_SERVICES_AUTH_API_REFRESH_EMAIL} - API_URL=${API_URL} - redis: - image: 'redis:alpine' - command: redis-server - volumes: - - ./mockDB/redis-data:/var/lib/redis - celery: - depends_on: - - redis - - fuseki - volumes: - - ./mockDB/fuseki:/app/run - - ./whyis/materialsmine:/app - command: - /opt/venv/bin/celery -A wsgi.celery worker -l INFO -c 4 --uid=nobody - --gid=nogroup - environment: - - CHOKIDAR_USEPOLLING=true - build: whyis - celerybeat: - depends_on: - - redis - - fuseki - volumes: - - ./mockDB/fuseki:/app/run - - ./whyis/materialsmine:/app - command: - /opt/venv/bin/celery -A wsgi.celery beat -l INFO --uid=nobody - --gid=nogroup - environment: - - CHOKIDAR_USEPOLLING=true - build: whyis - whyis: - container_name: whyis - depends_on: - - redis - - fuseki - stdin_open: true - command: - /opt/venv/bin/gunicorn wsgi:application --workers ${WEB_CONCURRENCY:-8} - --timeout 0 -b :8000 - environment: - - CHOKIDAR_USEPOLLING=true - - NM_GRAPH_AUTH_SECRET=${TKNS} - build: whyis - volumes: - - ./whyis/materialsmine:/app - - ./mockDB/fuseki:/app/run - ports: - - '8000:8000' - mem_limit: 2048m - cpus: '0.25' - fuseki: - build: whyis - command: /opt/venv/bin/fuseki-server --mem /ds - # Needs fuseki 4.3.2 to use our full text search, stain is still at 4.0.0. - # image: 'stain/jena-fuseki:4.0.0 - restart: always - # mem_limit: 9G - # mem_reservation: 9G - environment: - # - ADMIN_USER=admin - # - ADMIN_PASSWORD=${ADMIN_PASSWORD} - - JVM_ARGS=-Xmx4g - - FUSEKI_BASE=/fuseki - ports: - - '3030:3030' - volumes: - - ./mockDB/fuseki:/fuseki - mem_limit: 3072m - cpus: '0.45' +# redis: +# image: 'redis:alpine' +# command: redis-server +# volumes: +# - ./mockDB/redis-data:/var/lib/redis +# celery: +# depends_on: +# - redis +# - fuseki +# volumes: +# - ./mockDB/fuseki:/app/run +# - ./whyis/materialsmine:/app +# command: +# /opt/venv/bin/celery -A wsgi.celery worker -l INFO -c 4 --uid=nobody +# --gid=nogroup +# environment: +# - CHOKIDAR_USEPOLLING=true +# build: whyis +# celerybeat: +# depends_on: +# - redis +# - fuseki +# volumes: +# - ./mockDB/fuseki:/app/run +# - ./whyis/materialsmine:/app +# command: +# /opt/venv/bin/celery -A wsgi.celery beat -l INFO --uid=nobody +# --gid=nogroup +# environment: +# - CHOKIDAR_USEPOLLING=true +# build: whyis +# whyis: +# container_name: whyis +# depends_on: +# - redis +# - fuseki +# stdin_open: true +# command: +# /opt/venv/bin/gunicorn wsgi:application --workers ${WEB_CONCURRENCY:-8} +# --timeout 0 -b :8000 +# environment: +# - CHOKIDAR_USEPOLLING=true +# - NM_GRAPH_AUTH_SECRET=${TKNS} +# build: whyis +# volumes: +# - ./whyis/materialsmine:/app +# - ./mockDB/fuseki:/app/run +# ports: +# - '8000:8000' +# mem_limit: 2048m +# cpus: '0.25' +# fuseki: +# build: whyis +# command: /opt/venv/bin/fuseki-server --mem /ds +# # Needs fuseki 4.3.2 to use our full text search, stain is still at 4.0.0. +# # image: 'stain/jena-fuseki:4.0.0 +# restart: always +# # mem_limit: 9G +# # mem_reservation: 9G +# environment: +# # - ADMIN_USER=admin +# # - ADMIN_PASSWORD=${ADMIN_PASSWORD} +# - JVM_ARGS=-Xmx4g +# - FUSEKI_BASE=/fuseki +# ports: +# - '3030:3030' +# volumes: +# - ./mockDB/fuseki:/fuseki +# mem_limit: 3072m +# cpus: '0.45' volumes: mockDB: diff --git a/services/app/__init__.py b/services/app/__init__.py index af9751fd..7f05ca20 100644 --- a/services/app/__init__.py +++ b/services/app/__init__.py @@ -26,12 +26,16 @@ def create_app(config_class = Config): from app.errors.handlers import error from app.chemprops.routes import chemprops from app.dynamfit.routes import dynamfit + from app.yaml_converter.routes import yaml_converter from app.main.routes import main app.register_blueprint(error) app.register_blueprint(chemprops) app.register_blueprint(dynamfit) + app.register_blueprint(yaml_converter) app.register_blueprint(main) - - return app \ No newline at end of file + return app + + + diff --git a/services/app/yaml_converter/__init__.py b/services/app/yaml_converter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/services/app/yaml_converter/fetch_routes.py b/services/app/yaml_converter/fetch_routes.py new file mode 100644 index 00000000..bd0910e5 --- /dev/null +++ b/services/app/yaml_converter/fetch_routes.py @@ -0,0 +1,37 @@ +from flask import Flask, Blueprint, Response, jsonify +import requests + + +# Create a Blueprint named 'yaml_converter' with a URL prefix '/yaml_converter' +yaml_converter = Blueprint("yaml_converter", __name__, url_prefix="/yaml_converter") + + +# Define a route that accepts "XML GET" requests +@yaml_converter.route("/fetch_xml/", methods=["GET"]) +def fetch_xml(control_id): + + # Construct the external URL using the provided control_id + external_url = f"https://qa.materialsmine.org/api/xml/{control_id}?format=xml" + + try: + # Send a GET request to the external URL + response = requests.get(external_url) + + # Check the status code of the response + if response.status_code == 200: + # If the request is successful, return the XML content with a message + xml_content = response.content # XML content is in bytes + return Response(xml_content, mimetype='application/xml') + + elif response.status_code == 404: + # If the resource is not found, return a 404 error message + return jsonify({"message": "Curation sample not found"}), 404 + else: + # For any other status code, return a generic error message + return jsonify({"message": "Fetch Failed"}), 500 + + except requests.exceptions.RequestException as e: + # If there is a network-related error, return an error message with exception details + return jsonify({"message": f"An error occurred: {str(e)}"}), 500 + + diff --git a/services/app/yaml_converter/routes.py b/services/app/yaml_converter/routes.py new file mode 100644 index 00000000..6da650fe --- /dev/null +++ b/services/app/yaml_converter/routes.py @@ -0,0 +1,123 @@ +from flask import Flask, Blueprint, Response, jsonify, request +import requests +import xml.etree.ElementTree as ET +from collections import OrderedDict +from ruamel.yaml import YAML + + +# Create a Blueprint named 'yaml_converter' with a URL prefix '/yaml_converter' +yaml_converter = Blueprint("yaml_converter", __name__, url_prefix="/yaml_converter") + + +# Define a route that accepts "XML GET" requests and converts the XML to YAML +@yaml_converter.route("/", methods=["GET"]) +def fetch_xml(control_id): + + # Check for query parameters to determine which external URL to use + # Default to 'qa.materialsmine.org', but allow switching to 'materialsmine.org' + use_production = request.args.get('production', 'false').lower() == 'true' + + if use_production: + external_url = f"https://materialsmine.org/api/xml/{control_id}?format=xml" + else: + external_url = f"https://qa.materialsmine.org/api/xml/{control_id}?format=xml" + + try: + # Send a GET request to the external URL + response = requests.get(external_url) + + # Check the status code of the response + if response.status_code == 200: + # If the request is successful, parse the XML content + xml_content = response.content + xml_tree = ET.ElementTree(ET.fromstring(xml_content)) + xml_root = xml_tree.getroot() + xml_dict = OrderedDict({xml_root.tag: parse_xml_to_dict(xml_root)}) # Ensure the root tag is included properly + + # Convert the OrderedDict to YAML format + yaml_data = convert_dict_to_yaml(xml_dict) + + # Return the YAML data as a response + return Response(yaml_data, mimetype='text/yaml') + + elif response.status_code == 404: + # If the resource is not found, return a 404 error message + return jsonify({"message": "Curation sample not found"}), 404 + else: + # For any other status code, return a generic error message + return jsonify({"message": "Fetch Failed"}), 500 + + except requests.exceptions.RequestException as e: + # If there is a network-related error, return an error message with exception details + return jsonify({"message": f"An error occurred: {str(e)}"}), 500 + +def parse_xml_to_dict(element): + """ + Recursively parses an XML element into an OrderedDict. + + Args: + - element (xml.etree.ElementTree.Element): The XML element to parse. + + Returns: + - OrderedDict: OrderedDict representing the XML element. + """ + parsed_dict = OrderedDict() + + # Process element attributes + if element.attrib: + parsed_dict["@attributes"] = element.attrib + + # Process child elements + for child in element: + child_dict = parse_xml_to_dict(child) + if child.tag not in parsed_dict: + parsed_dict[child.tag] = child_dict + else: + if not isinstance(parsed_dict[child.tag], list): + parsed_dict[child.tag] = [parsed_dict[child.tag]] + parsed_dict[child.tag].append(child_dict) + + # Process element text + if element.text and element.text.strip(): + if parsed_dict: + parsed_dict["@text"] = element.text.strip() + else: + return element.text.strip() + + return parsed_dict + +def convert_dict_to_yaml(data_dict): + """ + Converts a dictionary to a YAML string. + + Args: + - data_dict (dict): The dictionary to convert. + + Returns: + - str: The YAML string. + """ + yaml = YAML() + yaml.default_flow_style = False + yaml.allow_unicode = True + yaml.explicit_start = False + yaml.default_style = None + + # Convert OrderedDict to regular dict to avoid !!omap + def convert_ordereddict(d): + if isinstance(d, OrderedDict): + return dict((k, convert_ordereddict(v)) for k, v in d.items()) + elif isinstance(d, list): + return [convert_ordereddict(i) for i in d] + else: + return d + + clean_dict = convert_ordereddict(data_dict) + + from io import StringIO + yaml_stream = StringIO() + yaml.dump(clean_dict, yaml_stream) + return yaml_stream.getvalue() + + + + diff --git a/services/requirements/prod_requirements.txt b/services/requirements/prod_requirements.txt index afe944ad..f17726fa 100644 --- a/services/requirements/prod_requirements.txt +++ b/services/requirements/prod_requirements.txt @@ -42,4 +42,8 @@ typing_extensions==4.9.0 tzdata==2023.4 urllib3==2.1.0 Werkzeug==3.0.1 -xlrd==1.2.0 \ No newline at end of file +xlrd==1.2.0 +ruamel.yaml==0.17.32 + + + diff --git a/services/requirements/requirements.txt b/services/requirements/requirements.txt index e991df9b..0bfa2070 100644 --- a/services/requirements/requirements.txt +++ b/services/requirements/requirements.txt @@ -16,6 +16,7 @@ packaging==23.2 plotly==5.15.0 tenacity==8.2.3 PyJWT==2.8.0 +ruamel.yaml==0.17.32 # NOT NEEDED # flask-bcrypt==0.7.1 From b34c75736434e30efab74f55e656e28b694e86bd Mon Sep 17 00:00:00 2001 From: tholulomo Date: Tue, 8 Oct 2024 09:41:20 -0400 Subject: [PATCH 2/9] FEAT(Jhyang): Refactor managed services --- app/src/pages/explorer/xml/XmlLoader.vue | 88 ++--- docker-compose.yml | 335 ++++++++------------ services/app/yaml_converter/routes.py | 12 +- services/app/yaml_converter/yaml_utils.py | 17 + services/requirements/prod_requirements.txt | 5 +- 5 files changed, 207 insertions(+), 250 deletions(-) create mode 100644 services/app/yaml_converter/yaml_utils.py diff --git a/app/src/pages/explorer/xml/XmlLoader.vue b/app/src/pages/explorer/xml/XmlLoader.vue index 0dde9369..12032d1d 100644 --- a/app/src/pages/explorer/xml/XmlLoader.vue +++ b/app/src/pages/explorer/xml/XmlLoader.vue @@ -94,6 +94,14 @@ arrow_back + + Convert to Yaml + code + + diff --git a/docker-compose.yml b/docker-compose.yml index 682ea451..4000d616 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,136 +1,136 @@ version: '3' services: -# mongo: -# image: 'mongo:4.4' -# restart: always -# container_name: mongo -# environment: -# - MONGO_INITDB_ROOT_USERNAME=${MM_MONGO_USER} -# - MONGO_INITDB_ROOT_PASSWORD=${MM_MONGO_PWD} -# - MONGO_DEVS=${MONGO_DEVS} -# - MONGO_DEV_PWD=${MONGO_DEV_PWD} -# - MONGO_PORT=${MONGO_PORT} -# - MM_DB=${MM_DB} -# ports: -# - '27017:27017' -# volumes: -# - ./mockDB/db:/data/db -# - ./mockDB/restore:/restore -# - ./mockDB/log:/var/log/mongodb -# - ./mockDB/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh -# es: -# image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.1' -# container_name: es -# restart: always -# environment: -# - node.name=es -# - bootstrap.memory_lock=true -# - cluster.initial_master_nodes=es -# - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' -# - xpack.security.enabled=false -# ulimits: -# memlock: -# soft: -1 -# hard: -1 -# ports: -# - '9200:9200' -# volumes: -# - ./mockDB/es:/usr/share/elasticsearch/data -# minio: -# container_name: minio -# image: minio/minio -# command: server --console-address ":9001" /data -# environment: -# - MINIO_ROOT_USER=${MINIO_ROOT_USER} -# - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} -# ports: -# - '9000:9000' -# - '9001:9001' -# volumes: -# - ./mockDB/minio:/data -# proxy: -# container_name: proxy -# depends_on: -# - api -# - client -# restart: always -# build: -# dockerfile: Dockerfile.dev -# context: ./nginx -# ports: -# - '80:80' -# api: -# depends_on: -# - es -# - mongo -# - minio -# container_name: restful -# restart: always -# build: -# dockerfile: Dockerfile.dev -# context: ./resfulservice -# volumes: -# - /app/node_modules -# - ./resfulservice:/app -# environment: -# - DB_USERNAME=${MONGO_DEVS} -# - DB_PASSWORD=${MONGO_DEV_PWD} -# - MM_DB=${MM_DB} -# - MONGO_ADDRESS=${MONGO_ADDRESS} -# - MONGO_PORT=${MONGO_PORT} -# - TKNS=${TKNS} -# - FILES_DIRECTORY=${FILES_DIRECTORY} -# - ESADDRESS=${ESADDRESS} -# - PORT=${PORT} -# - WORKER_PORT=${WORKER_PORT} -# - MINIO_PORT=${MINIO_PORT} -# - MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT} -# - MINIO_BUCKET=${MINIO_BUCKET} -# - MINIO_ROOT_USER=${MINIO_ROOT_USER} -# - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} -# - ROUTER=${HOST_PORT} -# - MM_RUNTIME_ENV=${MM_RUNTIME_ENV} -# - MM_USER=${MM_USER} -# - MM_USER_EMAIL=${MM_USER_EMAIL} -# - MM_AUTH_GIVEN_NAME_HEADER=${MM_AUTH_GIVEN_NAME_HEADER} -# - MM_AUTH_DISPLAYNAME_HEADER=${MM_AUTH_DISPLAYNAME_HEADER} -# - MM_AUTH_SURNAME_HEADER=${MM_AUTH_SURNAME_HEADER} -# - MM_AUTH_EMAIL_HEADER=${MM_AUTH_EMAIL_HEADER} -# - MM_AUTH_USER_HEADER=${MM_AUTH_USER_HEADER} -# - KNOWLEDGE_ADDRESS=${KNOWLEDGE_ADDRESS} -# - MAIL_SERVER_HOST=${MAIL_SERVER_HOST} -# - MAIL_SERVER_PORT=${MAIL_SERVER_PORT} -# - MAIL_USER_ADDRESS=${MAIL_USER_ADDRESS} -# - MAIL_USER_PASSWORD=${MAIL_USER_PASSWORD} -# - TERM_OF_SERVICE=${TERM_OF_SERVICE} -# - SWAGGER_CONTACT_NAME=${SWAGGER_CONTACT_NAME} -# - SWAGGER_CONTACT_EMAIL=${SWAGGER_CONTACT_EMAIL} -# - DEV_ENDPOINT=${DEV_ENDPOINT} -# - QA_SERVER_ENDPOINT=${QA_SERVER_ENDPOINT} -# - PROD_SERVER_ENDPOINT=${PROD_SERVER_ENDPOINT} -# - MANAGED_SERVICE_ADDRESS=${MANAGED_SERVICE_ADDRESS} -# - SYSTEM_EMAIL=${SYSTEM_EMAIL} -# - DEPLOYMENT_ADDRESS=${DEPLOYMENT_ADDRESS} -# - DOCKER_HUB_ADDRESS=${DOCKER_HUB_ADDRESS} -# - DEPLOYMENT_SECRET=${DEPLOYMENT_SECRET} -# - DYNAMFIT_TEST_FILE=${DYNAMFIT_TEST_FILE} -# ports: -# - '3001:3001' -# - '3002:3002' -# client: -# stdin_open: true -# container_name: frontend -# environment: -# - CHOKIDAR_USEPOLLING=true -# - API_PORT=${PORT} -# - MINIO_PORT=${MINIO_PORT} -# - SERVICE_PORT=${HOST_PORT} -# build: -# dockerfile: Dockerfile.dev -# context: ./app -# volumes: -# - /app/node_modules -# - ./app:/app + # mongo: + # image: 'mongo:4.4' + # restart: always + # container_name: mongo + # environment: + # - MONGO_INITDB_ROOT_USERNAME=${MM_MONGO_USER} + # - MONGO_INITDB_ROOT_PASSWORD=${MM_MONGO_PWD} + # - MONGO_DEVS=${MONGO_DEVS} + # - MONGO_DEV_PWD=${MONGO_DEV_PWD} + # - MONGO_PORT=${MONGO_PORT} + # - MM_DB=${MM_DB} + # ports: + # - '27017:27017' + # volumes: + # - ./mockDB/db:/data/db + # - ./mockDB/restore:/restore + # - ./mockDB/log:/var/log/mongodb + # - ./mockDB/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh + # es: + # image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.1' + # container_name: es + # restart: always + # environment: + # - node.name=es + # - bootstrap.memory_lock=true + # - cluster.initial_master_nodes=es + # - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' + # - xpack.security.enabled=false + # ulimits: + # memlock: + # soft: -1 + # hard: -1 + # ports: + # - '9200:9200' + # volumes: + # - ./mockDB/es:/usr/share/elasticsearch/data + # minio: + # container_name: minio + # image: minio/minio + # command: server --console-address ":9001" /data + # environment: + # - MINIO_ROOT_USER=${MINIO_ROOT_USER} + # - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + # ports: + # - '9000:9000' + # - '9001:9001' + # volumes: + # - ./mockDB/minio:/data + # proxy: + # container_name: proxy + # depends_on: + # - api + # - client + # restart: always + # build: + # dockerfile: Dockerfile.dev + # context: ./nginx + # ports: + # - '80:80' + # api: + # depends_on: + # - es + # - mongo + # - minio + # container_name: restful + # restart: always + # build: + # dockerfile: Dockerfile.dev + # context: ./resfulservice + # volumes: + # - /app/node_modules + # - ./resfulservice:/app + # environment: + # - DB_USERNAME=${MONGO_DEVS} + # - DB_PASSWORD=${MONGO_DEV_PWD} + # - MM_DB=${MM_DB} + # - MONGO_ADDRESS=${MONGO_ADDRESS} + # - MONGO_PORT=${MONGO_PORT} + # - TKNS=${TKNS} + # - FILES_DIRECTORY=${FILES_DIRECTORY} + # - ESADDRESS=${ESADDRESS} + # - PORT=${PORT} + # - WORKER_PORT=${WORKER_PORT} + # - MINIO_PORT=${MINIO_PORT} + # - MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT} + # - MINIO_BUCKET=${MINIO_BUCKET} + # - MINIO_ROOT_USER=${MINIO_ROOT_USER} + # - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + # - ROUTER=${HOST_PORT} + # - MM_RUNTIME_ENV=${MM_RUNTIME_ENV} + # - MM_USER=${MM_USER} + # - MM_USER_EMAIL=${MM_USER_EMAIL} + # - MM_AUTH_GIVEN_NAME_HEADER=${MM_AUTH_GIVEN_NAME_HEADER} + # - MM_AUTH_DISPLAYNAME_HEADER=${MM_AUTH_DISPLAYNAME_HEADER} + # - MM_AUTH_SURNAME_HEADER=${MM_AUTH_SURNAME_HEADER} + # - MM_AUTH_EMAIL_HEADER=${MM_AUTH_EMAIL_HEADER} + # - MM_AUTH_USER_HEADER=${MM_AUTH_USER_HEADER} + # - KNOWLEDGE_ADDRESS=${KNOWLEDGE_ADDRESS} + # - MAIL_SERVER_HOST=${MAIL_SERVER_HOST} + # - MAIL_SERVER_PORT=${MAIL_SERVER_PORT} + # - MAIL_USER_ADDRESS=${MAIL_USER_ADDRESS} + # - MAIL_USER_PASSWORD=${MAIL_USER_PASSWORD} + # - TERM_OF_SERVICE=${TERM_OF_SERVICE} + # - SWAGGER_CONTACT_NAME=${SWAGGER_CONTACT_NAME} + # - SWAGGER_CONTACT_EMAIL=${SWAGGER_CONTACT_EMAIL} + # - DEV_ENDPOINT=${DEV_ENDPOINT} + # - QA_SERVER_ENDPOINT=${QA_SERVER_ENDPOINT} + # - PROD_SERVER_ENDPOINT=${PROD_SERVER_ENDPOINT} + # - MANAGED_SERVICE_ADDRESS=${MANAGED_SERVICE_ADDRESS} + # - SYSTEM_EMAIL=${SYSTEM_EMAIL} + # - DEPLOYMENT_ADDRESS=${DEPLOYMENT_ADDRESS} + # - DOCKER_HUB_ADDRESS=${DOCKER_HUB_ADDRESS} + # - DEPLOYMENT_SECRET=${DEPLOYMENT_SECRET} + # - DYNAMFIT_TEST_FILE=${DYNAMFIT_TEST_FILE} + # ports: + # - '3001:3001' + # - '3002:3002' + # client: + # stdin_open: true + # container_name: frontend + # environment: + # - CHOKIDAR_USEPOLLING=true + # - API_PORT=${PORT} + # - MINIO_PORT=${MINIO_PORT} + # - SERVICE_PORT=${HOST_PORT} + # build: + # dockerfile: Dockerfile.dev + # context: ./app + # volumes: + # - /app/node_modules + # - ./app:/app managedservices: container_name: managedservices # depends_on: @@ -151,78 +151,6 @@ services: - AUTH_API_TOKEN_EMAIL=${MANAGED_SERVICES_AUTH_API_TOKEN_EMAIL} - AUTH_API_REFRESH_EMAIL=${MANAGED_SERVICES_AUTH_API_REFRESH_EMAIL} - API_URL=${API_URL} -<<<<<<< HEAD -# redis: -# image: 'redis:alpine' -# command: redis-server -# volumes: -# - ./mockDB/redis-data:/var/lib/redis -# celery: -# depends_on: -# - redis -# - fuseki -# volumes: -# - ./mockDB/fuseki:/app/run -# - ./whyis/materialsmine:/app -# command: -# /opt/venv/bin/celery -A wsgi.celery worker -l INFO -c 4 --uid=nobody -# --gid=nogroup -# environment: -# - CHOKIDAR_USEPOLLING=true -# build: whyis -# celerybeat: -# depends_on: -# - redis -# - fuseki -# volumes: -# - ./mockDB/fuseki:/app/run -# - ./whyis/materialsmine:/app -# command: -# /opt/venv/bin/celery -A wsgi.celery beat -l INFO --uid=nobody -# --gid=nogroup -# environment: -# - CHOKIDAR_USEPOLLING=true -# build: whyis -# whyis: -# container_name: whyis -# depends_on: -# - redis -# - fuseki -# stdin_open: true -# command: -# /opt/venv/bin/gunicorn wsgi:application --workers ${WEB_CONCURRENCY:-8} -# --timeout 0 -b :8000 -# environment: -# - CHOKIDAR_USEPOLLING=true -# - NM_GRAPH_AUTH_SECRET=${TKNS} -# build: whyis -# volumes: -# - ./whyis/materialsmine:/app -# - ./mockDB/fuseki:/app/run -# ports: -# - '8000:8000' -# mem_limit: 2048m -# cpus: '0.25' -# fuseki: -# build: whyis -# command: /opt/venv/bin/fuseki-server --mem /ds -# # Needs fuseki 4.3.2 to use our full text search, stain is still at 4.0.0. -# # image: 'stain/jena-fuseki:4.0.0 -# restart: always -# # mem_limit: 9G -# # mem_reservation: 9G -# environment: -# # - ADMIN_USER=admin -# # - ADMIN_PASSWORD=${ADMIN_PASSWORD} -# - JVM_ARGS=-Xmx4g -# - FUSEKI_BASE=/fuseki -# ports: -# - '3030:3030' -# volumes: -# - ./mockDB/fuseki:/fuseki -# mem_limit: 3072m -# cpus: '0.45' -======= - GITHUB_USERNAME=${GITHUB_USERNAME} - GITHUB_TOKEN=${GITHUB_TOKEN} - REPO_OWNER=${REPO_OWNER} @@ -304,7 +232,6 @@ services: - ./mockDB/fuseki:/fuseki mem_limit: 3072m cpus: '0.45' ->>>>>>> 7fce17aac9e8d3c6051f21145da6b33dd8b5cfbe volumes: mockDB: diff --git a/services/app/yaml_converter/routes.py b/services/app/yaml_converter/routes.py index 6da650fe..e572faf0 100644 --- a/services/app/yaml_converter/routes.py +++ b/services/app/yaml_converter/routes.py @@ -3,6 +3,7 @@ import xml.etree.ElementTree as ET from collections import OrderedDict from ruamel.yaml import YAML +from .yaml_utils import validate_control_id # Create a Blueprint named 'yaml_converter' with a URL prefix '/yaml_converter' @@ -23,6 +24,10 @@ def fetch_xml(control_id): external_url = f"https://qa.materialsmine.org/api/xml/{control_id}?format=xml" try: + is_valid, validation_msg = validate_control_id(control_id) + if not is_valid: + return jsonify({"message": validation_msg}), 400 + # Send a GET request to the external URL response = requests.get(external_url) @@ -38,7 +43,7 @@ def fetch_xml(control_id): yaml_data = convert_dict_to_yaml(xml_dict) # Return the YAML data as a response - return Response(yaml_data, mimetype='text/yaml') + return Response(yaml_data, mimetype='text/yaml', content_type='text/yaml; charset=utf-8', status=200) elif response.status_code == 404: # If the resource is not found, return a 404 error message @@ -103,6 +108,7 @@ def convert_dict_to_yaml(data_dict): yaml.default_style = None # Convert OrderedDict to regular dict to avoid !!omap + # TODO (@Jhyang): Use descriptive argument names here, and add type hints. def convert_ordereddict(d): if isinstance(d, OrderedDict): return dict((k, convert_ordereddict(v)) for k, v in d.items()) @@ -117,7 +123,3 @@ def convert_ordereddict(d): yaml_stream = StringIO() yaml.dump(clean_dict, yaml_stream) return yaml_stream.getvalue() - - - - diff --git a/services/app/yaml_converter/yaml_utils.py b/services/app/yaml_converter/yaml_utils.py new file mode 100644 index 00000000..1f89c067 --- /dev/null +++ b/services/app/yaml_converter/yaml_utils.py @@ -0,0 +1,17 @@ +""" +`validate_control_id` function checks if the `control_id` is provided, is a string, and is 24 characters long. +""" +def validate_control_id(control_id): + # Check if control_id is provided + if not control_id: + return False, "control_id is required" + + # Check the type of control_id + if not isinstance(control_id, str): + return False, "control_id must be a string" + + # Check the format of control_id (example: must be alphanumeric and 24 characters long) + if not control_id.isalnum() or len(control_id) != 24: + return False, "control_id must be 8 alphanumeric characters" + + return True, "control_id is valid" \ No newline at end of file diff --git a/services/requirements/prod_requirements.txt b/services/requirements/prod_requirements.txt index 03137fe1..555bf4df 100644 --- a/services/requirements/prod_requirements.txt +++ b/services/requirements/prod_requirements.txt @@ -44,7 +44,4 @@ tzdata==2023.4 urllib3==2.1.0 Werkzeug==3.0.1 xlrd==1.2.0 -ruamel.yaml==0.17.32 - - - +ruamel.yaml==0.17.32 \ No newline at end of file From fea2d0d3674d62f077389da980f29cbae69e4ed4 Mon Sep 17 00:00:00 2001 From: tholulomo Date: Wed, 9 Oct 2024 13:09:05 -0400 Subject: [PATCH 3/9] FEAT(Jhyang): Adding yaml visualizer page --- app/src/pages/explorer/xml/YamlLoader.vue | 210 ++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 app/src/pages/explorer/xml/YamlLoader.vue diff --git a/app/src/pages/explorer/xml/YamlLoader.vue b/app/src/pages/explorer/xml/YamlLoader.vue new file mode 100644 index 00000000..0e703383 --- /dev/null +++ b/app/src/pages/explorer/xml/YamlLoader.vue @@ -0,0 +1,210 @@ + + + From 6bbedf98770c48af1535392a7df4dccacd8a408c Mon Sep 17 00:00:00 2001 From: tholulomo Date: Wed, 9 Oct 2024 13:09:40 -0400 Subject: [PATCH 4/9] FEAT(Jhyang): Linter fixes --- app/src/assets/css/style.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/assets/css/style.scss b/app/src/assets/css/style.scss index a2baba46..e2216553 100644 --- a/app/src/assets/css/style.scss +++ b/app/src/assets/css/style.scss @@ -18,4 +18,4 @@ @import './modules/_vega.scss'; @import './modules/_visualize.scss'; @import './modules/_tools.scss'; -@import './modules/_nuplot.scss'; \ No newline at end of file +@import './modules/_nuplot.scss'; From eb8f011c5593886dfc0195864516e8d9abb10e56 Mon Sep 17 00:00:00 2001 From: tholulomo Date: Wed, 9 Oct 2024 13:11:20 -0400 Subject: [PATCH 5/9] FEAT(Jhyang): Refactor existing xml visualizer with link to yaml --- app/src/components/explorer/XmlView.vue | 14 +-- app/src/pages/explorer/xml/XmlLoader.vue | 127 +++++++++++++---------- 2 files changed, 81 insertions(+), 60 deletions(-) diff --git a/app/src/components/explorer/XmlView.vue b/app/src/components/explorer/XmlView.vue index b558022f..cc5e8d11 100644 --- a/app/src/components/explorer/XmlView.vue +++ b/app/src/components/explorer/XmlView.vue @@ -1,11 +1,11 @@ From e3ba09a20dd4da542f8a192b499d3c6e11dc3e12 Mon Sep 17 00:00:00 2001 From: tholulomo Date: Wed, 9 Oct 2024 13:11:56 -0400 Subject: [PATCH 6/9] FEAT(Jhyang): Adding yaml visualizer route --- app/src/router/module/explorer.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/src/router/module/explorer.js b/app/src/router/module/explorer.js index 399c9a3c..4e579c1b 100644 --- a/app/src/router/module/explorer.js +++ b/app/src/router/module/explorer.js @@ -278,6 +278,12 @@ const explorerRoutes = [ component: () => import('@/pages/explorer/xml/XmlLoader.vue'), meta: { requiresAuth: false } }, + { + path: 'yamlvisualizer/:id', + name: 'YamlVisualizer', + component: () => import('@/pages/explorer/xml/YamlLoader.vue'), + meta: { requiresAuth: false } + }, { path: 'dataset', component: ChartBase, From 295989971cca27208510c7d6325c923502d3df2d Mon Sep 17 00:00:00 2001 From: tholulomo Date: Wed, 9 Oct 2024 13:12:29 -0400 Subject: [PATCH 7/9] FEAT(Jhyang): Enabling all services --- docker-compose.yml | 262 ++++++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 131 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4000d616..5a8beec3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,136 +1,136 @@ version: '3' services: - # mongo: - # image: 'mongo:4.4' - # restart: always - # container_name: mongo - # environment: - # - MONGO_INITDB_ROOT_USERNAME=${MM_MONGO_USER} - # - MONGO_INITDB_ROOT_PASSWORD=${MM_MONGO_PWD} - # - MONGO_DEVS=${MONGO_DEVS} - # - MONGO_DEV_PWD=${MONGO_DEV_PWD} - # - MONGO_PORT=${MONGO_PORT} - # - MM_DB=${MM_DB} - # ports: - # - '27017:27017' - # volumes: - # - ./mockDB/db:/data/db - # - ./mockDB/restore:/restore - # - ./mockDB/log:/var/log/mongodb - # - ./mockDB/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh - # es: - # image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.1' - # container_name: es - # restart: always - # environment: - # - node.name=es - # - bootstrap.memory_lock=true - # - cluster.initial_master_nodes=es - # - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' - # - xpack.security.enabled=false - # ulimits: - # memlock: - # soft: -1 - # hard: -1 - # ports: - # - '9200:9200' - # volumes: - # - ./mockDB/es:/usr/share/elasticsearch/data - # minio: - # container_name: minio - # image: minio/minio - # command: server --console-address ":9001" /data - # environment: - # - MINIO_ROOT_USER=${MINIO_ROOT_USER} - # - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - # ports: - # - '9000:9000' - # - '9001:9001' - # volumes: - # - ./mockDB/minio:/data - # proxy: - # container_name: proxy - # depends_on: - # - api - # - client - # restart: always - # build: - # dockerfile: Dockerfile.dev - # context: ./nginx - # ports: - # - '80:80' - # api: - # depends_on: - # - es - # - mongo - # - minio - # container_name: restful - # restart: always - # build: - # dockerfile: Dockerfile.dev - # context: ./resfulservice - # volumes: - # - /app/node_modules - # - ./resfulservice:/app - # environment: - # - DB_USERNAME=${MONGO_DEVS} - # - DB_PASSWORD=${MONGO_DEV_PWD} - # - MM_DB=${MM_DB} - # - MONGO_ADDRESS=${MONGO_ADDRESS} - # - MONGO_PORT=${MONGO_PORT} - # - TKNS=${TKNS} - # - FILES_DIRECTORY=${FILES_DIRECTORY} - # - ESADDRESS=${ESADDRESS} - # - PORT=${PORT} - # - WORKER_PORT=${WORKER_PORT} - # - MINIO_PORT=${MINIO_PORT} - # - MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT} - # - MINIO_BUCKET=${MINIO_BUCKET} - # - MINIO_ROOT_USER=${MINIO_ROOT_USER} - # - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} - # - ROUTER=${HOST_PORT} - # - MM_RUNTIME_ENV=${MM_RUNTIME_ENV} - # - MM_USER=${MM_USER} - # - MM_USER_EMAIL=${MM_USER_EMAIL} - # - MM_AUTH_GIVEN_NAME_HEADER=${MM_AUTH_GIVEN_NAME_HEADER} - # - MM_AUTH_DISPLAYNAME_HEADER=${MM_AUTH_DISPLAYNAME_HEADER} - # - MM_AUTH_SURNAME_HEADER=${MM_AUTH_SURNAME_HEADER} - # - MM_AUTH_EMAIL_HEADER=${MM_AUTH_EMAIL_HEADER} - # - MM_AUTH_USER_HEADER=${MM_AUTH_USER_HEADER} - # - KNOWLEDGE_ADDRESS=${KNOWLEDGE_ADDRESS} - # - MAIL_SERVER_HOST=${MAIL_SERVER_HOST} - # - MAIL_SERVER_PORT=${MAIL_SERVER_PORT} - # - MAIL_USER_ADDRESS=${MAIL_USER_ADDRESS} - # - MAIL_USER_PASSWORD=${MAIL_USER_PASSWORD} - # - TERM_OF_SERVICE=${TERM_OF_SERVICE} - # - SWAGGER_CONTACT_NAME=${SWAGGER_CONTACT_NAME} - # - SWAGGER_CONTACT_EMAIL=${SWAGGER_CONTACT_EMAIL} - # - DEV_ENDPOINT=${DEV_ENDPOINT} - # - QA_SERVER_ENDPOINT=${QA_SERVER_ENDPOINT} - # - PROD_SERVER_ENDPOINT=${PROD_SERVER_ENDPOINT} - # - MANAGED_SERVICE_ADDRESS=${MANAGED_SERVICE_ADDRESS} - # - SYSTEM_EMAIL=${SYSTEM_EMAIL} - # - DEPLOYMENT_ADDRESS=${DEPLOYMENT_ADDRESS} - # - DOCKER_HUB_ADDRESS=${DOCKER_HUB_ADDRESS} - # - DEPLOYMENT_SECRET=${DEPLOYMENT_SECRET} - # - DYNAMFIT_TEST_FILE=${DYNAMFIT_TEST_FILE} - # ports: - # - '3001:3001' - # - '3002:3002' - # client: - # stdin_open: true - # container_name: frontend - # environment: - # - CHOKIDAR_USEPOLLING=true - # - API_PORT=${PORT} - # - MINIO_PORT=${MINIO_PORT} - # - SERVICE_PORT=${HOST_PORT} - # build: - # dockerfile: Dockerfile.dev - # context: ./app - # volumes: - # - /app/node_modules - # - ./app:/app + mongo: + image: 'mongo:4.4' + restart: always + container_name: mongo + environment: + - MONGO_INITDB_ROOT_USERNAME=${MM_MONGO_USER} + - MONGO_INITDB_ROOT_PASSWORD=${MM_MONGO_PWD} + - MONGO_DEVS=${MONGO_DEVS} + - MONGO_DEV_PWD=${MONGO_DEV_PWD} + - MONGO_PORT=${MONGO_PORT} + - MM_DB=${MM_DB} + ports: + - '27017:27017' + volumes: + - ./mockDB/db:/data/db + - ./mockDB/restore:/restore + - ./mockDB/log:/var/log/mongodb + - ./mockDB/mongo-init.sh:/docker-entrypoint-initdb.d/mongo-init.sh + es: + image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.1' + container_name: es + restart: always + environment: + - node.name=es + - bootstrap.memory_lock=true + - cluster.initial_master_nodes=es + - 'ES_JAVA_OPTS=-Xms512m -Xmx512m' + - xpack.security.enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + ports: + - '9200:9200' + volumes: + - ./mockDB/es:/usr/share/elasticsearch/data + minio: + container_name: minio + image: minio/minio + command: server --console-address ":9001" /data + environment: + - MINIO_ROOT_USER=${MINIO_ROOT_USER} + - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + ports: + - '9000:9000' + - '9001:9001' + volumes: + - ./mockDB/minio:/data + proxy: + container_name: proxy + depends_on: + - api + - client + restart: always + build: + dockerfile: Dockerfile.dev + context: ./nginx + ports: + - '80:80' + api: + depends_on: + - es + - mongo + - minio + container_name: restful + restart: always + build: + dockerfile: Dockerfile.dev + context: ./resfulservice + volumes: + - /app/node_modules + - ./resfulservice:/app + environment: + - DB_USERNAME=${MONGO_DEVS} + - DB_PASSWORD=${MONGO_DEV_PWD} + - MM_DB=${MM_DB} + - MONGO_ADDRESS=${MONGO_ADDRESS} + - MONGO_PORT=${MONGO_PORT} + - TKNS=${TKNS} + - FILES_DIRECTORY=${FILES_DIRECTORY} + - ESADDRESS=${ESADDRESS} + - PORT=${PORT} + - WORKER_PORT=${WORKER_PORT} + - MINIO_PORT=${MINIO_PORT} + - MINIO_CONSOLE_PORT=${MINIO_CONSOLE_PORT} + - MINIO_BUCKET=${MINIO_BUCKET} + - MINIO_ROOT_USER=${MINIO_ROOT_USER} + - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD} + - ROUTER=${HOST_PORT} + - MM_RUNTIME_ENV=${MM_RUNTIME_ENV} + - MM_USER=${MM_USER} + - MM_USER_EMAIL=${MM_USER_EMAIL} + - MM_AUTH_GIVEN_NAME_HEADER=${MM_AUTH_GIVEN_NAME_HEADER} + - MM_AUTH_DISPLAYNAME_HEADER=${MM_AUTH_DISPLAYNAME_HEADER} + - MM_AUTH_SURNAME_HEADER=${MM_AUTH_SURNAME_HEADER} + - MM_AUTH_EMAIL_HEADER=${MM_AUTH_EMAIL_HEADER} + - MM_AUTH_USER_HEADER=${MM_AUTH_USER_HEADER} + - KNOWLEDGE_ADDRESS=${KNOWLEDGE_ADDRESS} + - MAIL_SERVER_HOST=${MAIL_SERVER_HOST} + - MAIL_SERVER_PORT=${MAIL_SERVER_PORT} + - MAIL_USER_ADDRESS=${MAIL_USER_ADDRESS} + - MAIL_USER_PASSWORD=${MAIL_USER_PASSWORD} + - TERM_OF_SERVICE=${TERM_OF_SERVICE} + - SWAGGER_CONTACT_NAME=${SWAGGER_CONTACT_NAME} + - SWAGGER_CONTACT_EMAIL=${SWAGGER_CONTACT_EMAIL} + - DEV_ENDPOINT=${DEV_ENDPOINT} + - QA_SERVER_ENDPOINT=${QA_SERVER_ENDPOINT} + - PROD_SERVER_ENDPOINT=${PROD_SERVER_ENDPOINT} + - MANAGED_SERVICE_ADDRESS=${MANAGED_SERVICE_ADDRESS} + - SYSTEM_EMAIL=${SYSTEM_EMAIL} + - DEPLOYMENT_ADDRESS=${DEPLOYMENT_ADDRESS} + - DOCKER_HUB_ADDRESS=${DOCKER_HUB_ADDRESS} + - DEPLOYMENT_SECRET=${DEPLOYMENT_SECRET} + - DYNAMFIT_TEST_FILE=${DYNAMFIT_TEST_FILE} + ports: + - '3001:3001' + - '3002:3002' + client: + stdin_open: true + container_name: frontend + environment: + - CHOKIDAR_USEPOLLING=true + - API_PORT=${PORT} + - MINIO_PORT=${MINIO_PORT} + - SERVICE_PORT=${HOST_PORT} + build: + dockerfile: Dockerfile.dev + context: ./app + volumes: + - /app/node_modules + - ./app:/app managedservices: container_name: managedservices # depends_on: From 9ab352555f853856acd6519db6f3736ff74725df Mon Sep 17 00:00:00 2001 From: tholulomo Date: Wed, 9 Oct 2024 13:13:00 -0400 Subject: [PATCH 8/9] FEAT(Jhyang): Adding yaml converter to the register --- resfulservice/config/constant.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resfulservice/config/constant.js b/resfulservice/config/constant.js index 115d1bc2..e314144e 100644 --- a/resfulservice/config/constant.js +++ b/resfulservice/config/constant.js @@ -153,7 +153,8 @@ module.exports = { ManagedServiceRegister: { dynamfit: '/dynamfit/extract/', chemprops: '/chemprops/call/', - ontology: '/ontology/extract/' + ontology: '/ontology/extract/', + 'yaml-loader': '/yaml_converter/' }, MGD_SVC_ERR_CODE: { default: { From bd4e0e98f2ffba62aa7158a0be2e6dd25d701c82 Mon Sep 17 00:00:00 2001 From: tholulomo Date: Wed, 9 Oct 2024 13:14:13 -0400 Subject: [PATCH 9/9] FEAT(Jhyang): Handling yaml responses differently --- .../src/controllers/managedServiceController.js | 14 ++++++++------ resfulservice/src/routes/managed-service.js | 2 +- services/app/yaml_converter/yaml_utils.py | 6 +++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/resfulservice/src/controllers/managedServiceController.js b/resfulservice/src/controllers/managedServiceController.js index 2b9cf31e..9126ff97 100644 --- a/resfulservice/src/controllers/managedServiceController.js +++ b/resfulservice/src/controllers/managedServiceController.js @@ -18,11 +18,8 @@ const { validateIsAdmin } = require('../middlewares/validations'); * @returns {*} response */ exports.manageServiceRequest = async (req, res, next) => { - const { - logger, - params: { appName }, - body - } = req; + const { logger, params, body } = req; + const [appName, controlId] = params.appName?.split('/'); logger.info(':::manageServiceRequest Function Entry'); const reqId = uuidv4(); logger.info(`${appName}::${JSON.stringify({ ...body, reqId })}`); @@ -41,7 +38,7 @@ exports.manageServiceRequest = async (req, res, next) => { } req.reqId = reqId; - req.url = `${req.env?.MANAGED_SERVICE_ADDRESS}${ManagedServiceRegister[appName]}`; + if (!controlId) { req.url = `${req.env?.MANAGED_SERVICE_ADDRESS}${ManagedServiceRegister[appName]}`; } else { req.url = `${req.env?.MANAGED_SERVICE_ADDRESS}${ManagedServiceRegister[appName]}${controlId}`; } return await _managedServiceCall(req, res); } catch (error) { const statusCode = error?.response?.status ?? 500; @@ -129,6 +126,11 @@ const _managedServiceCall = async (req, res) => { `mgdsvc.${appName} = (${reqId}) => response.status(${response.status})` ); if (response.status === 200) { + if (response.headers['content-type'] === 'text/yaml; charset=utf-8') { + res.setHeader('Content-Type', response.headers['content-type']); + return res.send(response.data); + // return res.status(200).json({ yamlString: YAML.stringify(yamlString) }); + } const errorObj = {}; if (response.headers.responseid !== reqId) { errorObj.error = { diff --git a/resfulservice/src/routes/managed-service.js b/resfulservice/src/routes/managed-service.js index 87ced044..6b9028bd 100644 --- a/resfulservice/src/routes/managed-service.js +++ b/resfulservice/src/routes/managed-service.js @@ -10,7 +10,7 @@ router .post(isAuth, latencyTimer, manageServiceController.chemPropsSeed); router - .route('/:appName') + .route('/:appName([^/]*)') .get(latencyTimer, manageServiceController.manageServiceRequest) .post(isAuth, latencyTimer, manageServiceController.manageServiceRequest); diff --git a/services/app/yaml_converter/yaml_utils.py b/services/app/yaml_converter/yaml_utils.py index 1f89c067..20442025 100644 --- a/services/app/yaml_converter/yaml_utils.py +++ b/services/app/yaml_converter/yaml_utils.py @@ -10,8 +10,8 @@ def validate_control_id(control_id): if not isinstance(control_id, str): return False, "control_id must be a string" - # Check the format of control_id (example: must be alphanumeric and 24 characters long) - if not control_id.isalnum() or len(control_id) != 24: - return False, "control_id must be 8 alphanumeric characters" + # Check the format of control_id (example: must be alphanumeric and 8 characters long) + if len(control_id) <= 8: + return False, "control_id must be greater than 8 alphanumeric characters" return True, "control_id is valid" \ No newline at end of file