diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 25590aa..f8a0ede 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [0.2.6](changes_0.2.6.md) * [0.2.5](changes_0.2.5.md) * [0.2.4](changes_0.2.4.md) * [0.2.3](changes_0.2.3.md) @@ -14,6 +15,7 @@ --- hidden: --- +changes_0.2.6 changes_0.2.5 changes_0.2.4 changes_0.2.3 diff --git a/doc/changes/changes_0.2.6.md b/doc/changes/changes_0.2.6.md new file mode 100644 index 0000000..23a297d --- /dev/null +++ b/doc/changes/changes_0.2.6.md @@ -0,0 +1,14 @@ +# Exasol Notebook Connector 0.2.6, released T.B.C. + +## Summary + +This release adds the extension wrappers and makes full use of the configuration enumeration. + +## Changes + +* #50: [Add iterable features to the secret store](https://github.com/exasol/notebook-connector/issues/50) +* #52: [Make data conversion utility for the secret store commonly accessible](https://github.com/exasol/notebook-connector/issues/52) +* #55: [Unified language activation SQL command](https://github.com/exasol/notebook-connector/pull/55) +* #56: [Transformers extension wrapper](https://github.com/exasol/notebook-connector/pull/56) +* #47: [Create a Sagemaker Extension wrapper](https://github.com/exasol/notebook-connector/issues/47) +* #60: [Start using the AILabConfig internally](https://github.com/exasol/notebook-connector/issues/60) diff --git a/exasol/ai_lab_config.py b/exasol/ai_lab_config.py index cf253b9..228f62c 100644 --- a/exasol/ai_lab_config.py +++ b/exasol/ai_lab_config.py @@ -1,24 +1,38 @@ -from enum import Enum +from enum import Enum, auto class AILabConfig(Enum): - use_itde = "USE_ITDE" - db_host_name = "EXTERNAL_HOST_NAME" - db_port = "DB_PORT" - db_schema = "SCHEMA" - db_user = "USER" - db_password = "PASSWORD" - db_encryption = "ENCRYPTION" - cert_vld = "CERTIFICATE_VALIDATION" - trusted_ca = "TRUSTED_CA" - client_cert = "CLIENT_CERTIFICATE" - client_key = "PRIVATE_KEY" - bfs_host_name = "BUCKETFS_HOST_NAME" - bfs_port = "BUCKETFS_PORT" - bfs_service = "BUCKETFS_SERVICE" - bfs_bucket = "BUCKETFS_BUCKET" - bfs_user = "BUCKETFS_USER" - bfs_password = "BUCKETFS_PASSWORD" - bfs_encryption = "BUCKETFS_ENCRYPTION" - mem_size = "MEMORY_SIZE" - disk_size = "DISK_SIZE" + use_itde = auto() + db_host_name = auto() + db_port = auto() + db_schema = auto() + db_user = auto() + db_password = auto() + db_encryption = auto() + cert_vld = auto() + trusted_ca = auto() + client_cert = auto() + client_key = auto() + bfs_host_name = auto() + bfs_port = auto() + bfs_service = auto() + bfs_bucket = auto() + bfs_user = auto() + bfs_password = auto() + bfs_encryption = auto() + mem_size = auto() + disk_size = auto() + huggingface_token = auto() + aws_bucket = auto() + aws_region = auto() + aws_access_key_id = auto() + aws_secret_access_key = auto() + aws_role = auto() + itde_container = auto() + itde_volume = auto() + itde_network = auto() + te_bfs_connection = auto() + te_models_bfs_dir = auto() + te_hf_connection = auto() + te_models_cache_dir = auto() + sme_aws_connection = auto() diff --git a/exasol/connections.py b/exasol/connections.py index cbb7c4a..2fcb3e5 100644 --- a/exasol/connections.py +++ b/exasol/connections.py @@ -11,9 +11,10 @@ import exasol.bucketfs as bfs # type: ignore from exasol.secret_store import Secrets from exasol.utils import optional_str_to_bool +from exasol.ai_lab_config import AILabConfig as CKey -def _optional_encryption(conf: Secrets, key: str = "ENCRYPTION") -> Optional[bool]: +def _optional_encryption(conf: Secrets, key: CKey = CKey.db_encryption) -> Optional[bool]: return optional_str_to_bool(conf.get(key)) @@ -26,14 +27,14 @@ def _extract_ssl_options(conf: Secrets) -> dict: sslopt: dict[str, object] = {} # Is server certificate validation required? - certificate_validation = optional_str_to_bool(conf.get("CERTIFICATE_VALIDATION")) + certificate_validation = optional_str_to_bool(conf.get(CKey.cert_vld)) if certificate_validation is not None: sslopt["cert_reqs"] = ( ssl.CERT_REQUIRED if certificate_validation else ssl.CERT_NONE ) # Is a bundle with trusted CAs provided? - trusted_ca = conf.get("TRUSTED_CA") + trusted_ca = conf.get(CKey.trusted_ca) if trusted_ca: trusted_ca_path = Path(trusted_ca) if trusted_ca_path.is_dir(): @@ -44,12 +45,12 @@ def _extract_ssl_options(conf: Secrets) -> dict: raise ValueError(f"Trusted CA location {trusted_ca} doesn't exist.") # Is client's own certificate provided? - client_certificate = conf.get("CLIENT_CERTIFICATE") + client_certificate = conf.get(CKey.client_cert) if client_certificate: if not Path(client_certificate).is_file(): raise ValueError(f"Certificate file {client_certificate} doesn't exist.") sslopt["certfile"] = client_certificate - private_key = conf.get("PRIVATE_KEY") + private_key = conf.get(CKey.client_key) if private_key: if not Path(private_key).is_file(): raise ValueError(f"Private key file {private_key} doesn't exist.") @@ -60,7 +61,7 @@ def _extract_ssl_options(conf: Secrets) -> dict: def get_external_host(conf: Secrets) -> str: """Constructs the host part of a DB URL using provided configuration parameters.""" - return f"{conf.EXTERNAL_HOST_NAME}:{conf.DB_PORT}" + return f"{conf.get(CKey.db_host_name)}:{conf.get(CKey.db_port)}" def get_udf_bucket_path(conf: Secrets) -> str: @@ -68,7 +69,7 @@ def get_udf_bucket_path(conf: Secrets) -> str: Builds the path of the BucketFS bucket specified in the configuration, as it's seen in the udf's file system. """ - return f"/buckets/{conf.BUCKETFS_SERVICE}/{conf.BUCKETFS_BUCKET}" + return f"/buckets/{conf.get(CKey.bfs_service)}/{conf.get(CKey.bfs_bucket)}" def open_pyexasol_connection(conf: Secrets, **kwargs) -> pyexasol.ExaConnection: @@ -80,19 +81,19 @@ def open_pyexasol_connection(conf: Secrets, **kwargs) -> pyexasol.ExaConnection: Parameters in kwargs override the correspondent values in the configuration. The configuration should provide the following parameters: - - Server address and port (EXTERNAL_HOST_NAME, DB_PORT), - - Client security credentials (USER, PASSWORD). + - Server address and port (db_host_name, db_port), + - Client security credentials (db_user, db_password). Optional parameters include: - - Secured comm flag (ENCRYPTION), - - Some of the SSL options (CERTIFICATE_VALIDATION, TRUSTED_CA, CLIENT_CERTIFICATE). + - Secured comm flag (db_encryption), + - Some of the SSL options (cert_vld, trusted_ca, client_cert). If the schema is not provided then it should be set explicitly in every SQL statement. For other optional parameters the default settings are as per the pyexasol interface. """ conn_params: dict[str, Any] = { "dsn": get_external_host(conf), - "user": conf.USER, - "password": conf.PASSWORD, + "user": conf.get(CKey.db_user), + "password": conf.get(CKey.db_password), } encryption = _optional_encryption(conf) @@ -113,11 +114,11 @@ def open_sqlalchemy_connection(conf: Secrets): Does NOT set the default schema, even if it is defined in the configuration. The configuration should provide the following parameters: - - Server address and port (EXTERNAL_HOST_NAME, DB_PORT), - - Client security credentials (USER, PASSWORD). + - Server address and port (db_host_name, db_port), + - Client security credentials (db_user, db_password). Optional parameters include: - - Secured comm flag (ENCRYPTION). - - Validation of the server's TLS/SSL certificate by the client (CERTIFICATE_VALIDATION). + - Secured comm flag (db_encryption). + - Validation of the server's TLS/SSL certificate by the client (cert_vld). If the schema is not provided then it should be set explicitly in every SQL statement. For other optional parameters the default settings are as per the Exasol SQLAlchemy interface. Currently, it's not possible to use a bundle of trusted CAs other than the default. Neither @@ -125,7 +126,7 @@ def open_sqlalchemy_connection(conf: Secrets): """ websocket_url = ( - f"exa+websocket://{conf.USER}:{conf.PASSWORD}@{get_external_host(conf)}" + f"exa+websocket://{conf.get(CKey.db_user)}:{conf.get(CKey.db_password)}@{get_external_host(conf)}" ) delimiter = "?" @@ -149,11 +150,11 @@ def open_bucketfs_connection(conf: Secrets) -> bfs.Bucket: Returns the Bucket object for the bucket selected in the configuration. The configuration should provide the following parameters; - - Host name and port of the BucketFS service (EXTERNAL_HOST_NAME, BUCKETFS_PORT), - - Client security credentials (BUCKETFS_USER, BUCKETFS_PASSWORD). - - Bucket name (BUCKETFS_BUCKET) + - Host name and port of the BucketFS service (bfs_host_name or db_host_name, bfs_port), + - Client security credentials (bfs_user, bfs_password). + - Bucket name (bfs_bucket) Optional parameters include: - - Secured comm flag (ENCRYPTION), defaults to False. + - Secured comm flag (bfs_encryption), defaults to False. Currently, it's not possible to set any of the TLS/SSL parameters. If secured comm is selected it automatically sets the certificate validation on. """ @@ -161,17 +162,18 @@ def open_bucketfs_connection(conf: Secrets) -> bfs.Bucket: # Set up the connection parameters. # For now, just use the http. Once the exasol.bucketfs is capable of using the # https without validating the server certificate choose between the http and - # https depending on the ENCRYPTION setting like in the code below: - # buckfs_url_prefix = "https" if _optional_encryption(conf) else "http" + # https depending on the bfs_encryption setting like in the code below: + # buckfs_url_prefix = "https" if _optional_encryption(conf, CKey.bfs_encryption) else "http" buckfs_url_prefix = "http" - buckfs_url = f"{buckfs_url_prefix}://{conf.EXTERNAL_HOST_NAME}:{conf.BUCKETFS_PORT}" + buckfs_host = conf.get(CKey.bfs_host_name, conf.get(CKey.db_host_name)) + buckfs_url = f"{buckfs_url_prefix}://{buckfs_host}:{conf.get(CKey.bfs_port)}" buckfs_credentials = { - conf.BUCKETFS_BUCKET: { - "username": conf.BUCKETFS_USER, - "password": conf.BUCKETFS_PASSWORD, + conf.get(CKey.bfs_bucket): { + "username": conf.get(CKey.bfs_user), + "password": conf.get(CKey.bfs_password), } } # Connect to the BucketFS service and navigate to the bucket of choice. bucketfs = bfs.Service(buckfs_url, buckfs_credentials) - return bucketfs[conf.BUCKETFS_BUCKET] + return bucketfs[conf.get(CKey.bfs_bucket)] diff --git a/exasol/extension_wrapper_common.py b/exasol/extension_wrapper_common.py index cf32ab8..1ae070e 100644 --- a/exasol/extension_wrapper_common.py +++ b/exasol/extension_wrapper_common.py @@ -1,9 +1,10 @@ from exasol.connections import open_pyexasol_connection from exasol.secret_store import Secrets from exasol.utils import optional_str_to_bool +from exasol.ai_lab_config import AILabConfig as CKey -def str_to_bool(conf: Secrets, key: str, default_value: bool) -> bool: +def str_to_bool(conf: Secrets, key: CKey, default_value: bool) -> bool: """ Tries to read a binary (i.e. yes/no) value from the secret store. If found returns the correspondent boolean. Otherwise, returns the provided default @@ -31,9 +32,9 @@ def encapsulate_bucketfs_credentials( Parameters: conf: The secret store. The store must hold the bucket-fs service - parameters (BUCKETFS_HOST_NAME or EXTERNAL_HOST_NAME, BUCKETFS_PORT, - BUCKETFS_SERVICE), the access credentials (BUCKETFS_USER, - BUCKETFS_PASSWORD), and the bucket name (BUCKETFS_BUCKET), as well + parameters (bfs_host_name or db_host_name, bfs_port, + bfs_service), the access credentials (bfs_user, + bfs_password), and the bucket name (bfs_bucket), as well as the DB connection parameters. path_in_bucket: Path identifying a location in the bucket. @@ -41,16 +42,16 @@ def encapsulate_bucketfs_credentials( Name for the connection object to be created. """ - bfs_host = conf.get("BUCKETFS_HOST_NAME", conf.EXTERNAL_HOST_NAME) + bfs_host = conf.get(CKey.bfs_host_name, conf.get(CKey.db_host_name)) # For now, just use the http. Once the exasol.bucketfs is capable of using # the https without validating the server certificate choose between the - # http and https depending on the BUCKETFS_ENCRYPTION setting, like this: - # bfs_protocol = "https" if str_to_bool(conf, 'BUCKETFS_ENCRYPTION', True) + # http and https depending on the bfs_encryption setting, like this: + # bfs_protocol = "https" if str_to_bool(conf, CKey.bfs_encryption, True) # else "http" bfs_protocol = "http" bfs_dest = ( - f"{bfs_protocol}://{bfs_host}:{conf.BUCKETFS_PORT}/" - f"{conf.BUCKETFS_BUCKET}/{path_in_bucket};{conf.BUCKETFS_SERVICE}" + f"{bfs_protocol}://{bfs_host}:{conf.get(CKey.bfs_port)}/" + f"{conf.get(CKey.bfs_bucket)}/{path_in_bucket};{conf.get(CKey.bfs_service)}" ) sql = f""" @@ -60,8 +61,8 @@ def encapsulate_bucketfs_credentials( IDENTIFIED BY {{BUCKETFS_PASSWORD!s}} """ query_params = { - "BUCKETFS_USER": conf.BUCKETFS_USER, - "BUCKETFS_PASSWORD": conf.BUCKETFS_PASSWORD, + "BUCKETFS_USER": conf.get(CKey.bfs_user), + "BUCKETFS_PASSWORD": conf.get(CKey.bfs_password), } with open_pyexasol_connection(conf, compression=True) as conn: conn.execute(query=sql, query_params=query_params) @@ -73,7 +74,7 @@ def encapsulate_huggingface_token(conf: Secrets, connection_name: str) -> None: Parameters: conf: - The secret store. The store must hold the Huggingface token (HF_TOKEN), + The secret store. The store must hold the Huggingface token (huggingface_token), as well as the DB connection parameters. connection_name: Name for the connection object to be created. @@ -84,7 +85,7 @@ def encapsulate_huggingface_token(conf: Secrets, connection_name: str) -> None: TO '' IDENTIFIED BY {{TOKEN!s}} """ - query_params = {"TOKEN": conf.HF_TOKEN} + query_params = {"TOKEN": conf.get(CKey.huggingface_token)} with open_pyexasol_connection(conf, compression=True) as conn: conn.execute(query=sql, query_params=query_params) @@ -97,21 +98,21 @@ def encapsulate_aws_credentials(conf: Secrets, connection_name: str) -> None: Parameters: conf: The secret store. The store must hold the S3 bucket parameters - (AWS_BUCKET, AWS_REGION) and AWS access credentials (AWS_ACCESS_KEY_ID, - AWS_SECRET_ACCESS_KEY), as well as the DB connection parameters. + (aws_bucket, aws_region) and AWS access credentials (aws_access_key_id, + aws_secret_access_key), as well as the DB connection parameters. connection_name: Name for the connection object to be created. """ sql = f""" CREATE OR REPLACE CONNECTION [{connection_name}] - TO 'https://{conf.AWS_BUCKET}.s3.{conf.AWS_REGION}.amazonaws.com/' - USER {{AWS_ACCESS_KEY_ID!s}} - IDENTIFIED BY {{AWS_SECRET_ACCESS_KEY!s}} + TO 'https://{conf.get(CKey.aws_bucket)}.s3.{conf.get(CKey.aws_region)}.amazonaws.com/' + USER {{ACCESS_ID!s}} + IDENTIFIED BY {{SECRET_KEY!s}} """ query_params = { - "AWS_ACCESS_KEY_ID": conf.AWS_ACCESS_KEY_ID, - "AWS_SECRET_ACCESS_KEY": conf.AWS_SECRET_ACCESS_KEY, + "ACCESS_ID": conf.get(CKey.aws_access_key_id), + "SECRET_KEY": conf.get(CKey.aws_secret_access_key), } with open_pyexasol_connection(conf, compression=True) as conn: conn.execute(query=sql, query_params=query_params) diff --git a/exasol/itde_manager.py b/exasol/itde_manager.py index e3a6e22..61580ce 100644 --- a/exasol/itde_manager.py +++ b/exasol/itde_manager.py @@ -18,10 +18,6 @@ ENVIRONMENT_NAME = "DemoDb" NAME_SERVER_ADDRESS = "8.8.8.8" -CONTAINER_NAME_KEY = "ITDE_CONTAINER_NAME" -VOLUME_NAME_KEY = "ITDE_VOLUME_NAME" -NETWORK_NAME_KEY = "ITDE_NETWORK_NAME" - def bring_itde_up(conf: Secrets) -> None: """ @@ -44,8 +40,8 @@ def bring_itde_up(conf: Secrets) -> None: bucket-fs connection parameters, in the secret store. """ - mem_size = f'{conf.get(AILabConfig.mem_size.value, "4")} GiB' - disk_size = f'{conf.get(AILabConfig.disk_size.value, "10")} GiB' + mem_size = f'{conf.get(AILabConfig.mem_size, "4")} GiB' + disk_size = f'{conf.get(AILabConfig.disk_size, "10")} GiB' env_info, _ = api.spawn_test_environment( environment_name=ENVIRONMENT_NAME, @@ -57,26 +53,26 @@ def bring_itde_up(conf: Secrets) -> None: db_info = env_info.database_info container_info = db_info.container_info - conf.save(CONTAINER_NAME_KEY, container_info.container_name) - conf.save(VOLUME_NAME_KEY, container_info.volume_name) - conf.save(NETWORK_NAME_KEY, env_info.network_info.network_name) + conf.save(AILabConfig.itde_container, container_info.container_name) + conf.save(AILabConfig.itde_volume, container_info.volume_name) + conf.save(AILabConfig.itde_network, env_info.network_info.network_name) - conf.save(AILabConfig.db_host_name.value, db_info.host) - conf.save(AILabConfig.bfs_host_name.value, db_info.host) - conf.save(AILabConfig.db_port.value, str(db_info.ports.database)) - conf.save(AILabConfig.bfs_port.value, str(db_info.ports.bucketfs)) + conf.save(AILabConfig.db_host_name, db_info.host) + conf.save(AILabConfig.bfs_host_name, db_info.host) + conf.save(AILabConfig.db_port, str(db_info.ports.database)) + conf.save(AILabConfig.bfs_port, str(db_info.ports.bucketfs)) # Q. Can we draw any of the below constants from the ITDE configuration? - conf.save(AILabConfig.db_user.value, "sys") - conf.save(AILabConfig.db_password.value, "exasol") - conf.save(AILabConfig.bfs_user.value, "w") - conf.save(AILabConfig.bfs_password.value, "write") - conf.save(AILabConfig.bfs_service.value, "bfsdefault") - conf.save(AILabConfig.bfs_bucket.value, "default") - conf.save(AILabConfig.db_encryption.value, "True") + conf.save(AILabConfig.db_user, "sys") + conf.save(AILabConfig.db_password, "exasol") + conf.save(AILabConfig.bfs_user, "w") + conf.save(AILabConfig.bfs_password, "write") + conf.save(AILabConfig.bfs_service, "bfsdefault") + conf.save(AILabConfig.bfs_bucket, "default") + conf.save(AILabConfig.db_encryption, "True") # The bucket-fs encryption is turned off temporarily. - conf.save(AILabConfig.bfs_encryption.value, "False") - conf.save(AILabConfig.cert_vld.value, "False") + conf.save(AILabConfig.bfs_encryption, "False") + conf.save(AILabConfig.cert_vld, "False") def is_itde_running(conf: Secrets) -> bool: @@ -86,7 +82,7 @@ def is_itde_running(conf: Secrets) -> bool: If the name cannot be found in the secret store the function returns False. """ - container_name = conf.get(CONTAINER_NAME_KEY) + container_name = conf.get(AILabConfig.itde_container) if not container_name: return False @@ -106,37 +102,37 @@ def take_itde_down(conf: Secrets) -> None: remove_volume(conf) remove_network(conf) - conf.remove(AILabConfig.db_host_name.value) - conf.remove(AILabConfig.bfs_host_name.value) - conf.remove(AILabConfig.db_port.value) - conf.remove(AILabConfig.bfs_port.value) - conf.remove(AILabConfig.db_user.value) - conf.remove(AILabConfig.db_password.value) - conf.remove(AILabConfig.bfs_user.value) - conf.remove(AILabConfig.bfs_password.value) - conf.remove(AILabConfig.bfs_service.value) - conf.remove(AILabConfig.bfs_bucket.value) - conf.remove(AILabConfig.db_encryption.value) - conf.remove(AILabConfig.bfs_encryption.value) - conf.remove(AILabConfig.cert_vld.value) + conf.remove(AILabConfig.db_host_name) + conf.remove(AILabConfig.bfs_host_name) + conf.remove(AILabConfig.db_port) + conf.remove(AILabConfig.bfs_port) + conf.remove(AILabConfig.db_user) + conf.remove(AILabConfig.db_password) + conf.remove(AILabConfig.bfs_user) + conf.remove(AILabConfig.bfs_password) + conf.remove(AILabConfig.bfs_service) + conf.remove(AILabConfig.bfs_bucket) + conf.remove(AILabConfig.db_encryption) + conf.remove(AILabConfig.bfs_encryption) + conf.remove(AILabConfig.cert_vld) def remove_network(conf): - network_name = conf.get(NETWORK_NAME_KEY) + network_name = conf.get(AILabConfig.itde_network) if network_name: remove_docker_networks(iter([network_name])) - conf.remove(NETWORK_NAME_KEY) + conf.remove(AILabConfig.itde_network) def remove_volume(conf): - volume_name = conf.get(VOLUME_NAME_KEY) + volume_name = conf.get(AILabConfig.itde_volume) if volume_name: remove_docker_volumes([volume_name]) - conf.remove(VOLUME_NAME_KEY) + conf.remove(AILabConfig.itde_volume) def remove_container(conf): - container_name = conf.get(CONTAINER_NAME_KEY) + container_name = conf.get(AILabConfig.itde_container) if container_name: remove_docker_container([container_name]) - conf.remove(CONTAINER_NAME_KEY) + conf.remove(AILabConfig.itde_container) diff --git a/exasol/sagemaker_extension_wrapper.py b/exasol/sagemaker_extension_wrapper.py index 463ad61..62c0fa5 100644 --- a/exasol/sagemaker_extension_wrapper.py +++ b/exasol/sagemaker_extension_wrapper.py @@ -15,6 +15,7 @@ get_activation_sql ) from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig as CKey # Root directory in a bucket-fs bucket where all stuff of the Sagemaker # Extension, including its language container, will be uploaded. @@ -26,10 +27,6 @@ # store with this key. ACTIVATION_KEY = ACTIVATION_KEY_PREFIX + "sme" -# Name of the connection object with AWS credentials and S3 location -# will be saved in the secret store with this key. -AWS_CONNECTION_KEY = "SME_AWS_CONN" - # Name of the connection object with AWS credentials and S3 location # will be prefixed with this string. AWS_CONNECTION_PREFIX = "SME_AWS" @@ -59,20 +56,20 @@ def deploy_language_container(conf: Secrets, version: str) -> None: deployer = SmeLanguageContainerDeployer.create( dsn=get_external_host(conf), - db_user=conf.USER, - db_password=conf.PASSWORD, - bucketfs_name=conf.BUCKETFS_SERVICE, - bucketfs_host=conf.get("BUCKETFS_HOST_NAME", conf.EXTERNAL_HOST_NAME), - bucketfs_port=int(conf.BUCKETFS_PORT), - bucketfs_user=conf.BUCKETFS_USER, - bucketfs_password=conf.BUCKETFS_PASSWORD, - bucketfs_use_https=str_to_bool(conf, "BUCKETFS_ENCRYPTION", True), - bucket=conf.BUCKETFS_BUCKET, + db_user=conf.get(CKey.db_user), + db_password=conf.get(CKey.db_password), + bucketfs_name=conf.get(CKey.bfs_service), + bucketfs_host=conf.get(CKey.bfs_host_name, conf.get(CKey.db_host_name)), + bucketfs_port=int(str(conf.get(CKey.bfs_port))), + bucketfs_user=conf.get(CKey.bfs_user), + bucketfs_password=conf.get(CKey.bfs_password), + bucketfs_use_https=str_to_bool(conf, CKey.bfs_encryption, True), + bucket=conf.get(CKey.bfs_bucket), path_in_bucket=PATH_IN_BUCKET, - use_ssl_cert_validation=str_to_bool(conf, "CERTIFICATE_VALIDATION", True), - ssl_trusted_ca=conf.get("TRUSTED_CA"), - ssl_client_certificate=conf.get("CLIENT_CERTIFICATE"), - ssl_private_key=conf.get("PRIVATE_KEY"), + use_ssl_cert_validation=str_to_bool(conf, CKey.cert_vld, True), + ssl_trusted_ca=conf.get(CKey.trusted_ca), + ssl_client_certificate=conf.get(CKey.client_cert), + ssl_private_key=conf.get(CKey.client_key), ) # Install the language container. @@ -102,7 +99,7 @@ def deploy_scripts(conf: Secrets) -> None: conn.execute(activation_sql) scripts_deployer = DeployCreateStatements( - exasol_conn=conn, schema=conf.SCHEMA, to_print=False, develop=False + exasol_conn=conn, schema=conf.get(CKey.db_schema), to_print=False, develop=False ) scripts_deployer.run() @@ -134,7 +131,7 @@ def initialize_sme_extension(conf: Secrets, """ # Make the connection object name - aws_conn_name = "_".join([AWS_CONNECTION_PREFIX, conf.USER]) + aws_conn_name = "_".join([AWS_CONNECTION_PREFIX, str(conf.get(CKey.db_user))]) if run_deploy_container: deploy_language_container(conf, version) @@ -146,4 +143,4 @@ def initialize_sme_extension(conf: Secrets, encapsulate_aws_credentials(conf, aws_conn_name) # Save the connection object name in the secret store. - conf.save(AWS_CONNECTION_KEY, aws_conn_name) + conf.save(CKey.sme_aws_connection, aws_conn_name) diff --git a/exasol/secret_store.py b/exasol/secret_store.py index 02de111..b0d2686 100644 --- a/exasol/secret_store.py +++ b/exasol/secret_store.py @@ -6,10 +6,13 @@ Iterable, Optional, Tuple, + Union ) from sqlcipher3 import dbapi2 as sqlcipher # type: ignore +from exasol.ai_lab_config import AILabConfig as CKey + _logger = logging.getLogger(__name__) TABLE_NAME = "secrets" @@ -97,8 +100,9 @@ def _cursor( finally: cur.close() - def save(self, key: str, value: str) -> "Secrets": + def save(self, key: Union[str, CKey], value: str) -> "Secrets": """key represents a system, service, or application""" + key = key.name if isinstance(key, CKey) else key def entry_exists(cur) -> bool: res = cur.execute(f"SELECT * FROM {TABLE_NAME} WHERE key=?", [key]) @@ -119,7 +123,10 @@ def insert(cur) -> None: insert(cur) return self - def get(self, key: str, default_value: Optional[str] = None) -> Optional[str]: + def get(self, key: Union[str, CKey], default_value: Optional[str] = None) -> Optional[str]: + + key = key.name if isinstance(key, CKey) else key + with self._cursor() as cur: res = cur.execute(f"SELECT value FROM {TABLE_NAME} WHERE key=?", [key]) row = res.fetchone() if res else None @@ -152,10 +159,12 @@ def items(self) -> Iterable[Tuple[str, str]]: for row in res: yield row[0], row[1] - def remove(self, key: str) -> None: + def remove(self, key: Union[str, CKey]) -> None: """ Deletes entry with the specified key if it exists. Doesn't raise any exception if the key doesn't exist. """ + key = key.name if isinstance(key, CKey) else key + with self._cursor() as cur: cur.execute(f"DELETE FROM {TABLE_NAME} WHERE key=?", [key]) diff --git a/exasol/transformers_extension_wrapper.py b/exasol/transformers_extension_wrapper.py index 4550189..13cc258 100644 --- a/exasol/transformers_extension_wrapper.py +++ b/exasol/transformers_extension_wrapper.py @@ -22,6 +22,7 @@ get_activation_sql ) from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig as CKey # Root directory in a bucket-fs bucket where all stuff of the Transformers # Extension, including its language container, will be uploaded. @@ -35,10 +36,6 @@ # store with this key. ACTIVATION_KEY = ACTIVATION_KEY_PREFIX + "te" -# The name of the connection object with bucket-fs location and credentials -# will be saved in the secret store with this key. -BFS_CONNECTION_KEY = "TE_BFS_CONN" - # The name of the connection object with bucket-fs location and credentials # will be prefixed with this string. BFS_CONNECTION_PREFIX = "TE_BFS" @@ -46,14 +43,6 @@ # Models will be uploaded into this directory in bucket-fs. BFS_MODELS_DIR = 'te_models' -# The name of the models' directory in bucket-fs will be saved in the secret -# store with this key. -BFS_MODELS_DIR_KEY = "TE_MODELS_BFS_DIR" - -# The name of the connection object with a Huggingface token will be saved in -# the secret store with this key. -HF_CONNECTION_KEY = "TE_TOKEN_CONN" - # The name of the connection object with a Huggingface token will be prefixed # with this string. HF_CONNECTION_PREFIX = "TE_HF" @@ -62,10 +51,6 @@ # cached in this directory. MODELS_CACHE_DIR = "models_cache" -# The name of the models' cache directory will be saved in the secret store -# with this key. -MODELS_CACHE_DIR_KEY = "TE_MODELS_CACHE_DIR" - def deploy_language_container(conf: Secrets, version: str, @@ -95,21 +80,21 @@ def deploy_language_container(conf: Secrets, deployer = TeLanguageContainerDeployer.create( dsn=get_external_host(conf), - db_user=conf.USER, - db_password=conf.PASSWORD, - bucketfs_name=conf.BUCKETFS_SERVICE, - bucketfs_host=conf.get("BUCKETFS_HOST_NAME", conf.EXTERNAL_HOST_NAME), - bucketfs_port=int(conf.BUCKETFS_PORT), - bucketfs_user=conf.BUCKETFS_USER, - bucketfs_password=conf.BUCKETFS_PASSWORD, - bucketfs_use_https=str_to_bool(conf, "BUCKETFS_ENCRYPTION", True), - bucket=conf.BUCKETFS_BUCKET, + db_user=conf.get(CKey.db_user), + db_password=conf.get(CKey.db_password), + bucketfs_name=conf.get(CKey.bfs_service), + bucketfs_host=conf.get(CKey.bfs_host_name, conf.get(CKey.db_host_name)), + bucketfs_port=int(str(conf.get(CKey.bfs_port))), + bucketfs_user=conf.get(CKey.bfs_user), + bucketfs_password=conf.get(CKey.bfs_password), + bucketfs_use_https=str_to_bool(conf, CKey.bfs_encryption, True), + bucket=conf.get(CKey.bfs_bucket), path_in_bucket=PATH_IN_BUCKET, language_alias=language_alias, - use_ssl_cert_validation=str_to_bool(conf, "CERTIFICATE_VALIDATION", True), - ssl_trusted_ca=conf.get("TRUSTED_CA"), - ssl_client_certificate=conf.get("CLIENT_CERTIFICATE"), - ssl_private_key=conf.get("PRIVATE_KEY"), + use_ssl_cert_validation=str_to_bool(conf, CKey.cert_vld, True), + ssl_trusted_ca=conf.get(CKey.trusted_ca), + ssl_client_certificate=conf.get(CKey.client_cert), + ssl_private_key=conf.get(CKey.client_key) ) # Install the language container. @@ -141,7 +126,7 @@ def deploy_scripts(conf: Secrets, activation_sql = get_activation_sql(conf) conn.execute(activation_sql) - scripts_deployer = ScriptsDeployer(language_alias, conf.SCHEMA, conn) + scripts_deployer = ScriptsDeployer(language_alias, conf.get(CKey.db_schema), conn) scripts_deployer.deploy_scripts() @@ -180,9 +165,10 @@ def initialize_te_extension(conf: Secrets, """ # Make the connection object names - bfs_conn_name = "_".join([BFS_CONNECTION_PREFIX, conf.USER]) - token = conf.get("HF_TOKEN") - hf_conn_name = "_".join([HF_CONNECTION_PREFIX, conf.USER]) if token else "" + db_user = str(conf.get(CKey.db_user)) + bfs_conn_name = "_".join([BFS_CONNECTION_PREFIX, db_user]) + token = conf.get(CKey.huggingface_token) + hf_conn_name = "_".join([HF_CONNECTION_PREFIX, db_user]) if token else "" if run_deploy_container: deploy_language_container(conf, version, language_alias) @@ -198,11 +184,11 @@ def initialize_te_extension(conf: Secrets, encapsulate_huggingface_token(conf, hf_conn_name) # Save the connection object names in the secret store. - conf.save(BFS_CONNECTION_KEY, bfs_conn_name) - conf.save(HF_CONNECTION_KEY, hf_conn_name) + conf.save(CKey.te_bfs_connection, bfs_conn_name) + conf.save(CKey.te_hf_connection, hf_conn_name) # Save the directory names in the secret store - conf.save(BFS_MODELS_DIR_KEY, BFS_MODELS_DIR) - conf.save(MODELS_CACHE_DIR_KEY, MODELS_CACHE_DIR) + conf.save(CKey.te_models_bfs_dir, BFS_MODELS_DIR) + conf.save(CKey.te_models_cache_dir, MODELS_CACHE_DIR) def upload_model_from_cache( @@ -228,13 +214,15 @@ def upload_model_from_cache( """ # Create bucketfs location + bfs_host = conf.get(CKey.bfs_host_name, conf.get(CKey.db_host_name)) bucketfs_location = create_bucketfs_location( - conf.BUCKETFS_SERVICE, conf.get('BUCKETFS_HOST_NAME', conf.EXTERNAL_HOST_NAME), - int(conf.BUCKETFS_PORT), conf.BUCKETFS_ENCRYPTION.lower() == 'true', - conf.BUCKETFS_USER, conf.BUCKETFS_PASSWORD, conf.BUCKETFS_BUCKET, PATH_IN_BUCKET) + conf.get(CKey.bfs_service), bfs_host, + int(str(conf.get(CKey.bfs_port))), str(conf.get(CKey.bfs_encryption)).lower() == 'true', + conf.get(CKey.bfs_user), conf.get(CKey.bfs_password), conf.get(CKey.bfs_bucket), + PATH_IN_BUCKET) # Upload the downloaded model files into bucketfs - upload_path = get_model_path(conf.get(BFS_MODELS_DIR_KEY), model_name) + upload_path = get_model_path(conf.get(CKey.te_models_bfs_dir), model_name) upload_model_files_to_bucketfs(cache_dir, upload_path, bucketfs_location) @@ -264,7 +252,7 @@ def upload_model( from transformers import AutoTokenizer, AutoModel # type: ignore if 'token' not in kwargs: - token = conf.HF_TOKEN + token = conf.get(CKey.huggingface_token) if token: kwargs['token'] = token diff --git a/pyproject.toml b/pyproject.toml index 972e13c..48c7d7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "exasol-notebook-connector" -version = "0.2.5" +version = "0.2.6" description = "Components, tools, APIs, and configurations in order to connect Jupyter notebooks to Exasol and various other systems." packages = [ {include = "exasol"}, ] authors = [ "Christoph Kuhnke " ] diff --git a/test/integration/test_itde_manager.py b/test/integration/test_itde_manager.py index 50bd143..08f9582 100644 --- a/test/integration/test_itde_manager.py +++ b/test/integration/test_itde_manager.py @@ -10,9 +10,6 @@ from exasol.ai_lab_config import AILabConfig from exasol.itde_manager import ( - CONTAINER_NAME_KEY, - NETWORK_NAME_KEY, - VOLUME_NAME_KEY, bring_itde_up, is_itde_running, take_itde_down, @@ -32,34 +29,32 @@ def remove_itde(): def test_bring_itde_up(secrets): - secrets.save(AILabConfig.mem_size.value, "2") - secrets.save(AILabConfig.disk_size.value, "4") + secrets.save(AILabConfig.mem_size, "2") + secrets.save(AILabConfig.disk_size, "4") try: bring_itde_up(secrets) - assert secrets.get(CONTAINER_NAME_KEY) == DB_CONTAINER_NAME - assert secrets.get(VOLUME_NAME_KEY) == DB_VOLUME_NAME - assert secrets.get(NETWORK_NAME_KEY) == DB_NETWORK_NAME - assert secrets.get(AILabConfig.db_host_name.value) == secrets.get( - AILabConfig.bfs_host_name.value - ) - assert secrets.get(AILabConfig.db_user.value) == "sys" - assert secrets.get(AILabConfig.db_password.value) == "exasol" - assert secrets.get(AILabConfig.db_encryption.value) == "True" - assert secrets.get(AILabConfig.db_port.value) == "8563" - assert secrets.get(AILabConfig.bfs_service.value) == "bfsdefault" - assert secrets.get(AILabConfig.bfs_bucket.value) == "default" - assert secrets.get(AILabConfig.bfs_encryption.value) == "False" - assert secrets.get(AILabConfig.bfs_user.value) == "w" - assert secrets.get(AILabConfig.bfs_password.value) == "write" - assert secrets.get(AILabConfig.bfs_port.value) == "2580" + assert secrets.get(AILabConfig.itde_container) == DB_CONTAINER_NAME + assert secrets.get(AILabConfig.itde_volume) == DB_VOLUME_NAME + assert secrets.get(AILabConfig.itde_network) == DB_NETWORK_NAME + assert secrets.get(AILabConfig.db_host_name) == secrets.get(AILabConfig.bfs_host_name) + assert secrets.get(AILabConfig.db_user) == "sys" + assert secrets.get(AILabConfig.db_password) == "exasol" + assert secrets.get(AILabConfig.db_encryption) == "True" + assert secrets.get(AILabConfig.db_port) == "8563" + assert secrets.get(AILabConfig.bfs_service) == "bfsdefault" + assert secrets.get(AILabConfig.bfs_bucket) == "default" + assert secrets.get(AILabConfig.bfs_encryption) == "False" + assert secrets.get(AILabConfig.bfs_user) == "w" + assert secrets.get(AILabConfig.bfs_password) == "write" + assert secrets.get(AILabConfig.bfs_port) == "2580" finally: remove_itde() def test_is_itde_running(secrets): - secrets.save(AILabConfig.mem_size.value, "2") - secrets.save(AILabConfig.disk_size.value, "4") + secrets.save(AILabConfig.mem_size, "2") + secrets.save(AILabConfig.disk_size, "4") try: bring_itde_up(secrets) @@ -76,34 +71,34 @@ def test_is_not_itde_running(secrets): def test_take_itde_down(secrets): - secrets.save(AILabConfig.mem_size.value, "2") - secrets.save(AILabConfig.disk_size.value, "4") + secrets.save(AILabConfig.mem_size, "2") + secrets.save(AILabConfig.disk_size, "4") try: bring_itde_up(secrets) take_itde_down(secrets) - assert secrets.get(CONTAINER_NAME_KEY) is None - assert secrets.get(VOLUME_NAME_KEY) is None - assert secrets.get(NETWORK_NAME_KEY) is None - assert secrets.get(AILabConfig.db_host_name.value) is None - assert secrets.get(AILabConfig.bfs_host_name.value) is None - assert secrets.get(AILabConfig.db_user.value) is None - assert secrets.get(AILabConfig.db_password.value) is None - assert secrets.get(AILabConfig.db_encryption.value) is None - assert secrets.get(AILabConfig.db_port.value) is None - assert secrets.get(AILabConfig.bfs_service.value) is None - assert secrets.get(AILabConfig.bfs_bucket.value) is None - assert secrets.get(AILabConfig.bfs_encryption.value) is None - assert secrets.get(AILabConfig.bfs_user.value) is None - assert secrets.get(AILabConfig.bfs_password.value) is None - assert secrets.get(AILabConfig.bfs_port.value) is None + assert secrets.get(AILabConfig.itde_container) is None + assert secrets.get(AILabConfig.itde_volume) is None + assert secrets.get(AILabConfig.itde_network) is None + assert secrets.get(AILabConfig.db_host_name) is None + assert secrets.get(AILabConfig.bfs_host_name) is None + assert secrets.get(AILabConfig.db_user) is None + assert secrets.get(AILabConfig.db_password) is None + assert secrets.get(AILabConfig.db_encryption) is None + assert secrets.get(AILabConfig.db_port) is None + assert secrets.get(AILabConfig.bfs_service) is None + assert secrets.get(AILabConfig.bfs_bucket) is None + assert secrets.get(AILabConfig.bfs_encryption) is None + assert secrets.get(AILabConfig.bfs_user) is None + assert secrets.get(AILabConfig.bfs_password) is None + assert secrets.get(AILabConfig.bfs_port) is None finally: remove_itde() def test_take_itde_down_is_not_itde_running(secrets): - secrets.save(AILabConfig.mem_size.value, "2") - secrets.save(AILabConfig.disk_size.value, "4") + secrets.save(AILabConfig.mem_size, "2") + secrets.save(AILabConfig.disk_size, "4") try: bring_itde_up(secrets) take_itde_down(secrets) diff --git a/test/integration/test_sagemaker_extension_wrapper.py b/test/integration/test_sagemaker_extension_wrapper.py index 0a6e8c1..50f1477 100644 --- a/test/integration/test_sagemaker_extension_wrapper.py +++ b/test/integration/test_sagemaker_extension_wrapper.py @@ -1,9 +1,7 @@ -from exasol.sagemaker_extension_wrapper import ( - AWS_CONNECTION_KEY, - initialize_sme_extension, -) +from exasol.sagemaker_extension_wrapper import initialize_sme_extension from exasol.connections import open_pyexasol_connection from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig as CKey from test.utils.integration_test_utils import ( setup_itde, activate_languages, @@ -19,10 +17,10 @@ def test_initialize_sme_extension( ): # Here are fake AWS credentials. Should be fine since we are only testing # the deployment. - secrets.save("AWS_BUCKET", "NoneExistent") - secrets.save("AWS_REGION", "neverland") - secrets.save("AWS_ACCESS_KEY_ID", "FAKEKEYIDDONTUSEIT") - secrets.save("AWS_SECRET_ACCESS_KEY", "FakeSecretAccessKeyDontTryToUseIt") + secrets.save(CKey.aws_bucket, "NoneExistent") + secrets.save(CKey.aws_region, "neverland") + secrets.save(CKey.aws_access_key_id, "FAKEKEYIDDONTUSEIT") + secrets.save(CKey.aws_secret_access_key, "FakeSecretAccessKeyDontTryToUseIt") # Run the extension deployment. initialize_sme_extension(secrets) @@ -33,4 +31,4 @@ def test_initialize_sme_extension( script_counts = get_script_counts(pyexasol_connection, secrets) assert script_counts["SCRIPTING"] >= 4 assert script_counts["UDF"] >= 5 - assert_connection_exists(secrets.get(AWS_CONNECTION_KEY), pyexasol_connection) + assert_connection_exists(secrets.get(CKey.sme_aws_connection), pyexasol_connection) diff --git a/test/integration/test_transformers_extension_wrapper.py b/test/integration/test_transformers_extension_wrapper.py index d427fca..7ccb268 100644 --- a/test/integration/test_transformers_extension_wrapper.py +++ b/test/integration/test_transformers_extension_wrapper.py @@ -2,11 +2,8 @@ from exasol.connections import open_pyexasol_connection from exasol.secret_store import Secrets -from exasol.transformers_extension_wrapper import ( - BFS_CONNECTION_KEY, - HF_CONNECTION_KEY, - initialize_te_extension, -) +from exasol.ai_lab_config import AILabConfig as CKey +from exasol.transformers_extension_wrapper import initialize_te_extension from test.utils.integration_test_utils import ( setup_itde, activate_languages, @@ -23,7 +20,7 @@ def test_initialize_te_extension( ): test_name: str = request.node.name language_alias = f"PYTHON3_TE_{test_name.upper()}" - secrets.save("HF_TOKEN", "abc") + secrets.save(CKey.huggingface_token, "abc") with open_pyexasol_connection(secrets) as pyexasol_connection: @@ -34,5 +31,5 @@ def test_initialize_te_extension( assert_run_empty_udf(language_alias, pyexasol_connection, secrets) script_counts = get_script_counts(pyexasol_connection, secrets) assert script_counts["UDF"] > 5 - assert_connection_exists(secrets.get(BFS_CONNECTION_KEY), pyexasol_connection) - assert_connection_exists(secrets.get(HF_CONNECTION_KEY), pyexasol_connection) + assert_connection_exists(secrets.get(CKey.te_bfs_connection), pyexasol_connection) + assert_connection_exists(secrets.get(CKey.te_hf_connection), pyexasol_connection) diff --git a/test/unit/test_connections.py b/test/unit/test_connections.py index 690a742..0023efe 100644 --- a/test/unit/test_connections.py +++ b/test/unit/test_connections.py @@ -15,6 +15,7 @@ open_sqlalchemy_connection, ) from exasol.secret_store import Secrets +from exasol.ai_lab_config import AILabConfig as CKey @pytest.fixture @@ -30,27 +31,27 @@ def mock_get(self, key: str, default_value: Optional[str] = None) -> Optional[st mock_conf._params = {} mock_conf.save = types.MethodType(mock_save, mock_conf) mock_conf.get = types.MethodType(mock_get, mock_conf) - mock_conf.EXTERNAL_HOST_NAME = "24.134.96.2" - mock_conf.DB_PORT = "8888" - mock_conf.USER = "me" - mock_conf.PASSWORD = "let_me_in" - mock_conf.BUCKETFS_PORT = "6666" - mock_conf.BUCKETFS_USER = "buck_user" - mock_conf.BUCKETFS_PASSWORD = "buck_pwd" - mock_conf.BUCKETFS_BUCKET = "my_bucket" + mock_conf.save(CKey.db_host_name, "24.134.96.2") + mock_conf.save(CKey.db_port, "8888") + mock_conf.save(CKey.db_user, "me") + mock_conf.save(CKey.db_password, "let_me_in") + mock_conf.save(CKey.bfs_port, "6666") + mock_conf.save(CKey.bfs_user, "buck_user") + mock_conf.save(CKey.bfs_password, "buck_pwd") + mock_conf.save(CKey.bfs_bucket, "my_bucket") return mock_conf def test_get_external_host(conf): - assert get_external_host(conf) == f"{conf.EXTERNAL_HOST_NAME}:{conf.DB_PORT}" + assert get_external_host(conf) == f"{conf.get(CKey.db_host_name)}:{conf.get(CKey.db_port)}" @unittest.mock.patch("pyexasol.connect") def test_open_pyexasol_connection(mock_connect, conf): open_pyexasol_connection(conf) mock_connect.assert_called_once_with( - dsn=get_external_host(conf), user=conf.USER, password=conf.PASSWORD + dsn=get_external_host(conf), user=conf.get(CKey.db_user), password=conf.get(CKey.db_password) ) @@ -59,8 +60,8 @@ def test_open_pyexasol_connection_kwargs(mock_connect, conf): open_pyexasol_connection(conf, connection_timeout=3, query_timeout=10) mock_connect.assert_called_once_with( dsn=get_external_host(conf), - user=conf.USER, - password=conf.PASSWORD, + user=conf.get(CKey.db_user), + password=conf.get(CKey.db_password), connection_timeout=3, query_timeout=10, ) @@ -72,17 +73,17 @@ def test_open_pyexasol_connection_ssl(mock_connect, conf): tmp_files = [ stack.enter_context(tempfile.NamedTemporaryFile()) for _ in range(3) ] - conf.save("ENCRYPTION", "True") - conf.save("CERTIFICATE_VALIDATION", "Yes") - conf.save("TRUSTED_CA", tmp_files[0].name) - conf.save("CLIENT_CERTIFICATE", tmp_files[1].name) - conf.save("PRIVATE_KEY", tmp_files[2].name) + conf.save(CKey.db_encryption, "True") + conf.save(CKey.cert_vld, "Yes") + conf.save(CKey.trusted_ca, tmp_files[0].name) + conf.save(CKey.client_cert, tmp_files[1].name) + conf.save(CKey.client_key, tmp_files[2].name) open_pyexasol_connection(conf) mock_connect.assert_called_once_with( dsn=get_external_host(conf), - user=conf.USER, - password=conf.PASSWORD, + user=conf.get(CKey.db_user), + password=conf.get(CKey.db_password), encryption=True, websocket_sslopt={ "cert_reqs": ssl.CERT_REQUIRED, @@ -95,9 +96,9 @@ def test_open_pyexasol_connection_ssl(mock_connect, conf): @unittest.mock.patch("pyexasol.connect") def test_open_pyexasol_connection_error(mock_connect, conf): - conf.save("ENCRYPTION", "True") - conf.save("CERTIFICATE_VALIDATION", "Yes") - conf.save("TRUSTED_CA", "# non % existent & file") + conf.save(CKey.db_encryption, "True") + conf.save(CKey.cert_vld, "Yes") + conf.save(CKey.trusted_ca, "# non % existent & file") with pytest.raises(ValueError): open_pyexasol_connection(conf) @@ -107,18 +108,18 @@ def test_open_pyexasol_connection_error(mock_connect, conf): def test_open_sqlalchemy_connection(mock_create_engine, conf): open_sqlalchemy_connection(conf) mock_create_engine.assert_called_once_with( - f"exa+websocket://{conf.USER}:{conf.PASSWORD}@{get_external_host(conf)}" + f"exa+websocket://{conf.get(CKey.db_user)}:{conf.get(CKey.db_password)}@{get_external_host(conf)}" ) @unittest.mock.patch("sqlalchemy.create_engine") def test_open_sqlalchemy_connection_ssl(mock_create_engine, conf): - conf.save("ENCRYPTION", "True") - conf.save("CERTIFICATE_VALIDATION", "False") + conf.save(CKey.db_encryption, "True") + conf.save(CKey.cert_vld, "False") open_sqlalchemy_connection(conf) mock_create_engine.assert_called_once_with( - f"exa+websocket://{conf.USER}:{conf.PASSWORD}@{get_external_host(conf)}" + f"exa+websocket://{conf.get(CKey.db_user)}:{conf.get(CKey.db_password)}@{get_external_host(conf)}" "?ENCRYPTION=Yes&SSLCertificate=SSL_VERIFY_NONE" ) @@ -127,11 +128,11 @@ def test_open_sqlalchemy_connection_ssl(mock_create_engine, conf): def test_open_bucketfs_connection(mock_bfs_service, conf): open_bucketfs_connection(conf) mock_bfs_service.assert_called_once_with( - f"http://{conf.EXTERNAL_HOST_NAME}:{conf.BUCKETFS_PORT}", + f"http://{conf.get(CKey.db_host_name)}:{conf.get(CKey.bfs_port)}", { - conf.BUCKETFS_BUCKET: { - "username": conf.BUCKETFS_USER, - "password": conf.BUCKETFS_PASSWORD, + conf.get(CKey.bfs_bucket): { + "username": conf.get(CKey.bfs_user), + "password": conf.get(CKey.bfs_password), } }, ) diff --git a/test/unit/test_itde_manager.py b/test/unit/test_itde_manager.py index 7585ea2..51c60df 100644 --- a/test/unit/test_itde_manager.py +++ b/test/unit/test_itde_manager.py @@ -15,13 +15,11 @@ ) from exasol_integration_test_docker_environment.lib.test_environment.ports import Ports -from exasol.ai_lab_config import AILabConfig + +from exasol.ai_lab_config import AILabConfig as CKey from exasol.itde_manager import ( - CONTAINER_NAME_KEY, ENVIRONMENT_NAME, NAME_SERVER_ADDRESS, - NETWORK_NAME_KEY, - VOLUME_NAME_KEY, bring_itde_up, take_itde_down, ) @@ -49,8 +47,8 @@ def env_info() -> EnvironmentInfo: def test_bring_itde_up(mock_spawn_env, secrets, env_info): mock_spawn_env.return_value = (env_info, None) - secrets.save(AILabConfig.mem_size.value, "4") - secrets.save(AILabConfig.disk_size.value, "10") + secrets.save(CKey.mem_size, "4") + secrets.save(CKey.disk_size, "10") bring_itde_up(secrets) @@ -61,21 +59,21 @@ def test_bring_itde_up(mock_spawn_env, secrets, env_info): db_disk_size="10 GiB", ) - assert secrets.get(CONTAINER_NAME_KEY) == TEST_CONTAINER_NAME - assert secrets.get(VOLUME_NAME_KEY) == TEST_VOLUME_NAME - assert secrets.get(NETWORK_NAME_KEY) == TEST_NETWORK_NAME - assert secrets.get(AILabConfig.db_host_name.value) == TEST_DB_HOST - assert secrets.get(AILabConfig.bfs_host_name.value) == TEST_DB_HOST - assert int(secrets.get(AILabConfig.db_port.value)) == TEST_DB_PORT - assert int(secrets.get(AILabConfig.bfs_port.value)) == TEST_BFS_PORT - assert secrets.get(AILabConfig.db_user.value) == "sys" - assert secrets.get(AILabConfig.db_password.value) == "exasol" - assert secrets.get(AILabConfig.db_encryption.value) == "True" - assert secrets.get(AILabConfig.bfs_service.value) == "bfsdefault" - assert secrets.get(AILabConfig.bfs_bucket.value) == "default" - assert secrets.get(AILabConfig.bfs_encryption.value) == "False" - assert secrets.get(AILabConfig.bfs_user.value) == "w" - assert secrets.get(AILabConfig.bfs_password.value) == "write" + assert secrets.get(CKey.itde_container) == TEST_CONTAINER_NAME + assert secrets.get(CKey.itde_volume) == TEST_VOLUME_NAME + assert secrets.get(CKey.itde_network) == TEST_NETWORK_NAME + assert secrets.get(CKey.db_host_name) == TEST_DB_HOST + assert secrets.get(CKey.bfs_host_name) == TEST_DB_HOST + assert int(secrets.get(CKey.db_port)) == TEST_DB_PORT + assert int(secrets.get(CKey.bfs_port)) == TEST_BFS_PORT + assert secrets.get(CKey.db_user) == "sys" + assert secrets.get(CKey.db_password) == "exasol" + assert secrets.get(CKey.db_encryption) == "True" + assert secrets.get(CKey.bfs_service) == "bfsdefault" + assert secrets.get(CKey.bfs_bucket) == "default" + assert secrets.get(CKey.bfs_encryption) == "False" + assert secrets.get(CKey.bfs_user) == "w" + assert secrets.get(CKey.bfs_password) == "write" @mock.patch( @@ -88,24 +86,24 @@ def test_bring_itde_up(mock_spawn_env, secrets, env_info): "exasol_integration_test_docker_environment.lib.docker.networks.utils.remove_docker_networks" ) def test_take_itde_down(mock_util1, mock_util2, mock_util3, secrets): - secrets.save(CONTAINER_NAME_KEY, TEST_CONTAINER_NAME) - secrets.save(VOLUME_NAME_KEY, TEST_VOLUME_NAME) - secrets.save(NETWORK_NAME_KEY, TEST_NETWORK_NAME) + secrets.save(CKey.itde_container, TEST_CONTAINER_NAME) + secrets.save(CKey.itde_volume, TEST_VOLUME_NAME) + secrets.save(CKey.itde_network, TEST_NETWORK_NAME) take_itde_down(secrets) - assert secrets.get(CONTAINER_NAME_KEY) is None - assert secrets.get(VOLUME_NAME_KEY) is None - assert secrets.get(NETWORK_NAME_KEY) is None - assert secrets.get(AILabConfig.db_host_name.value) is None - assert secrets.get(AILabConfig.bfs_host_name.value) is None - assert secrets.get(AILabConfig.db_user.value) is None - assert secrets.get(AILabConfig.db_password.value) is None - assert secrets.get(AILabConfig.db_encryption.value) is None - assert secrets.get(AILabConfig.db_port.value) is None - assert secrets.get(AILabConfig.bfs_service.value) is None - assert secrets.get(AILabConfig.bfs_bucket.value) is None - assert secrets.get(AILabConfig.bfs_encryption.value) is None - assert secrets.get(AILabConfig.bfs_user.value) is None - assert secrets.get(AILabConfig.bfs_password.value) is None - assert secrets.get(AILabConfig.bfs_port.value) is None + assert secrets.get(CKey.itde_container) is None + assert secrets.get(CKey.itde_volume) is None + assert secrets.get(CKey.itde_network) is None + assert secrets.get(CKey.db_host_name) is None + assert secrets.get(CKey.bfs_host_name) is None + assert secrets.get(CKey.db_user) is None + assert secrets.get(CKey.db_password) is None + assert secrets.get(CKey.db_encryption) is None + assert secrets.get(CKey.db_port) is None + assert secrets.get(CKey.bfs_service) is None + assert secrets.get(CKey.bfs_bucket) is None + assert secrets.get(CKey.bfs_encryption) is None + assert secrets.get(CKey.bfs_user) is None + assert secrets.get(CKey.bfs_password) is None + assert secrets.get(CKey.bfs_port) is None diff --git a/test/unit/test_secret_store.py b/test/unit/test_secret_store.py index 4457471..67deb37 100644 --- a/test/unit/test_secret_store.py +++ b/test/unit/test_secret_store.py @@ -6,6 +6,7 @@ InvalidPassword, Secrets, ) +from exasol.ai_lab_config import AILabConfig as CKey def test_no_database_file(secrets): @@ -23,6 +24,12 @@ def test_value(secrets): assert secrets.get("key") == value +def test_db_user(secrets): + name = "Beethoven" + secrets.save(CKey.db_user, name).close() + assert secrets.get(CKey.db_user) == name + + def test_dup_values(secrets): # Here we test that it is possible to save the same value with different keys value = "my value" @@ -111,3 +118,9 @@ def test_remove_key(secrets): secrets.save("key", "value") secrets.remove("key") assert secrets.get("key") is None + + +def test_remove_db_schema(secrets): + secrets.save(CKey.db_schema, "the_schema") + secrets.remove(CKey.db_schema) + assert secrets.get(CKey.db_schema) is None diff --git a/test/utils/integration_test_utils.py b/test/utils/integration_test_utils.py index 4cd6e15..1e2c37a 100644 --- a/test/utils/integration_test_utils.py +++ b/test/utils/integration_test_utils.py @@ -24,7 +24,7 @@ def setup_itde(secrets) -> None: bring_itde_up(secrets) schema = 'INTEGRATION_TEST' - secrets.save(AILabConfig.db_schema.value, schema) + secrets.save(AILabConfig.db_schema, schema) with open_pyexasol_connection(secrets) as pyexasol_connection: pyexasol_connection.execute(f"CREATE SCHEMA {schema};") @@ -55,7 +55,7 @@ def assert_run_empty_udf( pyexasol_connection.execute( textwrap.dedent( f""" - CREATE OR REPLACE {language_alias} SCALAR SCRIPT {secrets.SCHEMA}."TEST_UDF"() + CREATE OR REPLACE {language_alias} SCALAR SCRIPT {secrets.get(AILabConfig.db_schema)}."TEST_UDF"() RETURNS BOOLEAN AS def run(ctx): return True @@ -64,7 +64,7 @@ def run(ctx): ) ) result = pyexasol_connection.execute( - f'SELECT {secrets.SCHEMA}."TEST_UDF"()' + f'SELECT {secrets.get(AILabConfig.db_schema)}."TEST_UDF"()' ).fetchall() assert result[0][0] @@ -79,7 +79,7 @@ def get_script_counts( result = pyexasol_connection.execute( f""" SELECT SCRIPT_TYPE, COUNT(*) FROM SYS.EXA_ALL_SCRIPTS - WHERE SCRIPT_SCHEMA='{secrets.SCHEMA.upper()}' GROUP BY SCRIPT_TYPE; + WHERE SCRIPT_SCHEMA='{secrets.get(AILabConfig.db_schema).upper()}' GROUP BY SCRIPT_TYPE; """ ).fetchall() return dict(result) diff --git a/version.py b/version.py index 2930663..feccce2 100644 --- a/version.py +++ b/version.py @@ -6,5 +6,5 @@ # If you need to change the version, do so in the project.toml, e.g. by using `poetry version X.Y.Z`. MAJOR = 0 MINOR = 2 -PATCH = 5 +PATCH = 6 VERSION = f"{MAJOR}.{MINOR}.{PATCH}"