From d9b6c21ea89ad23c176b126a8be8afa985ef5c6c Mon Sep 17 00:00:00 2001 From: Phoebus Mak Date: Wed, 10 Jan 2024 18:11:52 +0000 Subject: [PATCH] Add S3 ca path support --- cpp/arcticdb/storage/python_bindings.cpp | 6 +- cpp/arcticdb/storage/s3/s3_storage.hpp | 4 + cpp/arcticdb/storage/storage_override.hpp | 41 +++++++++ .../arcticc/pb2/nfs_backed_storage.proto | 2 + cpp/proto/arcticc/pb2/s3_storage.proto | 2 + environment_unix.yml | 3 +- .../arcticdb/adapters/s3_library_adapter.py | 35 ++++++++ python/arcticdb/storage_fixtures/s3.py | 90 +++++++++++++++---- python/arcticdb/storage_fixtures/utils.py | 3 +- python/arcticdb/version_store/helper.py | 19 +++- python/tests/conftest.py | 30 ++++++- .../tests/integration/arcticdb/test_arctic.py | 49 +++++++++- .../integration/storage_fixtures/test_s3.py | 14 +-- python/tests/util/mark.py | 11 +++ setup.cfg | 1 + 15 files changed, 274 insertions(+), 36 deletions(-) diff --git a/cpp/arcticdb/storage/python_bindings.cpp b/cpp/arcticdb/storage/python_bindings.cpp index 368afebb54..c573852554 100644 --- a/cpp/arcticdb/storage/python_bindings.cpp +++ b/cpp/arcticdb/storage/python_bindings.cpp @@ -114,7 +114,11 @@ void register_bindings(py::module& storage) { .def_property("bucket_name", &S3Override::bucket_name, &S3Override::set_bucket_name) .def_property("region", &S3Override::region, &S3Override::set_region) .def_property( - "use_virtual_addressing", &S3Override::use_virtual_addressing, &S3Override::set_use_virtual_addressing); + "use_virtual_addressing", &S3Override::use_virtual_addressing, &S3Override::set_use_virtual_addressing) + .def_property("ca_cert_path", &S3Override::ca_cert_path, &S3Override::set_ca_cert_path) + .def_property("ca_cert_dir", &S3Override::ca_cert_dir, &S3Override::set_ca_cert_dir) + .def_property("https", &S3Override::https, &S3Override::set_https) + .def_property("ssl", &S3Override::ssl, &S3Override::set_ssl); py::class_(storage, "AzureOverride") .def(py::init<>()) diff --git a/cpp/arcticdb/storage/s3/s3_storage.hpp b/cpp/arcticdb/storage/s3/s3_storage.hpp index a2958c0017..1c8f870710 100644 --- a/cpp/arcticdb/storage/s3/s3_storage.hpp +++ b/cpp/arcticdb/storage/s3/s3_storage.hpp @@ -224,6 +224,10 @@ auto get_s3_config(const ConfigType& conf) { const bool verify_ssl = ConfigsMap::instance()->get_int("S3Storage.VerifySSL", conf.ssl()); ARCTICDB_RUNTIME_DEBUG(log::storage(), "Verify ssl: {}", verify_ssl); client_configuration.verifySSL = verify_ssl; + if (client_configuration.verifySSL && (!conf.ca_cert_path().empty() || !conf.ca_cert_dir().empty())) { + client_configuration.caFile = conf.ca_cert_path(); + client_configuration.caPath = conf.ca_cert_dir(); + } client_configuration.maxConnections = conf.max_connections() == 0 ? ConfigsMap::instance()->get_int("VersionStore.NumIOThreads", 16) : conf.max_connections(); diff --git a/cpp/arcticdb/storage/storage_override.hpp b/cpp/arcticdb/storage/storage_override.hpp index fdefe93965..bb4d2ccb92 100644 --- a/cpp/arcticdb/storage/storage_override.hpp +++ b/cpp/arcticdb/storage/storage_override.hpp @@ -14,7 +14,11 @@ class S3Override { std::string endpoint_; std::string bucket_name_; std::string region_; + std::string ca_cert_path_; + std::string ca_cert_dir_; bool use_virtual_addressing_ = false; + bool https_; + bool ssl_; public: std::string credential_name() const { @@ -65,6 +69,39 @@ class S3Override { use_virtual_addressing_ = use_virtual_addressing; } + std::string ca_cert_path() const { + return ca_cert_path_; + } + + void set_ca_cert_path(std::string_view ca_cert_path){ + ca_cert_path_ = ca_cert_path; + } + + + std::string ca_cert_dir() const { + return ca_cert_dir_; + } + + void set_ca_cert_dir(std::string_view ca_cert_dir){ + ca_cert_dir_ = ca_cert_dir; + } + + bool https() const { + return https_; + } + + void set_https(bool https){ + https_ = https; + } + + bool ssl() const { + return ssl_; + } + + void set_ssl(bool ssl){ + ssl_ = ssl; + } + void modify_storage_config(arcticdb::proto::storage::VariantStorage& storage) const { if(storage.config().Is()) { arcticdb::proto::s3_storage::Config s3_storage; @@ -76,6 +113,10 @@ class S3Override { s3_storage.set_endpoint(endpoint_); s3_storage.set_region(region_); s3_storage.set_use_virtual_addressing(use_virtual_addressing_); + s3_storage.set_ca_cert_path(ca_cert_path_); + s3_storage.set_ca_cert_dir(ca_cert_dir_); + s3_storage.set_https(https_); + s3_storage.set_ssl(ssl_); util::pack_to_any(s3_storage, *storage.mutable_config()); } diff --git a/cpp/proto/arcticc/pb2/nfs_backed_storage.proto b/cpp/proto/arcticc/pb2/nfs_backed_storage.proto index 93e6ad90a2..d8837d1aaa 100644 --- a/cpp/proto/arcticc/pb2/nfs_backed_storage.proto +++ b/cpp/proto/arcticc/pb2/nfs_backed_storage.proto @@ -22,4 +22,6 @@ message Config { bool https = 10; string region = 11; bool use_virtual_addressing = 12; + string ca_cert_path = 13; + string ca_cert_dir = 14; } diff --git a/cpp/proto/arcticc/pb2/s3_storage.proto b/cpp/proto/arcticc/pb2/s3_storage.proto index b832aacb4a..ac2420c81c 100644 --- a/cpp/proto/arcticc/pb2/s3_storage.proto +++ b/cpp/proto/arcticc/pb2/s3_storage.proto @@ -23,4 +23,6 @@ message Config { string region = 11; bool use_virtual_addressing = 12; bool use_mock_storage_for_testing = 13; + string ca_cert_path = 14; + string ca_cert_dir = 15; } diff --git a/environment_unix.yml b/environment_unix.yml index ae770c3182..980d1114cd 100644 --- a/environment_unix.yml +++ b/environment_unix.yml @@ -31,7 +31,7 @@ dependencies: - prometheus-cpp - libprotobuf < 4 - openssl - - libcurl + - libcurl==8.5.0 #Until https://github.com/conda-forge/curl-feedstock/issues/135 addressed and release build available - bitmagic - spdlog - azure-core-cpp @@ -81,3 +81,4 @@ dependencies: - asv - pymongo - pytest + - trustme diff --git a/python/arcticdb/adapters/s3_library_adapter.py b/python/arcticdb/adapters/s3_library_adapter.py index 0bd061804f..d3b202843f 100644 --- a/python/arcticdb/adapters/s3_library_adapter.py +++ b/python/arcticdb/adapters/s3_library_adapter.py @@ -8,6 +8,8 @@ import re import time from typing import Optional +import ssl +import platform from arcticdb.options import LibraryOptions from arcticc.pb2.storage_pb2 import EnvironmentConfigsMap, LibraryConfig @@ -45,6 +47,13 @@ class ParsedQuery: # DEPRECATED - see https://github.com/man-group/ArcticDB/pull/833 force_uri_lib_config: Optional[bool] = True + # winhttp is used as s3 backend support on Winodws by default; winhttp itself mainatains ca cert. + # The options has no effect on Windows + CA_cert_path: Optional[str] = "" # CURLOPT_CAINFO in curl + CA_cert_dir: Optional[str] = "" # CURLOPT_CAPATH in curl + + ssl: Optional[bool] = False + class S3LibraryAdapter(ArcticLibraryAdapter): REGEX = r"s3(s)?://(?P.*):(?P[-_a-zA-Z0-9.]+)(?P\?.*)?" @@ -73,6 +82,18 @@ def __init__(self, uri: str, encoding_version: EncodingVersion, *args, **kwargs) self._https = uri.startswith("s3s") self._encoding_version = encoding_version + if platform.system() != "Linux" and (self._query_params.CA_cert_path or self._query_params.CA_cert_dir): + raise ValueError("You have provided `ca_cert_path` or `ca_cert_dir` in the URI which is only supported on Linux. " \ + "Remove the setting in the connection URI and use your operating system defaults.") + self._ca_cert_path = self._query_params.CA_cert_path + self._ca_cert_dir = self._query_params.CA_cert_dir + if not self._ca_cert_path and not self._ca_cert_dir and platform.system() == "Linux": + if ssl.get_default_verify_paths().cafile is not None: + self._ca_cert_path = ssl.get_default_verify_paths().cafile + if ssl.get_default_verify_paths().capath is not None: + self._ca_cert_dir = ssl.get_default_verify_paths().capath + + self._ssl = self._query_params.ssl if "amazonaws" in self._endpoint: self._configure_aws() @@ -103,6 +124,9 @@ def config_library(self): with_prefix=with_prefix, region=self._query_params.region, use_virtual_addressing=self._query_params.use_virtual_addressing, + ca_cert_path=self._ca_cert_path, + ca_cert_dir=self._ca_cert_dir, + ssl=self._ssl, ) lib = NativeVersionStore.create_store_from_config( @@ -160,6 +184,14 @@ def get_storage_override(self) -> StorageOverride: s3_override.endpoint = self._endpoint if self._bucket: s3_override.bucket_name = self._bucket + if self._https: + s3_override.https = self._https + if self._ca_cert_path: + s3_override.ca_cert_path = self._ca_cert_path + if self._ca_cert_dir: + s3_override.ca_cert_dir = self._ca_cert_dir + if self._ssl: + s3_override.ssl = self._ssl s3_override.use_virtual_addressing = self._query_params.use_virtual_addressing @@ -196,6 +228,9 @@ def add_library_to_env(self, env_cfg: EnvironmentConfigsMap, name: str): env_name=_DEFAULT_ENV, region=self._query_params.region, use_virtual_addressing=self._query_params.use_virtual_addressing, + ca_cert_path=self._ca_cert_path, + ca_cert_dir=self._ca_cert_dir, + ssl=self._ssl, ) def _configure_aws(self): diff --git a/python/arcticdb/storage_fixtures/s3.py b/python/arcticdb/storage_fixtures/s3.py index 77dc0da7fc..7a5bf2e85b 100644 --- a/python/arcticdb/storage_fixtures/s3.py +++ b/python/arcticdb/storage_fixtures/s3.py @@ -12,13 +12,17 @@ import os import re import sys -import time +import trustme +import subprocess +import platform +from tempfile import mkdtemp + import requests from typing import NamedTuple, Optional, Any, Type from .api import * -from .utils import get_ephemeral_port, GracefulProcessUtils, wait_for_server_to_come_up +from .utils import get_ephemeral_port, GracefulProcessUtils, wait_for_server_to_come_up, safer_rmtree from arcticc.pb2.storage_pb2 import EnvironmentConfigsMap from arcticdb.version_store.helper import add_s3_library_to_env @@ -57,6 +61,12 @@ def __init__(self, factory: "BaseS3StorageFixtureFactory", bucket: str): self.arctic_uri += f"&port={port}" if factory.default_prefix: self.arctic_uri += f"&path_prefix={factory.default_prefix}" + if factory.ssl: + self.arctic_uri += "&ssl=True" + if platform.system() == "Linux": + if factory.client_cert_file: + self.arctic_uri += f"&CA_cert_path={self.factory.client_cert_file}" + # client_cert_dir is skipped on purpose; It will be test manually in other tests def __exit__(self, exc_type, exc_value, traceback): if self.factory.clean_bucket_on_fixture_exit: @@ -81,7 +91,9 @@ def create_test_cfg(self, lib_name: str) -> EnvironmentConfigsMap: is_https=self.factory.endpoint.startswith("s3s:"), region=self.factory.region, use_mock_storage_for_testing=self.factory.use_mock_storage_for_testing, - ) + ssl=self.factory.ssl, + ca_cert_path=self.factory.client_cert_file, + )# client_cert_dir is skipped on purpose; It will be test manually in other tests return cfg def set_permission(self, *, read: bool, write: bool): @@ -124,6 +136,11 @@ class BaseS3StorageFixtureFactory(StorageFixtureFactory): clean_bucket_on_fixture_exit = True use_mock_storage_for_testing = None # If set to true allows error simulation + def __init__(self): + self.client_cert_file = "" + self.client_cert_dir = "" + self.ssl = False + def __str__(self): return f"{type(self).__name__}[{self.default_bucket or self.endpoint}]" @@ -137,7 +154,8 @@ def _boto(self, service: str, key: Key, api="client"): region_name=self.region, aws_access_key_id=key.id, aws_secret_access_key=key.secret, - ) + verify=self.client_cert_file if self.client_cert_file else False, + ) # verify=False cannot skip verification on buggy boto3 in py3.6 def create_fixture(self) -> S3Bucket: return S3Bucket(self, self.default_bucket) @@ -194,8 +212,12 @@ class MotoS3StorageFixtureFactory(BaseS3StorageFixtureFactory): _bucket_id = 0 _live_buckets: List[S3Bucket] = [] + def __init__(self, use_ssl: bool): + self.http_protocol = "https" if use_ssl else "http" + + @staticmethod - def run_server(port): + def run_server(port, key_file, cert_file): import werkzeug from moto.server import DomainDispatcherApplication, create_backend_app @@ -244,26 +266,55 @@ def __call__(self, environ, start_response): self._reqs_till_rate_limit -= 1 return super().__call__(environ, start_response) - werkzeug.run_simple( "0.0.0.0", port, _HostDispatcherApplication(create_backend_app), threaded=True, - ssl_context=None, + ssl_context=(cert_file, key_file) if cert_file and key_file else None, ) def _start_server(self): port = self.port = get_ephemeral_port(2) - self.endpoint = f"http://{self.host}:{port}" - self._iam_endpoint = f"http://127.0.0.1:{port}" - - p = self._p = multiprocessing.Process(target=self.run_server, args=(port,)) - p.start() - wait_for_server_to_come_up(self.endpoint, "moto", p) + self.endpoint = f"{self.http_protocol}://{self.host}:{port}" + self.working_dir = mkdtemp(suffix="MotoS3StorageFixtureFactory") + self._iam_endpoint = f"{self.http_protocol}://localhost:{port}" + + self.ssl = self.http_protocol == "https" # In real world, using https protocol doesn't necessarily mean ssl will be verified + if self.http_protocol == "https": + self.key_file = os.path.join(self.working_dir, "key.pem") + self.cert_file = os.path.join(self.working_dir, "cert.pem") + self.client_cert_file = os.path.join(self.working_dir, "client.pem") + ca = trustme.CA() + server_cert = ca.issue_cert("localhost") + server_cert.private_key_pem.write_to_path(self.key_file) + server_cert.cert_chain_pems[0].write_to_path(self.cert_file) + ca.cert_pem.write_to_path(self.client_cert_file) + self.client_cert_dir = self.working_dir + # Create the sym link for curl CURLOPT_CAPATH option; rehash only available on openssl >=1.1.1 + subprocess.run( + f'ln -s "{self.client_cert_file}" "$(openssl x509 -hash -noout -in "{self.client_cert_file}")".0', + cwd=self.working_dir, + shell=True, + ) + else: + self.key_file = "" + self.cert_file = "" + self.client_cert_file = "" + self.client_cert_dir = "" + + self._p = multiprocessing.Process( + target=self.run_server, + args=(port, + self.key_file if self.http_protocol == "https" else None, + self.cert_file if self.http_protocol == "https" else None, + ) + ) + self._p.start() + wait_for_server_to_come_up(self.endpoint, "moto", self._p) def _safe_enter(self): - for attempt in range(3): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start + for _ in range(3): # For unknown reason, Moto, when running in pytest-xdist, will randomly fail to start try: self._start_server() break @@ -271,11 +322,12 @@ def _safe_enter(self): sys.stderr.write(repr(e)) GracefulProcessUtils.terminate(self._p) - self._s3_admin = self._boto("s3", self.default_key) + self._s3_admin = self._boto(service="s3", key=self.default_key) return self def __exit__(self, exc_type, exc_value, traceback): GracefulProcessUtils.terminate(self._p) + safer_rmtree(self, self.working_dir) def _create_user_get_key(self, user: str, iam=None): iam = iam or self._iam_admin @@ -293,7 +345,7 @@ def enforcing_permissions(self, enforcing: bool): if enforcing == self._enforcing_permissions: return if enforcing and not self._iam_admin: - iam = self._boto("iam", self.default_key) + iam = self._boto(service="iam", key=self.default_key) def _policy(*statements): return json.dumps({"Version": "2012-10-17", "Statement": statements}) @@ -309,8 +361,8 @@ def _policy(*statements): key = self._create_user_get_key("admin", iam) iam.attach_user_policy(UserName="admin", PolicyArn=policy_arn) - self._iam_admin = self._boto("iam", key) - self._s3_admin = self._boto("s3", key) + self._iam_admin = self._boto(service="iam", key=key) + self._s3_admin = self._boto(service="s3", key=key) # The number is the remaining requests before permission checks kick in requests.post(self._iam_endpoint + "/moto-api/reset-auth", "0" if enforcing else "inf") @@ -331,7 +383,7 @@ def cleanup_bucket(self, b: S3Bucket): b.slow_cleanup(failure_consequence="The following delete bucket call will also fail. ") self._s3_admin.delete_bucket(Bucket=b.bucket) else: - requests.post(self._iam_endpoint + "/moto-api/reset") + requests.post(self._iam_endpoint + "/moto-api/reset", verify=False) # If CA cert verify fails, it will take ages for this line to finish self._iam_admin = None diff --git a/python/arcticdb/storage_fixtures/utils.py b/python/arcticdb/storage_fixtures/utils.py index 338498bd03..86e36246fe 100644 --- a/python/arcticdb/storage_fixtures/utils.py +++ b/python/arcticdb/storage_fixtures/utils.py @@ -20,6 +20,7 @@ from typing import Union, Any from contextlib import AbstractContextManager from dataclasses import dataclass, field +import trustme _WINDOWS = platform.system() == "Windows" _DEBUG = os.getenv("ACTIONS_RUNNER_DEBUG", default=None) in (1, "True") @@ -97,7 +98,7 @@ def wait_for_server_to_come_up(url: str, service: str, process: ProcessUnion, *, assert alive(), service + " process died shortly after start up" time.sleep(sleep) try: - response = requests.get(url, timeout=req_timeout) # head() confuses Mongo + response = requests.get(url, timeout=req_timeout, verify=False) # head() confuses Mongo if response.status_code < 500: # We might not have permission, so not requiring 2XX response break except (requests.exceptions.ConnectionError, requests.exceptions.Timeout): diff --git a/python/arcticdb/version_store/helper.py b/python/arcticdb/version_store/helper.py index f597bd37e3..fe674f1821 100644 --- a/python/arcticdb/version_store/helper.py +++ b/python/arcticdb/version_store/helper.py @@ -226,6 +226,9 @@ def get_s3_proto( region=None, use_virtual_addressing=False, use_mock_storage_for_testing=None, + ca_cert_path="", + ca_cert_dir="", + ssl=False ): env = cfg.env_by_id[env_name] s3 = S3Config() @@ -254,6 +257,12 @@ def get_s3_proto( if region: s3.region = region + if ca_cert_path is not None: + s3.ca_cert_path = ca_cert_path + if ca_cert_dir is not None: + s3.ca_cert_dir = ca_cert_dir + if ssl is not None: + s3.ssl = ssl sid, storage = get_storage_for_lib_name(s3.prefix, env) storage.config.Pack(s3, type_url_prefix="cxx.arctic.org") @@ -274,6 +283,9 @@ def add_s3_library_to_env( region=None, use_virtual_addressing=False, use_mock_storage_for_testing=None, + ca_cert_path="", + ca_cert_dir="", + ssl=False, ): env = cfg.env_by_id[env_name] if with_prefix and isinstance(with_prefix, str) and (with_prefix.endswith("/") or "//" in with_prefix): @@ -282,7 +294,7 @@ def add_s3_library_to_env( f" [{with_prefix}]" ) - sid, storage = get_s3_proto( + sid, _ = get_s3_proto( cfg=cfg, lib_name=lib_name, env_name=env_name, @@ -295,6 +307,9 @@ def add_s3_library_to_env( region=region, use_virtual_addressing=use_virtual_addressing, use_mock_storage_for_testing=use_mock_storage_for_testing, + ca_cert_path=ca_cert_path, + ca_cert_dir=ca_cert_dir, + ssl=ssl, ) _add_lib_desc_to_env(env, lib_name, sid, description) @@ -340,7 +355,7 @@ def add_azure_library_to_env( ca_cert_path: str = "", ): env = cfg.env_by_id[env_name] - sid, storage = get_azure_proto( + sid, _ = get_azure_proto( cfg=cfg, lib_name=lib_name, env_name=env_name, diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 89d20a3f23..b687b219f7 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -32,7 +32,12 @@ from arcticdb.storage_fixtures.in_memory import InMemoryStorageFixture from arcticdb.version_store._normalization import MsgPackNormalizer from arcticdb.util.test import create_df -from tests.util.mark import AZURE_TESTS_MARK, MONGO_TESTS_MARK, REAL_S3_TESTS_MARK +from tests.util.mark import ( + AZURE_TESTS_MARK, + MONGO_TESTS_MARK, + REAL_S3_TESTS_MARK, + S3_SSL_TEST_ENABLED, +) # region =================================== Misc. Constants & Setup ==================================== hypothesis.settings.register_profile("ci_linux", max_examples=100) @@ -103,7 +108,13 @@ def lmdb_storage(tmp_path): @pytest.fixture(scope="session") def s3_storage_factory(): - with MotoS3StorageFixtureFactory() as f: + with MotoS3StorageFixtureFactory(use_ssl=S3_SSL_TEST_ENABLED) as f: + yield f + + +@pytest.fixture(scope="session") +def s3_no_ssl_storage_factory(): + with MotoS3StorageFixtureFactory(use_ssl=False) as f: yield f @@ -113,6 +124,12 @@ def s3_storage(s3_storage_factory): yield f +@pytest.fixture +def s3_no_ssl_storage(s3_no_ssl_storage_factory): + with s3_no_ssl_storage_factory.create_fixture() as f: + yield f + + @pytest.fixture def mock_s3_storage_with_error_simulation_factory(): return mock_s3_with_error_simulation() @@ -232,9 +249,9 @@ def s3_store_factory_mock_storage_exception(lib_name, s3_storage): # The following interger is for how many requests until 503 repsonse is sent # -1 means the 503 response is disabled # Setting persisted throughout the lifetime of moto server, so it needs to be reset - requests.post(endpoint + "/rate_limit", b"0").raise_for_status() + requests.post(endpoint + "/rate_limit", b"0", verify=False).raise_for_status() yield lib - requests.post(endpoint + "/rate_limit", b"-1").raise_for_status() + requests.post(endpoint + "/rate_limit", b"-1", verify=False).raise_for_status() @pytest.fixture @@ -242,6 +259,11 @@ def s3_store_factory(lib_name, s3_storage): return s3_storage.create_version_store_factory(lib_name) +@pytest.fixture +def s3_no_ssl_store_factory(lib_name, s3_no_ssl_storage): + return s3_no_ssl_storage.create_version_store_factory(lib_name) + + @pytest.fixture def mock_s3_store_with_error_simulation_factory(lib_name, mock_s3_storage_with_error_simulation): return mock_s3_storage_with_error_simulation.create_version_store_factory(lib_name) diff --git a/python/tests/integration/arcticdb/test_arctic.py b/python/tests/integration/arcticdb/test_arctic.py index b202685095..b5a9703409 100644 --- a/python/tests/integration/arcticdb/test_arctic.py +++ b/python/tests/integration/arcticdb/test_arctic.py @@ -13,6 +13,8 @@ import pandas as pd import numpy as np from datetime import datetime, timezone +from enum import Enum +import platform from arcticdb_ext.exceptions import InternalException, UserInputException from arcticdb_ext.storage import NoDataFoundException @@ -32,7 +34,52 @@ ArcticInvalidApiUsageException, ) -from tests.util.mark import AZURE_TESTS_MARK, MONGO_TESTS_MARK, REAL_S3_TESTS_MARK +from tests.util.mark import AZURE_TESTS_MARK, MONGO_TESTS_MARK, REAL_S3_TESTS_MARK, S3_SSL_TESTS_MARK + +class ParameterDisplayStatus(Enum): + NOT_SHOW = 1 + DISABLE = 2 + ENABLE = 3 + +parameter_display_status = [ParameterDisplayStatus.NOT_SHOW, ParameterDisplayStatus.DISABLE, ParameterDisplayStatus.ENABLE] + +@S3_SSL_TESTS_MARK +@pytest.mark.parametrize('client_cert_file', parameter_display_status) +@pytest.mark.parametrize('ssl', parameter_display_status) +def test_s3_no_ssl_verification(s3_no_ssl_storage, client_cert_file, ssl): + uri = s3_no_ssl_storage.arctic_uri + if ssl == ParameterDisplayStatus.DISABLE: + uri += "&ssl=False" + elif ssl == ParameterDisplayStatus.ENABLE: + uri += "&ssl=True" + if client_cert_file == ParameterDisplayStatus.DISABLE: + uri += "&CA_cert_path=" + elif client_cert_file == ParameterDisplayStatus.ENABLE: + uri += f"&CA_cert_path={s3_no_ssl_storage.factory.client_cert_file}" + ac = Arctic(uri) + lib = ac.create_library("test") + lib.write("sym", pd.DataFrame()) + + +@S3_SSL_TESTS_MARK +def test_s3_ca_directory_ssl_verification(s3_storage): + uri = f"s3s://{s3_storage.factory.host}:{s3_storage.bucket}?access={s3_storage.key.id}" \ + f"&secret={s3_storage.key.secret}&CA_cert_path=&CA_cert_dir={s3_storage.factory.client_cert_dir}&ssl=True" + if s3_storage.factory.port: + uri += f"&port={s3_storage.factory.port}" + ac = Arctic(uri) + lib = ac.create_library("test") + lib.write("sym", pd.DataFrame()) + + +@S3_SSL_TESTS_MARK +def test_s3_https_backend_without_ssl_verification(s3_storage): + uri = f"s3s://{s3_storage.factory.host}:{s3_storage.bucket}?access={s3_storage.key.id}&secret={s3_storage.key.secret}&ssl=False" + if s3_storage.factory.port: + uri += f"&port={s3_storage.factory.port}" + ac = Arctic(uri) + lib = ac.create_library("test") + lib.write("sym", pd.DataFrame()) def test_basic_write_read_update_and_append(arctic_library): diff --git a/python/tests/integration/storage_fixtures/test_s3.py b/python/tests/integration/storage_fixtures/test_s3.py index b90980394e..705761c57e 100644 --- a/python/tests/integration/storage_fixtures/test_s3.py +++ b/python/tests/integration/storage_fixtures/test_s3.py @@ -5,25 +5,25 @@ def test_rate_limit(s3_storage_factory: MotoS3StorageFixtureFactory): # Don't need to create buckets # Given a working Moto server s3 = s3_storage_factory - requests.head(s3.endpoint).raise_for_status() + requests.head(s3.endpoint, verify=s3.client_cert_file).raise_for_status() # When request limiting is enabled - requests.post(s3.endpoint + "/rate_limit", b"2").raise_for_status() + requests.post(s3.endpoint + "/rate_limit", b"2", verify=s3.client_cert_file).raise_for_status() # Then the specified number of requests work - requests.head(s3.endpoint).raise_for_status() - requests.head(s3.endpoint).raise_for_status() + requests.head(s3.endpoint, verify=s3.client_cert_file).raise_for_status() + requests.head(s3.endpoint, verify=s3.client_cert_file).raise_for_status() # Then rate limit how many times you call for _ in range(3): - resp = requests.head(s3.endpoint) + resp = requests.head(s3.endpoint, verify=s3.client_cert_file) assert resp.status_code == 503 assert resp.reason == "Slow Down" # TODO: If this test fails before this point, rate limit doesn't get reset and other tests which # share the same session will fail as well # When we then reset - requests.post(s3.endpoint + "/rate_limit", b"-1").raise_for_status() + requests.post(s3.endpoint + "/rate_limit", b"-1", verify=s3.client_cert_file).raise_for_status() # Then working again - requests.head(s3.endpoint).raise_for_status() + requests.head(s3.endpoint, verify=s3.client_cert_file).raise_for_status() diff --git a/python/tests/util/mark.py b/python/tests/util/mark.py index 959691dfa9..4b2da35e3e 100644 --- a/python/tests/util/mark.py +++ b/python/tests/util/mark.py @@ -49,6 +49,17 @@ Currently controlled by the ARCTICDB_PERSISTENT_STORAGE_TESTS and ARCTICDB_FAST_TESTS_ONLY env vars.""" +"""Windows and MacOS have different handling of self-signed CA cert for test. +TODO: https://github.com/man-group/ArcticDB/issues/1394""" +S3_SSL_TEST_ENABLED = sys.platform == "linux" + + +S3_SSL_TESTS_MARK = pytest.mark.skipif( + not S3_SSL_TEST_ENABLED, + reason="Additional S3 test only when SSL is ON", +) + + def _no_op_decorator(fun): return fun diff --git a/setup.cfg b/setup.cfg index b963979058..b6e3b31484 100644 --- a/setup.cfg +++ b/setup.cfg @@ -127,6 +127,7 @@ Testing = asv virtualenv pymongo + trustme [options.entry_points] console_scripts =