diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f1ec1b..d57d120 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,14 @@ jobs: fail-fast: false matrix: python-version: ["3.8","3.9"] + target: [x86_64] steps: - - name: Checkout source - uses: actions/checkout@v4 +# - name: Checkout source + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - name: Setup Conda Environment uses: mamba-org/setup-micromamba@v1 @@ -34,15 +38,28 @@ jobs: conda list conda --version - - name: Install dependencies - run: | - pip install .[tests] + # alluxiocommon related + - name: Build alluxiocommon wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release -i python${{ matrix.python-version }} --out dist -m rust/alluxiocommon/Cargo.toml + manylinux: auto - - name: Run Alluxio FileSystem tests + - name: Install all packages and Run AlluxioCommon tests + shell: bash run: | + set -e + pip install .[tests] + pip install alluxiocommon --no-index --find-links=dist/ --force-reinstall + pip install pytest + pytest -vv \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%H:%M:%S" \ + rust/alluxiocommon/tests/ pytest -vv \ - --log-format="%(asctime)s %(levelname)s %(message)s" \ - --log-date-format="%H:%M:%S" \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%H:%M:%S" \ tests/ lint: diff --git a/alluxiofs/client/const.py b/alluxiofs/client/const.py index 898d3f0..fa3914f 100644 --- a/alluxiofs/client/const.py +++ b/alluxiofs/client/const.py @@ -12,6 +12,8 @@ ALLUXIO_WORKER_HTTP_SERVER_PORT_DEFAULT_VALUE = 28080 ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE = 5 ALLUXIO_SUCCESS_IDENTIFIER = "success" +ALLUXIO_COMMON_EXTENSION_ENABLE = "alluxio.common.extension.enable" +ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE = "alluxio.common.ondemandpool.disable" LIST_URL_FORMAT = "http://{worker_host}:{http_port}/v1/files" FULL_PAGE_URL_FORMAT = ( "http://{worker_host}:{http_port}/v1/file/{path_id}/page/{page_index}" diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py index d64d751..26b6b12 100644 --- a/alluxiofs/client/core.py +++ b/alluxiofs/client/core.py @@ -15,7 +15,16 @@ import requests from requests.adapters import HTTPAdapter +try: + from alluxiocommon import _DataManager +except ModuleNotFoundError: + print( + "[WARNING]pkg 'alluxiocommon' not installed, relative modules unable to invoke." + ) + from .const import ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE +from .const import ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE +from .const import ALLUXIO_COMMON_EXTENSION_ENABLE from .const import ALLUXIO_HASH_NODE_PER_WORKER_KEY1 from .const import ALLUXIO_HASH_NODE_PER_WORKER_KEY2 from .const import ALLUXIO_PAGE_SIZE_DEFAULT_VALUE @@ -177,6 +186,7 @@ def __init__( # parse options page_size = ALLUXIO_PAGE_SIZE_DEFAULT_VALUE hash_node_per_worker = ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE + self.data_manager = None if options: if ALLUXIO_PAGE_SIZE_KEY in options: page_size = options[ALLUXIO_PAGE_SIZE_KEY] @@ -195,6 +205,20 @@ def __init__( self.logger.debug( f"Hash node per worker is set to {hash_node_per_worker}" ) + if ( + ALLUXIO_COMMON_EXTENSION_ENABLE in options + and options[ALLUXIO_COMMON_EXTENSION_ENABLE].lower() == "true" + ): + print("Using alluxiocommon extension..") + self.logger.debug("alluxiocommon extension enabled.") + ondemand_pool_disabled = ( + ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE in options + and options[ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE].lower() + == "true" + ) + self.data_manager = _DataManager( + concurrency, ondemand_pool_disabled=ondemand_pool_disabled + ) if ( not isinstance(hash_node_per_worker, int) or hash_node_per_worker <= 0 @@ -482,11 +506,18 @@ def read(self, file_path): ) path_id = self._get_path_hash(file_path) try: - return b"".join( - self._all_page_generator( - worker_host, worker_http_port, path_id + if self.data_manager: + return b"".join( + self._all_page_generator_alluxiocommon( + worker_host, worker_http_port, path_id + ) + ) + else: + return b"".join( + self._all_page_generator( + worker_host, worker_http_port, path_id + ) ) - ) except Exception as e: raise Exception( f"Error when reading file {file_path}: error {e}" @@ -504,6 +535,7 @@ def read_range(self, file_path, offset, length): Returns: file content (str): The file content with length from offset """ + self.logger.debug(f"read_range,off:{offset}:length:{length}") self._validate_path(file_path) if not isinstance(offset, int) or offset < 0: raise ValueError("Offset must be a non-negative integer") @@ -528,15 +560,20 @@ def read_range(self, file_path, offset, length): path_id = self._get_path_hash(file_path) try: - return b"".join( - self._range_page_generator( + if self.data_manager: + return self._range_page_generator_alluxiocommon( worker_host, worker_http_port, path_id, offset, length ) - ) + else: + return b"".join( + self._range_page_generator( + worker_host, worker_http_port, path_id, offset, length + ) + ) except Exception as e: raise Exception( - f"Error when reading file {file_path}: error {e}: " - f"worker_host{worker_host}, worker_http_port:{worker_http_port}" + f"Error when reading file:{file_path}: error:{e}: " + f"worker_host:{worker_host}, worker_http_port:{worker_http_port}" ) from e def write_page(self, file_path, page_index, page_bytes): @@ -574,6 +611,39 @@ def write_page(self, file_path, page_index, page_bytes): f"Error writing to file {file_path} at page {page_index}: {e}" ) + def _all_page_generator_alluxiocommon( + self, worker_host, worker_http_port, path_id + ): + page_index = 0 + fetching_pages_num_each_round = 4 + while True: + read_urls = [] + try: + for _ in range(fetching_pages_num_each_round): + page_url = FULL_PAGE_URL_FORMAT.format( + worker_host=worker_host, + http_port=worker_http_port, + path_id=path_id, + page_index=page_index, + ) + read_urls.append(page_url) + page_index += 1 + pages_content = self.data_manager.make_multi_http_req( + read_urls + ) + yield pages_content + if ( + len(pages_content) + < fetching_pages_num_each_round * self.page_size + ): + break + except Exception as e: + # data_manager won't throw exception if there are any first few content retrieved + # hence we always propagte exception from data_manager upwards + raise Exception( + f"Error when reading all pages of {path_id}: error {e}" + ) from e + def _all_page_generator(self, worker_host, worker_http_port, path_id): page_index = 0 while True: @@ -596,6 +666,40 @@ def _all_page_generator(self, worker_host, worker_http_port, path_id): break page_index += 1 + def _range_page_generator_alluxiocommon( + self, worker_host, worker_http_port, path_id, offset, length + ): + read_urls = [] + start = offset + while start < offset + length: + page_index = start // self.page_size + inpage_off = start % self.page_size + inpage_read_len = min( + self.page_size - inpage_off, offset + length - start + ) + page_url = None + if inpage_off == 0 and inpage_read_len == self.page_size: + page_url = FULL_PAGE_URL_FORMAT.format( + worker_host=worker_host, + http_port=worker_http_port, + path_id=path_id, + page_index=page_index, + ) + else: + page_url = PAGE_URL_FORMAT.format( + worker_host=worker_host, + http_port=worker_http_port, + path_id=path_id, + page_index=page_index, + page_offset=inpage_off, + page_length=inpage_read_len, + ) + read_urls.append(page_url) + start += inpage_read_len + self.logger.debug(f"read_urls:{read_urls}") + data = self.data_manager.make_multi_http_req(read_urls) + return data + def _range_page_generator( self, worker_host, worker_http_port, path_id, offset, length ): diff --git a/alluxiofs/core.py b/alluxiofs/core.py index 90f73fa..fa7f87a 100644 --- a/alluxiofs/core.py +++ b/alluxiofs/core.py @@ -164,6 +164,7 @@ def fallback_wrapper(self, path, *args, **kwargs): path = self._strip_alluxio_protocol(path) try: if self.alluxio: + self.logger.debug(f"calling {alluxio_impl.__name__}") return alluxio_impl(self, path, *args, **kwargs) except Exception as e: if not isinstance(e, NotImplementedError): diff --git a/benchmark/AbstractBench.py b/benchmark/AbstractBench.py index 8fc1904..99f3ae3 100644 --- a/benchmark/AbstractBench.py +++ b/benchmark/AbstractBench.py @@ -79,9 +79,14 @@ def get_protocol(self, full_path: str) -> str: def init(self): # protocol = self.get_protocol(self.args.path) + alluxio_options = { + # "alluxio.common.extension.enable": "True", + "alluxio.worker.page.store.page.size": "1MB" + } self.alluxio_fs = AlluxioFileSystem( etcd_hosts=self.args.etcd_hosts, worker_hosts=self.args.worker_hosts, + options=alluxio_options, # target_protocol=protocol ) self.traverse(self.args.path) diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..e2bba63 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,36 @@ +## A PyO3 based common native extension lib for alluxio python client + +### Developer Prerequisites: +- Install Rust: +https://www.rust-lang.org/tools/install +- Install maturin: +https://www.maturin.rs/installation + + +### To build developer version locally and play: + +1) create virtualenv, (a tool used to create isolated Python environments): + + + python3 -m venv .env + source .env/bin/activate + maturin develop + +2) then can start using: + + + python3 + >>> from alluxiocommon import _DataManager + >>> dm = _DataManager(4) + >>> # do something with dm... + +### To build wheel package and install with pip: + + #in rust/alluxiocommon dir: + $ maturin build --out /dist -m /rust/alluxiocommon/Cargo.toml -i python (python version such as 3.8) + #then find .whl package in /dist: + [root@ip-XXX-XX-XX-XX alluxiofs]# ls -l dist/ + total 21848 + -rw-r--r--. 1 root root 22318133 Apr 28 05:31 alluxiocommon-0.1.0-cp38-cp38-linux_x86_64.whl + #install with pip + $ pip install dist/alluxiocommon-0.1.0-cp38-cp38-linux_x86_64.whl --force-reinstall diff --git a/rust/alluxiocommon/Cargo.lock b/rust/alluxiocommon/Cargo.lock new file mode 100644 index 0000000..b6edbcd --- /dev/null +++ b/rust/alluxiocommon/Cargo.lock @@ -0,0 +1,1485 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alluxiocommon" +version = "0.1.0" +dependencies = [ + "bytes", + "env_logger", + "log", + "pyo3", + "rayon", + "reqwest", + "tokio", +] + +[[package]] +name = "anstream" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "bumpalo" +version = "3.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "h2" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fa0ae458eb99874f54c09f4f9174f8b45fb87e854536a4e608696247f0c23" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "proc-macro2" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a8b1990bd018761768d5e608a13df8bd1ac5f678456e0f301bb93e5f3ea16b" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650dca34d463b6cdbdb02b1d71bfd6eb6b6816afc708faebb3bac1380ff4aef7" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a7da8fc04a8a2084909b59f29e1b8474decac98b951d77b80b26dc45f046ad" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8a199fce11ebb28e3569387228836ea98110e43a804a530a9fd83ade36d513" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fbbfd7eb553d10036513cb122b888dcd362a945a00b06c165f2ab480d4cc3b" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "syn" +version = "2.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "target-lexicon" +version = "0.12.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] diff --git a/rust/alluxiocommon/Cargo.toml b/rust/alluxiocommon/Cargo.toml new file mode 100644 index 0000000..b0d7671 --- /dev/null +++ b/rust/alluxiocommon/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "alluxiocommon" +version = "0.1.0" +edition = "2018" + +[lib] +name = "alluxiocommon" +# "cdylib" is necessary to produce a shared library for Python to import from. +# +# Downstream Rust code (including code in `bin/`, `examples/`, and `tests/`) will not be able +# to `use string_sum;` unless the "rlib" or "lib" crate type is also included, e.g.: +# crate-type = ["cdylib", "rlib"] +crate-type = ["cdylib"] + +[dependencies] +reqwest = { version = "0.11.16", features = ["blocking", "json", "rustls-tls"], default-features = false } +tokio = { version = "1", features = ["full"] } +bytes = "1.5.0" +rayon = "1.7.0" +env_logger = "0.11.3" +log = "0.4.21" + +[dependencies.pyo3] +version = "0.21.1" +features = ["extension-module"] diff --git a/rust/alluxiocommon/pyproject.toml b/rust/alluxiocommon/pyproject.toml new file mode 100644 index 0000000..acacf2c --- /dev/null +++ b/rust/alluxiocommon/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[tool.maturin] +compatibility = "linux" diff --git a/rust/alluxiocommon/setup.py b/rust/alluxiocommon/setup.py new file mode 100644 index 0000000..2585310 --- /dev/null +++ b/rust/alluxiocommon/setup.py @@ -0,0 +1,13 @@ +from setuptools import setup +from setuptools_rust import Binding +from setuptools_rust import RustExtension + + +setup( + name="alluxiocommon", + version="0.1", + rust_extensions=[RustExtension("alluxiocommon", binding=Binding.PyO3)], + packages=["alluxiocommon"], + # include any other necessary package metadata + zip_safe=False, +) diff --git a/rust/alluxiocommon/src/lib.rs b/rust/alluxiocommon/src/lib.rs new file mode 100644 index 0000000..ef430b4 --- /dev/null +++ b/rust/alluxiocommon/src/lib.rs @@ -0,0 +1,225 @@ +use pyo3::prelude::*; +use std::{cmp}; +use std::sync::Arc; +use reqwest::blocking::Client; +use rayon::{ThreadPool, ThreadPoolBuildError}; +use env_logger::Env; +use log::debug; + +use pyo3::{ + exceptions::PyValueError, + exceptions::PyException, + types::PyBytes, +}; + +const DEFAULT_THREADPOOL_NAME: &str = "ALLUXIOCOMMON"; + + +#[pyclass(name = "_DataManager", module = "_lib")] +pub struct DataManager { + max_threads: usize, + ondemand_pool: bool, + thread_pool: Option>, + request_client: Option>, +} + +#[pymethods] +impl DataManager { + #[new] + fn new( + max_concurrency: usize, + ondemand_pool_disabled: Option + ) -> PyResult { + debug!("instantiating _DataManager"); + match ondemand_pool_disabled { + Some(disabled) => { + if disabled { + // Create reqwest client (The Client holds a connection pool internally, create to reuse this Client obj). + let client = Client::new(); + // Create threadpool + let pool_result = + create_pool(max_concurrency, String::from(DEFAULT_THREADPOOL_NAME)); + match pool_result { + Ok(pool) => { + return Ok(Self { + max_threads: max_concurrency, + ondemand_pool: !disabled, + thread_pool: Option::Some(Arc::new(pool)), + request_client: Option::Some(Arc::new(client)), + }); + }, + Err(err) => { + return Err(PyValueError::new_err(err.to_string())); + }, + } + } else { + return Ok(Self { + max_threads: max_concurrency, + ondemand_pool: true, + thread_pool: None, + request_client: None, + }); + } + }, + None => { + return Ok(Self { + max_threads: max_concurrency, + ondemand_pool: true, + thread_pool: None, + request_client: None, + }); + } + } + } + + fn make_multi_http_req(self_: PyRef<'_, Self>, urls: Vec) -> PyResult { + let num_reqs = urls.len(); + let mut content_results = Vec::with_capacity(num_reqs); + let mut senders = Vec::with_capacity(num_reqs); + for _ in 0..num_reqs { + senders.push(None); + } + + let thread_pool /*: Arc*/ = match self_.ondemand_pool { + true => { + match create_pool(cmp::min(self_.max_threads, num_reqs), + String::from(DEFAULT_THREADPOOL_NAME)) + { + Ok(pool) => Arc::new(pool), + Err(err) => { + PyException::new_err(err.to_string()) + .restore(self_.py()); + return Err(PyErr::fetch(self_.py())); + }, + } + }, + false => { + Arc::clone(&(self_.thread_pool.as_ref().unwrap())) // it can't be None here once instantiated + } + }; + + let request_client = match self_.ondemand_pool { + true => { + Arc::new(Client::new()) + }, + false => { + Arc::clone(&(self_.request_client.as_ref().unwrap())) + } + }; + for i in 0..num_reqs { + let url_owned = urls[i].to_owned(); + let client_clone = Arc::clone(&(request_client)); + let (send, recv) = tokio::sync::oneshot::channel(); + let install_res = thread_pool.install(move || -> Result<(), reqwest::Error> { + let body = perform_http_get(url_owned.as_str(), client_clone.as_ref()); + send.send(body).unwrap(); + Ok(()) + }); + match install_res { + Ok(_success) => {}, + Err(err) => { + PyException::new_err(err.to_string()) + .restore(self_.py()); + return Err(PyErr::fetch(self_.py())); + }, + } + senders[i] = Some(recv); + } + + for sender in senders { + let result = sender.unwrap().blocking_recv(); + match result { + Ok(content_result) => { + content_results.push(content_result); + }, + Err(err) => { + PyException::new_err(err.to_string()) + .restore(self_.py()); + return Err(PyErr::fetch(self_.py())); + } + } + } + let mut concatenated_data: Vec = Vec::new(); + let mut somedata_read: bool = false; + for content_result in &content_results { + match content_result { + Ok(content) => { + concatenated_data.extend(content); + somedata_read = true; + }, + Err(err) => { + if somedata_read { + break; + } + let err_str = err.to_string(); + PyException::new_err(format!("Error in getting result, {}", err_str)) + .restore(self_.py()); + return Err(PyErr::fetch(self_.py())); + } + } + } + Ok(PyBytes::new(self_.py(), &concatenated_data).to_object(self_.py())) + } +} + +fn perform_http_get(url: &str, client: &Client) -> Result, reqwest::Error> { + let bytes = client.get(url).send()? + .bytes()?; + let bytes_vec = bytes.to_vec(); // TODO! avoid copy here at least, one more additional copy in returning to py world + Ok(bytes_vec) +} + +fn create_pool(num_threads: usize, thread_name_prefix: String) -> Result> { + let name_prefix = thread_name_prefix.clone(); + match rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .thread_name(move |i| format!("{}-{}", &name_prefix, i)) + .build() + { + Err(e) => Err(Box::new(e.into())), + Ok(pool) => Ok(pool), + } +} + +#[cfg(test)] // Indicates that the following functions are only compiled when running tests +mod tests { + use super::*; + use log::info; + + fn init() { + let _ = env_logger::builder().is_test(true).try_init(); + } + + #[test] + fn test_logger() { + init(); + info!("Test this record will be captured by `cargo test`"); + } + + #[test] + fn test_perform_http_get() { + init(); + let client = Client::new(); + let res = perform_http_get("http://127.0.0.1:1000", &client); + match res { + Ok(_success) => { + print!("SUCCESS!"); + }, + Err(e) => { + print!("error!:{}", e); + } + } + print!("test_example"); + } +} + + +#[pymodule] +fn alluxiocommon(_py: Python, m: &PyModule) -> PyResult<()> { + let env = Env::new() + .filter_or("ALLUXIOCOMMON_LOG", "warn"); + env_logger::init_from_env(env); + + m.add_class::()?; + Ok(()) +} diff --git a/rust/alluxiocommon/tests/test_datamanager.py b/rust/alluxiocommon/tests/test_datamanager.py new file mode 100644 index 0000000..bac153a --- /dev/null +++ b/rust/alluxiocommon/tests/test_datamanager.py @@ -0,0 +1,10 @@ +def test_import_alluxiocommon(): + try: + from alluxiocommon import _DataManager + + dm = _DataManager(4) + print(f"alluxiocommon._DataManager:{dm} instantiated!") + except Exception as e: + assert ( + False + ), f"Unexpected exception when importing alluxiocommon package:{e}" diff --git a/tests/client/test_read_range_docker.py b/tests/client/test_read_docker.py similarity index 72% rename from tests/client/test_read_range_docker.py rename to tests/client/test_read_docker.py index 7389897..1e4bf25 100644 --- a/tests/client/test_read_range_docker.py +++ b/tests/client/test_read_docker.py @@ -1,5 +1,6 @@ import os import random +from hashlib import md5 from alluxiofs import AlluxioClient from tests.conftest import ALLUXIO_FILE_PATH @@ -12,6 +13,13 @@ LOGGER = logging.getLogger(__name__) +def _get_md5(payload): + m = md5() + m.update(payload) + m.update(payload) + return m.hexdigest() + + def validate_read_range( alluxio_client: AlluxioClient, alluxio_file_path, @@ -64,6 +72,31 @@ def validate_invalid_read_range( ) +def validate_full_read( + alluxio_client: AlluxioClient, + alluxio_file_path, + local_file_path, +): + alluxio_data = alluxio_client.read(alluxio_file_path) + + with open(local_file_path, "rb") as local_file: + local_data = local_file.read() + + try: + assert alluxio_data == local_data + except AssertionError: + md5_alluxio_data = _get_md5(alluxio_data) + md5_local_data = _get_md5(local_data) + error_message = ( + f"Data mismatch between Alluxio and local file\n" + f"Alluxio file path: {alluxio_file_path}\n" + f"Local file path: {local_file_path}\n" + f"Alluxio data md5: {md5_alluxio_data}, length:{len(alluxio_data)}\n" + f"Local data md5: {md5_local_data}, length:{len(local_data)}" + ) + raise AssertionError(error_message) + + def test_alluxio_client(alluxio_client: AlluxioClient): file_size = os.path.getsize(LOCAL_FILE_PATH) assert alluxio_client.load(ALLUXIO_FILE_PATH, 200) @@ -111,6 +144,16 @@ def test_alluxio_client(alluxio_client: AlluxioClient): ) LOGGER.debug("Passed corner test cases") + # test full data read + + validate_full_read(alluxio_client, ALLUXIO_FILE_PATH, LOCAL_FILE_PATH) + def test_etcd_alluxio_client(etcd_alluxio_client: AlluxioClient): test_alluxio_client(etcd_alluxio_client) + + +def test_alluxio_client_alluxiocommon( + alluxio_client_alluxiocommon: AlluxioClient, +): + test_alluxio_client(alluxio_client_alluxiocommon) diff --git a/tests/client/test_read_range.py b/tests/client/test_read_range.py index f1d949f..55c6e8c 100644 --- a/tests/client/test_read_range.py +++ b/tests/client/test_read_range.py @@ -3,6 +3,8 @@ import random from alluxiofs import AlluxioClient +from alluxiofs.client.const import ALLUXIO_COMMON_EXTENSION_ENABLE +from alluxiofs.client.const import ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE def parse_args(): @@ -35,6 +37,19 @@ def parse_args(): required=False, help="The total number of read range test to run", ) + parser.add_argument( + "--enable-alluxiocommon", + type=bool, + default=False, + help="To enable alluxiocommon extension", + ) + parser.add_argument( + "--disable-alluxiocommon-ondemandpool", + type=bool, + default=False, + help="To disable alluxiocommon ondemand pool, " + "effective when --enable-alluxiocommon is enabeld", + ) return parser.parse_args() @@ -87,7 +102,13 @@ def manual_test_invalid_read_range( def main(args): - alluxio_client = AlluxioClient(etcd_hosts=args.etcd_hosts) + options = {} + if args.enable_alluxiocommon: + options[ALLUXIO_COMMON_EXTENSION_ENABLE] = "True" + if args.disable_alluxiocommon_ondemandpool: + options[ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE] = "True" + + alluxio_client = AlluxioClient(etcd_hosts=args.etcd_hosts, options=options) file_size = os.path.getsize(args.local_file_path) invalid_test_cases = [(-1, 100), (file_size - 1, -2)] diff --git a/tests/conftest.py b/tests/conftest.py index 97f3729..c29c53b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -153,6 +153,21 @@ def alluxio_client(docker_alluxio): yield alluxio_client +@pytest.fixture +def alluxio_client_alluxiocommon(docker_alluxio): + LOGGER.debug( + f"get AlluxioClient with alluxiocommon connect to {docker_alluxio}" + ) + parsed_url = urlparse(docker_alluxio) + host = parsed_url.hostname + port = parsed_url.port + alluxio_options = {"alluxio.common.extension.enable": "True"} + alluxio_client = AlluxioClient( + worker_hosts=host, worker_http_port=port, options=alluxio_options + ) + yield alluxio_client + + @pytest.fixture def etcd_alluxio_client(docker_alluxio_with_etcd): LOGGER.debug( @@ -179,6 +194,26 @@ def alluxio_file_system(docker_alluxio): yield alluxio_file_system +@pytest.fixture +def alluxio_file_system_alluxiocommon(docker_alluxio): + LOGGER.debug( + f"get AlluxioFileSystem connect to {docker_alluxio}" + f" with alluxiocommon enabled." + ) + parsed_url = urlparse(docker_alluxio) + host = parsed_url.hostname + fsspec.register_implementation("alluxio", AlluxioFileSystem, clobber=True) + alluxio_options = {"alluxio.common.extension.enable": "True"} + alluxio_file_system = fsspec.filesystem( + "alluxio", + worker_hosts=host, + target_protocol="file", + options=alluxio_options, + preload_path=ALLUXIO_FILE_PATH, + ) + yield alluxio_file_system + + @pytest.fixture def etcd_alluxio_file_system(docker_alluxio_with_etcd): LOGGER.debug( diff --git a/tests/fs/test_docker_alluxio_fsspec.py b/tests/fs/test_docker_alluxio_fsspec.py index 5445974..b1c27e5 100644 --- a/tests/fs/test_docker_alluxio_fsspec.py +++ b/tests/fs/test_docker_alluxio_fsspec.py @@ -5,5 +5,9 @@ def test_simple_fsspec(alluxio_file_system): alluxio_file_system.ls(TEST_ROOT) # no error +def test_simple_fsspec_alluxiocommon(alluxio_file_system_alluxiocommon): + alluxio_file_system_alluxiocommon.ls(TEST_ROOT) + + def test_simple_etcd_fsspec(etcd_alluxio_file_system): etcd_alluxio_file_system.ls(TEST_ROOT) # no error