From 22c7f42a0c8a272f6a7fc52b41ce6410591a02de Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 12 Apr 2024 15:01:44 -0700 Subject: [PATCH 01/34] Port changes from https://github.com/lucyge2022/alluxiofs/tree/multithread 5f80059 small fixes for REST and metrics 1c426c2 add access debug logs for all fsspec ops & add quick sanity unit tests for read 7dcb833 checkin multi process http benchmark for worker http server 4b4ebce use alluxiocommon pyo3 package to try to do multithread http req 523f748 add rust pyo3 extension 8979d2b remove first 0367659 add related files 4d0eebe small fixes a0519b3 if any error on any thread running result, raise exception back to python directly --- alluxiofs/client/const.py | 1 + benchmark/AbstractBench.py | 4 + rust/README.md | 15 + rust/alluxiocommon/Cargo.lock | 1356 +++++++++++++++++++++++++++++ rust/alluxiocommon/Cargo.toml | 24 + rust/alluxiocommon/pyproject.toml | 6 + rust/alluxiocommon/setup.py | 12 + rust/alluxiocommon/src/lib.rs | 171 ++++ rust/alluxiocommon/tests/tests.py | 63 ++ rust/ray-e2e-report.md | 12 + 10 files changed, 1664 insertions(+) create mode 100644 rust/README.md create mode 100644 rust/alluxiocommon/Cargo.lock create mode 100644 rust/alluxiocommon/Cargo.toml create mode 100644 rust/alluxiocommon/pyproject.toml create mode 100644 rust/alluxiocommon/setup.py create mode 100644 rust/alluxiocommon/src/lib.rs create mode 100644 rust/alluxiocommon/tests/tests.py create mode 100644 rust/ray-e2e-report.md diff --git a/alluxiofs/client/const.py b/alluxiofs/client/const.py index 898d3f0..fc74f4d 100644 --- a/alluxiofs/client/const.py +++ b/alluxiofs/client/const.py @@ -12,6 +12,7 @@ ALLUXIO_WORKER_HTTP_SERVER_PORT_DEFAULT_VALUE = 28080 ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE = 5 ALLUXIO_SUCCESS_IDENTIFIER = "success" +ALLUXIO_COMMON_EXTENSION_ENABLE = "alluxio.common.extension.enable" LIST_URL_FORMAT = "http://{worker_host}:{http_port}/v1/files" FULL_PAGE_URL_FORMAT = ( "http://{worker_host}:{http_port}/v1/file/{path_id}/page/{page_index}" diff --git a/benchmark/AbstractBench.py b/benchmark/AbstractBench.py index 8fc1904..deea5af 100644 --- a/benchmark/AbstractBench.py +++ b/benchmark/AbstractBench.py @@ -79,9 +79,13 @@ def get_protocol(self, full_path: str) -> str: def init(self): # protocol = self.get_protocol(self.args.path) + alluxio_options = { + "alluxio.common.extension.enable": "True", + "alluxio.worker.page.store.page.size" : "1MB"} self.alluxio_fs = AlluxioFileSystem( etcd_hosts=self.args.etcd_hosts, worker_hosts=self.args.worker_hosts, + options=alluxio_options, # target_protocol=protocol ) self.traverse(self.args.path) diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..2bf7a04 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,15 @@ +## A common native lib for alluxio python client +to build and play: + +create virtualenv, (a tool used to create isolated Python environments): + + python3 -m venv .env + source .env/bin/activate + maturin develop + +then can do: + + python3 + >>> import alluxiocommon + >>> alluxiocommon.multi_http_requests(["http://google.com"],[(0,0)]) + diff --git a/rust/alluxiocommon/Cargo.lock b/rust/alluxiocommon/Cargo.lock new file mode 100644 index 0000000..7e47f0c --- /dev/null +++ b/rust/alluxiocommon/Cargo.lock @@ -0,0 +1,1356 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "alluxiocommon" +version = "0.1.0" +dependencies = [ + "bytes", + "pyo3", + "rayon", + "reqwest", + "tokio", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "bumpalo" +version = "3.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff69b9dd49fd426c69a0db9fc04dd934cdb6645ff000864d98f7e2af8830eaa" + +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + +[[package]] +name = "cc" +version = "1.0.90" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "h2" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fa0ae458eb99874f54c09f4f9174f8b45fb87e854536a4e608696247f0c23" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "2.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memchr" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.48.5", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + +[[package]] +name = "proc-macro2" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a8b1990bd018761768d5e608a13df8bd1ac5f678456e0f301bb93e5f3ea16b" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650dca34d463b6cdbdb02b1d71bfd6eb6b6816afc708faebb3bac1380ff4aef7" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a7da8fc04a8a2084909b59f29e1b8474decac98b951d77b80b26dc45f046ad" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8a199fce11ebb28e3569387228836ea98110e43a804a530a9fd83ade36d513" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fbbfd7eb553d10036513cb122b888dcd362a945a00b06c165f2ab480d4cc3b" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "syn" +version = "2.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "target-lexicon" +version = "0.12.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] diff --git a/rust/alluxiocommon/Cargo.toml b/rust/alluxiocommon/Cargo.toml new file mode 100644 index 0000000..0d9ffee --- /dev/null +++ b/rust/alluxiocommon/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "alluxiocommon" +version = "0.1.0" +edition = "2018" + +[lib] +name = "alluxiocommon" +# "cdylib" is necessary to produce a shared library for Python to import from. +# +# Downstream Rust code (including code in `bin/`, `examples/`, and `tests/`) will not be able +# to `use string_sum;` unless the "rlib" or "lib" crate type is also included, e.g.: +# crate-type = ["cdylib", "rlib"] +crate-type = ["cdylib"] + +[dependencies] +reqwest = { version = "0.11", features = ["blocking"] } +tokio = { version = "1", features = ["full"] } +bytes = "1.5.0" +rayon = "1.7.0" + +[dependencies.pyo3] +version = "0.21.1" +features = ["extension-module"] + diff --git a/rust/alluxiocommon/pyproject.toml b/rust/alluxiocommon/pyproject.toml new file mode 100644 index 0000000..acacf2c --- /dev/null +++ b/rust/alluxiocommon/pyproject.toml @@ -0,0 +1,6 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[tool.maturin] +compatibility = "linux" diff --git a/rust/alluxiocommon/setup.py b/rust/alluxiocommon/setup.py new file mode 100644 index 0000000..9d8c5d5 --- /dev/null +++ b/rust/alluxiocommon/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup +from setuptools_rust import Binding, RustExtension + + +setup( + name="alluxiocommon", + version="0.1", + rust_extensions=[RustExtension("alluxiocommon", binding=Binding.PyO3)], + packages=["alluxiocommon"], + # include any other necessary package metadata + zip_safe=False, +) diff --git a/rust/alluxiocommon/src/lib.rs b/rust/alluxiocommon/src/lib.rs new file mode 100644 index 0000000..a6f0b11 --- /dev/null +++ b/rust/alluxiocommon/src/lib.rs @@ -0,0 +1,171 @@ +use pyo3::prelude::*; +use pyo3::types::PyList; +use std::{thread,time}; +use std::error::Error; +use std::sync::Arc; +use bytes::Bytes; +use tokio::runtime::{Builder, Runtime}; +use reqwest::blocking; +use reqwest::blocking::Client; +use rayon::{ThreadPool, ThreadPoolBuilder, ThreadPoolBuildError}; + +use pyo3::{ + exceptions::PyIOError, + exceptions::PyIndexError, + exceptions::PyValueError, + exceptions::PyTypeError, + exceptions::PyException, + prelude::{pymodule, PyModule, PyResult, Python}, + types::PyBytes, + buffer::PyBuffer, + PyErr, +}; + + +#[pyclass(name = "_DataManager", module = "_lib")] +pub struct DataManager { + // #[pyo3(get)] + thread_num: usize, + thread_pool: Arc, + request_client: Arc, +} + +#[pymethods] +impl DataManager { + #[new] + fn new( + num_threads: usize + ) -> PyResult { + // Create reqwest client (The Client holds a connection pool internally, create to reuse this Client obj). + let client = Client::new(); + // Create threadpool + let pool_result = create_pool(num_threads); + match pool_result { + Ok(pool) => { + Ok(Self { + thread_num: num_threads, + thread_pool: Arc::new(pool), + request_client: Arc::new(client), + }) + }, + Err(err) => Err(PyValueError::new_err(err.to_string())), + } + } + + fn make_multi_http_req(self_: PyRef<'_, Self>, urls: Vec) -> PyResult { + let num_reqs = urls.len(); + let mut content_results = Vec::with_capacity(num_reqs); + let mut senders = Vec::with_capacity(num_reqs); + for _ in 0..num_reqs { + senders.push(None); + } + + for i in 0..num_reqs { + let url_owned = urls[i].to_owned(); + let client_clone = Arc::clone(&(self_.request_client)); + let (send, recv) = tokio::sync::oneshot::channel(); + let install_res = self_.thread_pool.install(move || -> Result<(), reqwest::Error> { + // println!("request idx:{}", i); + let body = perform_http_get(url_owned.as_str(), client_clone.as_ref()); + send.send(body).unwrap(); + Ok(()) + }); + match install_res { + Ok(success) => { + //DO NOTHING + }, + Err(err) => { + PyException::new_err(err.to_string()) + .restore(self_.py()); + return Err(PyErr::fetch(self_.py())); + }, + } + senders[i] = Some(recv); + } + for sender in senders { + let result = sender.unwrap().blocking_recv(); + match result { + Ok(content_result) => { + content_results.push(content_result); + }, + Err(err) => { + PyException::new_err(err.to_string()) + .restore(self_.py()); + return Err(PyErr::fetch(self_.py())); + } + } + // let result = sender.unwrap().blocking_recv().unwrap(); + // content_results.push(result); + } + let mut concatenated_data: Vec = Vec::new(); + for content_result in &content_results { + match content_result { + Ok(content) => { + concatenated_data.extend(content); + }, + Err(err) => { + let err_str = err.to_string(); + PyException::new_err(format!("Error in getting result, {}", err_str)) + .restore(self_.py()); + return Err(PyErr::fetch(self_.py())); + } + } + } + Ok(PyBytes::new(self_.py(), &concatenated_data).to_object(self_.py())) + } +} + +fn type_name_of_val(_: T) -> &'static str { + std::any::type_name::() +} + +fn perform_http_get(url: &str, client: &Client) -> Result, reqwest::Error> { + let bytes = client.get(url).send()? + .bytes()?; + let bytes_vec = bytes.to_vec(); // TODO! avoid copy here at least, one more additional copy in returning to py world + Ok(bytes_vec) +} + +fn create_pool(num_threads: usize) -> Result> { + match rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build() + { + Err(e) => Err(Box::new(e.into())), + Ok(pool) => Ok(pool), + } +} + +#[cfg(test)] // Indicates that the following functions are only compiled when running tests +mod tests { + use super::*; + use super::PyBuffer; + use crate::Python; + + #[test] + fn test_example() { + let client = Client::new(); + let res = perform_http_get("http://127.0.0.1:1000", &client); + match res { + Ok(success) => { + println!("SUCCESS!"); + }, + Err(e) => { + println!("error!:{}", e); + } + } + print!("test_example"); + } +} + + +#[pymodule] +fn alluxiocommon(_py: Python, m: &PyModule) -> PyResult<()> { + // let env = Env::new() + // .filter_or("ALLUXIOCOMMON_LOG", "warn") + // .write_style("ALLUXIOCOMMON_LOG_STYLE"); + // env_logger::init_from_env(env); + + m.add_class::()?; + Ok(()) +} diff --git a/rust/alluxiocommon/tests/tests.py b/rust/alluxiocommon/tests/tests.py new file mode 100644 index 0000000..54a1a71 --- /dev/null +++ b/rust/alluxiocommon/tests/tests.py @@ -0,0 +1,63 @@ +from alluxiocommon import _DataManager +from hashlib import md5 +import time + +KB = 1024 +MB = 1024 * KB + +FULL_PAGE_URL_FORMAT = ( + "http://{worker_host}:{http_port}/v1/file/{path_id}/page/{page_index}" +) +PAGE_URL_FORMAT = "http://{worker_host}:{http_port}/v1/file/{path_id}/page/{page_index}?offset={page_offset}&length={page_length}" + + +class TestSuite: + + def __init__(self): + self.page_size = 32 * MB + self.worker_host = "172.31.45.56" + self.worker_http_port = 28080 + self.dm = _DataManager(4) + + def multi_http_requests(self): + offset, length = 0,40 * MB + read_urls = [] + path_id = "addb83ae5f2025ec8f88b832c40177327ccd6aa2486712f34e68c3efbb2045e7" + start = offset + while start < offset + length: + page_index = start // self.page_size + inpage_off = start % self.page_size + inpage_read_len = min(self.page_size - inpage_off, length - start) + page_url = None + if inpage_off == 0 and inpage_read_len == self.page_size: + page_url = FULL_PAGE_URL_FORMAT.format( + worker_host=self.worker_host, + http_port=self.worker_http_port, + path_id=path_id, + page_index=page_index, + ) + else: + page_url = PAGE_URL_FORMAT.format( + worker_host=self.worker_host, + http_port=self.worker_http_port, + path_id=path_id, + page_index=page_index, + page_offset=inpage_off, + page_length=inpage_read_len, + ) + read_urls.append(page_url) + start += inpage_read_len + data = self.dm.make_multi_http_req(read_urls) + print(f"type:{type(data)}") + md5_instance = md5() + md5_instance.update(data) + print(f"data len:{len(data)}, md5:{md5_instance.hexdigest()}") + time.sleep(30) + +def main(): + ts = TestSuite() + for _ in range(1): + ts.multi_http_requests() + +if __name__ == "__main__": + main() diff --git a/rust/ray-e2e-report.md b/rust/ray-e2e-report.md new file mode 100644 index 0000000..07e5115 --- /dev/null +++ b/rust/ray-e2e-report.md @@ -0,0 +1,12 @@ +Before optimization + +| Test | Benchmark | Local FileSystem(nvme) | Local region S3 | AlluxioFsSpec | AlluxioFsSpec PyO3 optimized | +|------------------------|-----------|-------------------------|----------------|-----------------|------------------------------| +| Ray + XgBoost(10G) | | | | PyTorch | +| Ray + PyTorch(parquet) | | | | CosmoFlow | Tensorflow | +| Ray + PyTorch(image) | | | Forces mean absolute error 0.036 | DimeNet++ | PyTorch | +| | | | 0.8 Local Distance Difference Test (lDDT) | AlphaFold2 (PyTorch) | PyTorch | + + +After optimization + From 814629d9d861236dc500f65397734f001652f3ab Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 12 Apr 2024 15:31:34 -0700 Subject: [PATCH 02/34] missing changes from port --- alluxiofs/client/core.py | 50 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py index d64d751..6c7d91d 100644 --- a/alluxiofs/client/core.py +++ b/alluxiofs/client/core.py @@ -14,8 +14,10 @@ import humanfriendly import requests from requests.adapters import HTTPAdapter +from alluxiocommon import _DataManager from .const import ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE +from .const import ALLUXIO_COMMON_EXTENSION_ENABLE from .const import ALLUXIO_HASH_NODE_PER_WORKER_KEY1 from .const import ALLUXIO_HASH_NODE_PER_WORKER_KEY2 from .const import ALLUXIO_PAGE_SIZE_DEFAULT_VALUE @@ -177,6 +179,7 @@ def __init__( # parse options page_size = ALLUXIO_PAGE_SIZE_DEFAULT_VALUE hash_node_per_worker = ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE + self.data_manager = None if options: if ALLUXIO_PAGE_SIZE_KEY in options: page_size = options[ALLUXIO_PAGE_SIZE_KEY] @@ -195,6 +198,11 @@ def __init__( self.logger.debug( f"Hash node per worker is set to {hash_node_per_worker}" ) + if ALLUXIO_COMMON_EXTENSION_ENABLE in options: + self.logger.debug( + "alluxiocommon extension enabled." + ) + self.data_manager = _DataManager(concurrency) if ( not isinstance(hash_node_per_worker, int) or hash_node_per_worker <= 0 @@ -528,11 +536,16 @@ def read_range(self, file_path, offset, length): path_id = self._get_path_hash(file_path) try: - return b"".join( - self._range_page_generator( + if self.data_manager: + return self._multithread_range_page_generator( worker_host, worker_http_port, path_id, offset, length ) - ) + else: + return b"".join( + self._range_page_generator( + worker_host, worker_http_port, path_id, offset, length + ) + ) except Exception as e: raise Exception( f"Error when reading file {file_path}: error {e}: " @@ -596,6 +609,37 @@ def _all_page_generator(self, worker_host, worker_http_port, path_id): break page_index += 1 + def _multithread_range_page_generator( + self, worker_host, worker_http_port, path_id, offset, length + ): + read_urls = [] + start = offset + while start < offset + length: + page_index = start // self.page_size + inpage_off = start % self.page_size + inpage_read_len = min(self.page_size - inpage_off, length - start) + page_url = None + if inpage_off == 0 and inpage_read_len == self.page_size: + page_url = FULL_PAGE_URL_FORMAT.format( + worker_host=worker_host, + http_port=worker_http_port, + path_id=path_id, + page_index=page_index, + ) + else: + page_url = PAGE_URL_FORMAT.format( + worker_host=worker_host, + http_port=worker_http_port, + path_id=path_id, + page_index=page_index, + page_offset=inpage_off, + page_length=inpage_read_len, + ) + read_urls.append(page_url) + start += inpage_read_len + data = self.data_manager.make_multi_http_req(read_urls) + return data + def _range_page_generator( self, worker_host, worker_http_port, path_id, offset, length ): From 8913fa885abb8a032a0f3206177b68533cd7e387 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Thu, 18 Apr 2024 13:16:38 -0700 Subject: [PATCH 03/34] fix the bug of inpage_read_len calculation --- alluxiofs/client/core.py | 4 +++- alluxiofs/core.py | 1 + benchmark/AbstractBench.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py index 6c7d91d..b2e25c7 100644 --- a/alluxiofs/client/core.py +++ b/alluxiofs/client/core.py @@ -199,6 +199,7 @@ def __init__( f"Hash node per worker is set to {hash_node_per_worker}" ) if ALLUXIO_COMMON_EXTENSION_ENABLE in options: + print(f"Using alluxiocommon extension, concurrency:{concurrency}") self.logger.debug( "alluxiocommon extension enabled." ) @@ -512,6 +513,7 @@ def read_range(self, file_path, offset, length): Returns: file content (str): The file content with length from offset """ + self.logger.debug(f"read_range,off:{offset}:length:{length}") self._validate_path(file_path) if not isinstance(offset, int) or offset < 0: raise ValueError("Offset must be a non-negative integer") @@ -617,7 +619,7 @@ def _multithread_range_page_generator( while start < offset + length: page_index = start // self.page_size inpage_off = start % self.page_size - inpage_read_len = min(self.page_size - inpage_off, length - start) + inpage_read_len = min(self.page_size - inpage_off, offset + length - start) page_url = None if inpage_off == 0 and inpage_read_len == self.page_size: page_url = FULL_PAGE_URL_FORMAT.format( diff --git a/alluxiofs/core.py b/alluxiofs/core.py index 90f73fa..fa7f87a 100644 --- a/alluxiofs/core.py +++ b/alluxiofs/core.py @@ -164,6 +164,7 @@ def fallback_wrapper(self, path, *args, **kwargs): path = self._strip_alluxio_protocol(path) try: if self.alluxio: + self.logger.debug(f"calling {alluxio_impl.__name__}") return alluxio_impl(self, path, *args, **kwargs) except Exception as e: if not isinstance(e, NotImplementedError): diff --git a/benchmark/AbstractBench.py b/benchmark/AbstractBench.py index deea5af..76246b4 100644 --- a/benchmark/AbstractBench.py +++ b/benchmark/AbstractBench.py @@ -80,7 +80,7 @@ def get_protocol(self, full_path: str) -> str: def init(self): # protocol = self.get_protocol(self.args.path) alluxio_options = { - "alluxio.common.extension.enable": "True", + # "alluxio.common.extension.enable": "True", "alluxio.worker.page.store.page.size" : "1MB"} self.alluxio_fs = AlluxioFileSystem( etcd_hosts=self.args.etcd_hosts, From e321c846b8b5f2d9c622a59a13b82cd5a287aab6 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Tue, 23 Apr 2024 14:38:02 -0700 Subject: [PATCH 04/34] make ondemand pool default and enable it by flag during instantiation --- rust/alluxiocommon/Cargo.lock | 129 +++++++++++++++++++++++++++++ rust/alluxiocommon/Cargo.toml | 2 + rust/alluxiocommon/src/lib.rs | 132 +++++++++++++++++++++++------- rust/alluxiocommon/tests/tests.py | 4 +- 4 files changed, 237 insertions(+), 30 deletions(-) diff --git a/rust/alluxiocommon/Cargo.lock b/rust/alluxiocommon/Cargo.lock index 7e47f0c..b6edbcd 100644 --- a/rust/alluxiocommon/Cargo.lock +++ b/rust/alluxiocommon/Cargo.lock @@ -17,17 +17,76 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "alluxiocommon" version = "0.1.0" dependencies = [ "bytes", + "env_logger", + "log", "pyo3", "rayon", "reqwest", "tokio", ] +[[package]] +name = "anstream" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -91,6 +150,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "core-foundation" version = "0.9.4" @@ -147,6 +212,29 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -324,6 +412,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.28" @@ -733,6 +827,35 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" + [[package]] name = "reqwest" version = "0.11.27" @@ -1116,6 +1239,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/rust/alluxiocommon/Cargo.toml b/rust/alluxiocommon/Cargo.toml index 0d9ffee..d20d22e 100644 --- a/rust/alluxiocommon/Cargo.toml +++ b/rust/alluxiocommon/Cargo.toml @@ -17,6 +17,8 @@ reqwest = { version = "0.11", features = ["blocking"] } tokio = { version = "1", features = ["full"] } bytes = "1.5.0" rayon = "1.7.0" +env_logger = "0.11.3" +log = "0.4.21" [dependencies.pyo3] version = "0.21.1" diff --git a/rust/alluxiocommon/src/lib.rs b/rust/alluxiocommon/src/lib.rs index a6f0b11..fc9ad50 100644 --- a/rust/alluxiocommon/src/lib.rs +++ b/rust/alluxiocommon/src/lib.rs @@ -1,6 +1,6 @@ use pyo3::prelude::*; use pyo3::types::PyList; -use std::{thread,time}; +use std::{cmp, thread, time}; use std::error::Error; use std::sync::Arc; use bytes::Bytes; @@ -8,6 +8,7 @@ use tokio::runtime::{Builder, Runtime}; use reqwest::blocking; use reqwest::blocking::Client; use rayon::{ThreadPool, ThreadPoolBuilder, ThreadPoolBuildError}; +use env_logger::Env; use pyo3::{ exceptions::PyIOError, @@ -21,38 +22,68 @@ use pyo3::{ PyErr, }; +const DEFAULT_THREADPOOL_NAME: &str = "ALLUXIOCOMMON"; + #[pyclass(name = "_DataManager", module = "_lib")] pub struct DataManager { - // #[pyo3(get)] - thread_num: usize, - thread_pool: Arc, - request_client: Arc, + max_threads: usize, + ondemand_pool: bool, + thread_pool: Option>, + request_client: Option>, } #[pymethods] impl DataManager { #[new] fn new( - num_threads: usize + max_concurrency: usize, + ondemand_pool: Option ) -> PyResult { - // Create reqwest client (The Client holds a connection pool internally, create to reuse this Client obj). - let client = Client::new(); - // Create threadpool - let pool_result = create_pool(num_threads); - match pool_result { - Ok(pool) => { - Ok(Self { - thread_num: num_threads, - thread_pool: Arc::new(pool), - request_client: Arc::new(client), - }) + match ondemand_pool { + Some(is_ondemand_pool) => { + if (!is_ondemand_pool) { + // Create reqwest client (The Client holds a connection pool internally, create to reuse this Client obj). + let client = Client::new(); + // Create threadpool + let pool_result = + create_pool(max_concurrency, String::from(DEFAULT_THREADPOOL_NAME)); + match pool_result { + Ok(pool) => { + return Ok(Self { + max_threads: max_concurrency, + ondemand_pool: is_ondemand_pool, + thread_pool: Option::Some(Arc::new(pool)), + request_client: Option::Some(Arc::new(client)), + }); + }, + Err(err) => { + return Err(PyValueError::new_err(err.to_string())); + }, + } + } else { + return Ok(Self { + max_threads: max_concurrency, + ondemand_pool: true, + thread_pool: None, + request_client: None, + }); + } }, - Err(err) => Err(PyValueError::new_err(err.to_string())), + None => { + return Ok(Self { + max_threads: max_concurrency, + ondemand_pool: true, + thread_pool: None, + request_client: None, + }); + } } + println!("instantiate _DataManager"); } fn make_multi_http_req(self_: PyRef<'_, Self>, urls: Vec) -> PyResult { + println!("on demand pool and client!"); let num_reqs = urls.len(); let mut content_results = Vec::with_capacity(num_reqs); let mut senders = Vec::with_capacity(num_reqs); @@ -60,18 +91,49 @@ impl DataManager { senders.push(None); } + // let Some(kmeans) = self.trained_kmeans.as_ref() else { + // return Err(PyRuntimeError::new_err("KMeans must fit (train) first")); + // }; + + let thread_pool /*: Arc*/ = match self_.ondemand_pool { + true => { + match create_pool(cmp::min(self_.max_threads, num_reqs), + String::from(DEFAULT_THREADPOOL_NAME)) + { + Ok(pool) => Arc::new(pool), + Err(err) => { + PyException::new_err(err.to_string()) + .restore(self_.py()); + return Err(PyErr::fetch(self_.py())); + }, + } + }, + false => { + Arc::clone(&(self_.thread_pool.as_ref().unwrap())) // it can't be None here once instantiated + } + }; + + let request_client = match self_.ondemand_pool { + true => { + Arc::new(Client::new()) + }, + false => { + Arc::clone(&(self_.request_client.as_ref().unwrap())) + } + }; for i in 0..num_reqs { let url_owned = urls[i].to_owned(); - let client_clone = Arc::clone(&(self_.request_client)); + // let client_clone = Arc::clone(&(self_.request_client)); + let client_clone = Arc::clone(&(request_client)); let (send, recv) = tokio::sync::oneshot::channel(); - let install_res = self_.thread_pool.install(move || -> Result<(), reqwest::Error> { + let install_res = thread_pool.install(move || -> Result<(), reqwest::Error> { // println!("request idx:{}", i); let body = perform_http_get(url_owned.as_str(), client_clone.as_ref()); send.send(body).unwrap(); Ok(()) }); match install_res { - Ok(success) => { + Ok(_success) => { //DO NOTHING }, Err(err) => { @@ -126,9 +188,11 @@ fn perform_http_get(url: &str, client: &Client) -> Result, reqwest::Erro Ok(bytes_vec) } -fn create_pool(num_threads: usize) -> Result> { +fn create_pool(num_threads: usize, thread_name_prefix: String) -> Result> { + let name_prefix = thread_name_prefix.clone(); match rayon::ThreadPoolBuilder::new() .num_threads(num_threads) + .thread_name(move |i| format!("{}-{}", &name_prefix, i)) .build() { Err(e) => Err(Box::new(e.into())), @@ -141,17 +205,30 @@ mod tests { use super::*; use super::PyBuffer; use crate::Python; + use log::info; + + fn init() { + let _ = env_logger::builder().is_test(true).try_init(); + } + + #[test] + fn test_logger() { + init(); + info!("This record will be captured by `cargo test`"); + assert_eq!(2, 1 + 1); + } #[test] fn test_example() { + init(); let client = Client::new(); let res = perform_http_get("http://127.0.0.1:1000", &client); match res { Ok(success) => { - println!("SUCCESS!"); + print!("SUCCESS!"); }, Err(e) => { - println!("error!:{}", e); + print!("error!:{}", e); } } print!("test_example"); @@ -161,10 +238,9 @@ mod tests { #[pymodule] fn alluxiocommon(_py: Python, m: &PyModule) -> PyResult<()> { - // let env = Env::new() - // .filter_or("ALLUXIOCOMMON_LOG", "warn") - // .write_style("ALLUXIOCOMMON_LOG_STYLE"); - // env_logger::init_from_env(env); + let env = Env::new() + .filter_or("ALLUXIOCOMMON_LOG", "warn"); + env_logger::init_from_env(env); m.add_class::()?; Ok(()) diff --git a/rust/alluxiocommon/tests/tests.py b/rust/alluxiocommon/tests/tests.py index 54a1a71..0fd42f7 100644 --- a/rust/alluxiocommon/tests/tests.py +++ b/rust/alluxiocommon/tests/tests.py @@ -17,12 +17,12 @@ def __init__(self): self.page_size = 32 * MB self.worker_host = "172.31.45.56" self.worker_http_port = 28080 - self.dm = _DataManager(4) + self.dm = _DataManager(4, ondemand_pool=False) def multi_http_requests(self): offset, length = 0,40 * MB read_urls = [] - path_id = "addb83ae5f2025ec8f88b832c40177327ccd6aa2486712f34e68c3efbb2045e7" + path_id = "33b86e86ae983c17d23ac116a983c15ac9ed4254d56d288df891df4e1101adae" start = offset while start < offset + length: page_index = start // self.page_size From 1d5d5f8516c9bcd8f40027ed2d1a4d9b5803dbc1 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Tue, 23 Apr 2024 16:50:13 -0700 Subject: [PATCH 05/34] cleanup fixes --- rust/alluxiocommon/src/lib.rs | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/rust/alluxiocommon/src/lib.rs b/rust/alluxiocommon/src/lib.rs index fc9ad50..f67c02b 100644 --- a/rust/alluxiocommon/src/lib.rs +++ b/rust/alluxiocommon/src/lib.rs @@ -1,25 +1,15 @@ use pyo3::prelude::*; -use pyo3::types::PyList; -use std::{cmp, thread, time}; -use std::error::Error; +use std::{cmp}; use std::sync::Arc; -use bytes::Bytes; -use tokio::runtime::{Builder, Runtime}; -use reqwest::blocking; use reqwest::blocking::Client; -use rayon::{ThreadPool, ThreadPoolBuilder, ThreadPoolBuildError}; +use rayon::{ThreadPool, ThreadPoolBuildError}; use env_logger::Env; +use log::debug; use pyo3::{ - exceptions::PyIOError, - exceptions::PyIndexError, exceptions::PyValueError, - exceptions::PyTypeError, exceptions::PyException, - prelude::{pymodule, PyModule, PyResult, Python}, types::PyBytes, - buffer::PyBuffer, - PyErr, }; const DEFAULT_THREADPOOL_NAME: &str = "ALLUXIOCOMMON"; @@ -40,9 +30,10 @@ impl DataManager { max_concurrency: usize, ondemand_pool: Option ) -> PyResult { + debug!("instantiating _DataManager"); match ondemand_pool { Some(is_ondemand_pool) => { - if (!is_ondemand_pool) { + if !is_ondemand_pool { // Create reqwest client (The Client holds a connection pool internally, create to reuse this Client obj). let client = Client::new(); // Create threadpool @@ -79,7 +70,6 @@ impl DataManager { }); } } - println!("instantiate _DataManager"); } fn make_multi_http_req(self_: PyRef<'_, Self>, urls: Vec) -> PyResult { @@ -203,8 +193,8 @@ fn create_pool(num_threads: usize, thread_name_prefix: String) -> Result { + Ok(_success) => { print!("SUCCESS!"); }, Err(e) => { From 5b20fac81aa5f2a36dc018b831e6057d2f746f98 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Tue, 23 Apr 2024 18:17:32 -0700 Subject: [PATCH 06/34] WIP add fixture to test with alluxio client with alluxiocommon lib enabled --- alluxiofs/client/core.py | 10 ++++++---- rust/alluxiocommon/src/lib.rs | 6 ------ tests/client/test_read_range_docker.py | 3 +++ tests/conftest.py | 13 ++++++++++++- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py index b2e25c7..65f30e6 100644 --- a/alluxiofs/client/core.py +++ b/alluxiofs/client/core.py @@ -199,7 +199,7 @@ def __init__( f"Hash node per worker is set to {hash_node_per_worker}" ) if ALLUXIO_COMMON_EXTENSION_ENABLE in options: - print(f"Using alluxiocommon extension, concurrency:{concurrency}") + print(f"Using alluxiocommon extension..") self.logger.debug( "alluxiocommon extension enabled." ) @@ -513,7 +513,8 @@ def read_range(self, file_path, offset, length): Returns: file content (str): The file content with length from offset """ - self.logger.debug(f"read_range,off:{offset}:length:{length}") + # self.logger.debug(f"read_range,off:{offset}:length:{length}") + print(f"read_range,off:{offset}:length:{length}") self._validate_path(file_path) if not isinstance(offset, int) or offset < 0: raise ValueError("Offset must be a non-negative integer") @@ -550,8 +551,8 @@ def read_range(self, file_path, offset, length): ) except Exception as e: raise Exception( - f"Error when reading file {file_path}: error {e}: " - f"worker_host{worker_host}, worker_http_port:{worker_http_port}" + f"Error when reading file:{file_path}: error:{e}: " + f"worker_host:{worker_host}, worker_http_port:{worker_http_port}" ) from e def write_page(self, file_path, page_index, page_bytes): @@ -639,6 +640,7 @@ def _multithread_range_page_generator( ) read_urls.append(page_url) start += inpage_read_len + print(f"read_urls:{read_urls}") data = self.data_manager.make_multi_http_req(read_urls) return data diff --git a/rust/alluxiocommon/src/lib.rs b/rust/alluxiocommon/src/lib.rs index f67c02b..6ef04fe 100644 --- a/rust/alluxiocommon/src/lib.rs +++ b/rust/alluxiocommon/src/lib.rs @@ -81,10 +81,6 @@ impl DataManager { senders.push(None); } - // let Some(kmeans) = self.trained_kmeans.as_ref() else { - // return Err(PyRuntimeError::new_err("KMeans must fit (train) first")); - // }; - let thread_pool /*: Arc*/ = match self_.ondemand_pool { true => { match create_pool(cmp::min(self_.max_threads, num_reqs), @@ -146,8 +142,6 @@ impl DataManager { return Err(PyErr::fetch(self_.py())); } } - // let result = sender.unwrap().blocking_recv().unwrap(); - // content_results.push(result); } let mut concatenated_data: Vec = Vec::new(); for content_result in &content_results { diff --git a/tests/client/test_read_range_docker.py b/tests/client/test_read_range_docker.py index 7389897..492239c 100644 --- a/tests/client/test_read_range_docker.py +++ b/tests/client/test_read_range_docker.py @@ -114,3 +114,6 @@ def test_alluxio_client(alluxio_client: AlluxioClient): def test_etcd_alluxio_client(etcd_alluxio_client: AlluxioClient): test_alluxio_client(etcd_alluxio_client) + +def test_alluxio_client_alluxiocommon(alluxio_client_alluxiocommon: AlluxioClient): + test_alluxio_client(alluxio_client_alluxiocommon) diff --git a/tests/conftest.py b/tests/conftest.py index 97f3729..e2446c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,7 +129,7 @@ def docker_alluxio(): return launch_alluxio_dockers() yield yield_url() - stop_alluxio_dockers() + # stop_alluxio_dockers() @pytest.fixture(scope="session") @@ -152,6 +152,17 @@ def alluxio_client(docker_alluxio): alluxio_client = AlluxioClient(worker_hosts=host, worker_http_port=port) yield alluxio_client +@pytest.fixture +def alluxio_client_alluxiocommon(docker_alluxio): + LOGGER.debug(f"get AlluxioClient with alluxiocommon connect to {docker_alluxio}") + parsed_url = urlparse(docker_alluxio) + host = parsed_url.hostname + port = parsed_url.port + alluxio_options = { + "alluxio.common.extension.enable": "True"} + alluxio_client = AlluxioClient(worker_hosts=host, worker_http_port=port, + options=alluxio_options) + yield alluxio_client @pytest.fixture def etcd_alluxio_client(docker_alluxio_with_etcd): From e9bc8c0ea14f48ab4bbec1242c68b9defb0c86a9 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Tue, 23 Apr 2024 21:03:13 -0700 Subject: [PATCH 07/34] for read pages if we got any data just return without throwing any following exception --- rust/alluxiocommon/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rust/alluxiocommon/src/lib.rs b/rust/alluxiocommon/src/lib.rs index 6ef04fe..69e3511 100644 --- a/rust/alluxiocommon/src/lib.rs +++ b/rust/alluxiocommon/src/lib.rs @@ -130,6 +130,7 @@ impl DataManager { } senders[i] = Some(recv); } + for sender in senders { let result = sender.unwrap().blocking_recv(); match result { @@ -144,12 +145,18 @@ impl DataManager { } } let mut concatenated_data: Vec = Vec::new(); + let mut somedata_read: bool = false; for content_result in &content_results { match content_result { Ok(content) => { concatenated_data.extend(content); + somedata_read = true; }, Err(err) => { + if (somedata_read) { + println!("[LUCYDEBUG] some data read, not erroring out!"); + break; + } let err_str = err.to_string(); PyException::new_err(format!("Error in getting result, {}", err_str)) .restore(self_.py()); From 9efb902560ef67e97c69866bc945d70ba0289143 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Thu, 25 Apr 2024 15:32:09 -0700 Subject: [PATCH 08/34] add _all_page_generator with alluxiocommon --- alluxiofs/client/const.py | 1 + alluxiofs/client/core.py | 57 +++++++++++++++++++++++++++------ rust/alluxiocommon/src/lib.rs | 31 ++++++------------ tests/client/test_read_range.py | 25 +++++++++++++-- 4 files changed, 80 insertions(+), 34 deletions(-) diff --git a/alluxiofs/client/const.py b/alluxiofs/client/const.py index fc74f4d..fa3914f 100644 --- a/alluxiofs/client/const.py +++ b/alluxiofs/client/const.py @@ -13,6 +13,7 @@ ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE = 5 ALLUXIO_SUCCESS_IDENTIFIER = "success" ALLUXIO_COMMON_EXTENSION_ENABLE = "alluxio.common.extension.enable" +ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE = "alluxio.common.ondemandpool.disable" LIST_URL_FORMAT = "http://{worker_host}:{http_port}/v1/files" FULL_PAGE_URL_FORMAT = ( "http://{worker_host}:{http_port}/v1/file/{path_id}/page/{page_index}" diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py index 65f30e6..66a5d9b 100644 --- a/alluxiofs/client/core.py +++ b/alluxiofs/client/core.py @@ -17,6 +17,7 @@ from alluxiocommon import _DataManager from .const import ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE +from .const import ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE from .const import ALLUXIO_COMMON_EXTENSION_ENABLE from .const import ALLUXIO_HASH_NODE_PER_WORKER_KEY1 from .const import ALLUXIO_HASH_NODE_PER_WORKER_KEY2 @@ -198,12 +199,16 @@ def __init__( self.logger.debug( f"Hash node per worker is set to {hash_node_per_worker}" ) - if ALLUXIO_COMMON_EXTENSION_ENABLE in options: + if ALLUXIO_COMMON_EXTENSION_ENABLE in options \ + and options[ALLUXIO_COMMON_EXTENSION_ENABLE].lower() == "true": print(f"Using alluxiocommon extension..") self.logger.debug( "alluxiocommon extension enabled." ) - self.data_manager = _DataManager(concurrency) + ondemand_pool_disabled = ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE in options \ + and options[ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE].lower() == "true" + self.data_manager = _DataManager(concurrency, + ondemand_pool_disabled=ondemand_pool_disabled) if ( not isinstance(hash_node_per_worker, int) or hash_node_per_worker <= 0 @@ -491,11 +496,18 @@ def read(self, file_path): ) path_id = self._get_path_hash(file_path) try: - return b"".join( - self._all_page_generator( - worker_host, worker_http_port, path_id + if self.data_manager: + return b"".join( + self._all_page_generator_alluxiocommon( + worker_host, worker_http_port, path_id + ) + ) + else: + return b"".join( + self._all_page_generator( + worker_host, worker_http_port, path_id + ) ) - ) except Exception as e: raise Exception( f"Error when reading file {file_path}: error {e}" @@ -513,8 +525,7 @@ def read_range(self, file_path, offset, length): Returns: file content (str): The file content with length from offset """ - # self.logger.debug(f"read_range,off:{offset}:length:{length}") - print(f"read_range,off:{offset}:length:{length}") + self.logger.debug(f"read_range,off:{offset}:length:{length}") self._validate_path(file_path) if not isinstance(offset, int) or offset < 0: raise ValueError("Offset must be a non-negative integer") @@ -540,7 +551,7 @@ def read_range(self, file_path, offset, length): try: if self.data_manager: - return self._multithread_range_page_generator( + return self._range_page_generator_alluxiocommon( worker_host, worker_http_port, path_id, offset, length ) else: @@ -590,6 +601,32 @@ def write_page(self, file_path, page_index, page_bytes): f"Error writing to file {file_path} at page {page_index}: {e}" ) + def _all_page_generator_alluxiocommon(self, worker_host, worker_http_port, path_id): + page_index = 0 + fetching_pages_num_each_round = 4 + while True: + read_urls = [] + try: + for _ in range(fetching_pages_num_each_round): + page_url = FULL_PAGE_URL_FORMAT.format( + worker_host=worker_host, + http_port=worker_http_port, + path_id=path_id, + page_index=page_index, + ) + read_urls.append(page_url) + page_index += 1 + pages_content = self.data_manager.make_multi_http_req(read_urls) + yield pages_content + if len(pages_content) < fetching_pages_num_each_round * self.page_size: + break + except Exception as e: + # data_manager won't throw exception if there are any first few content retrieved + # hence we always propagte exception from data_manager upwards + raise Exception( + f"Error when reading all pages of {path_id}: error {e}" + ) from e + def _all_page_generator(self, worker_host, worker_http_port, path_id): page_index = 0 while True: @@ -612,7 +649,7 @@ def _all_page_generator(self, worker_host, worker_http_port, path_id): break page_index += 1 - def _multithread_range_page_generator( + def _range_page_generator_alluxiocommon( self, worker_host, worker_http_port, path_id, offset, length ): read_urls = [] diff --git a/rust/alluxiocommon/src/lib.rs b/rust/alluxiocommon/src/lib.rs index 69e3511..ef430b4 100644 --- a/rust/alluxiocommon/src/lib.rs +++ b/rust/alluxiocommon/src/lib.rs @@ -28,12 +28,12 @@ impl DataManager { #[new] fn new( max_concurrency: usize, - ondemand_pool: Option + ondemand_pool_disabled: Option ) -> PyResult { debug!("instantiating _DataManager"); - match ondemand_pool { - Some(is_ondemand_pool) => { - if !is_ondemand_pool { + match ondemand_pool_disabled { + Some(disabled) => { + if disabled { // Create reqwest client (The Client holds a connection pool internally, create to reuse this Client obj). let client = Client::new(); // Create threadpool @@ -43,7 +43,7 @@ impl DataManager { Ok(pool) => { return Ok(Self { max_threads: max_concurrency, - ondemand_pool: is_ondemand_pool, + ondemand_pool: !disabled, thread_pool: Option::Some(Arc::new(pool)), request_client: Option::Some(Arc::new(client)), }); @@ -73,7 +73,6 @@ impl DataManager { } fn make_multi_http_req(self_: PyRef<'_, Self>, urls: Vec) -> PyResult { - println!("on demand pool and client!"); let num_reqs = urls.len(); let mut content_results = Vec::with_capacity(num_reqs); let mut senders = Vec::with_capacity(num_reqs); @@ -109,19 +108,15 @@ impl DataManager { }; for i in 0..num_reqs { let url_owned = urls[i].to_owned(); - // let client_clone = Arc::clone(&(self_.request_client)); let client_clone = Arc::clone(&(request_client)); let (send, recv) = tokio::sync::oneshot::channel(); let install_res = thread_pool.install(move || -> Result<(), reqwest::Error> { - // println!("request idx:{}", i); let body = perform_http_get(url_owned.as_str(), client_clone.as_ref()); send.send(body).unwrap(); Ok(()) }); match install_res { - Ok(_success) => { - //DO NOTHING - }, + Ok(_success) => {}, Err(err) => { PyException::new_err(err.to_string()) .restore(self_.py()); @@ -153,8 +148,7 @@ impl DataManager { somedata_read = true; }, Err(err) => { - if (somedata_read) { - println!("[LUCYDEBUG] some data read, not erroring out!"); + if somedata_read { break; } let err_str = err.to_string(); @@ -168,10 +162,6 @@ impl DataManager { } } -fn type_name_of_val(_: T) -> &'static str { - std::any::type_name::() -} - fn perform_http_get(url: &str, client: &Client) -> Result, reqwest::Error> { let bytes = client.get(url).send()? .bytes()?; @@ -194,8 +184,6 @@ fn create_pool(num_threads: usize, thread_name_prefix: String) -> Result Date: Thu, 25 Apr 2024 16:10:34 -0700 Subject: [PATCH 09/34] 1. add pytest fixture for alluxiocommon enabled to use in test_docker_alluxio_fsspec.py 2. make tests/client/test_read_range_docker.py (now named as test_read_docker.py to handle full read as well add alluxiocommon fixture in it too --- alluxiofs/client/core.py | 2 +- ...ad_range_docker.py => test_read_docker.py} | 37 +++++++++++++++++++ tests/conftest.py | 20 ++++++++++ tests/fs/test_docker_alluxio_fsspec.py | 2 + 4 files changed, 60 insertions(+), 1 deletion(-) rename tests/client/{test_read_range_docker.py => test_read_docker.py} (74%) diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py index 66a5d9b..275f0fe 100644 --- a/alluxiofs/client/core.py +++ b/alluxiofs/client/core.py @@ -677,7 +677,7 @@ def _range_page_generator_alluxiocommon( ) read_urls.append(page_url) start += inpage_read_len - print(f"read_urls:{read_urls}") + self.logger.debug(f"read_urls:{read_urls}") data = self.data_manager.make_multi_http_req(read_urls) return data diff --git a/tests/client/test_read_range_docker.py b/tests/client/test_read_docker.py similarity index 74% rename from tests/client/test_read_range_docker.py rename to tests/client/test_read_docker.py index 492239c..6b7becb 100644 --- a/tests/client/test_read_range_docker.py +++ b/tests/client/test_read_docker.py @@ -4,6 +4,7 @@ from alluxiofs import AlluxioClient from tests.conftest import ALLUXIO_FILE_PATH from tests.conftest import LOCAL_FILE_PATH +from hashlib import md5 NUM_TESTS = 10 @@ -12,6 +13,12 @@ LOGGER = logging.getLogger(__name__) +def _get_md5(payload): + m = md5() + m.update(payload) + m.update(payload) + return m.hexdigest() + def validate_read_range( alluxio_client: AlluxioClient, alluxio_file_path, @@ -63,6 +70,30 @@ def validate_invalid_read_range( "Expected an exception from local file read but none occurred." ) +def validate_full_read( + alluxio_client: AlluxioClient, + alluxio_file_path, + local_file_path, +): + alluxio_data = alluxio_client.read(alluxio_file_path) + + with open(local_file_path, "rb") as local_file: + local_data = local_file.read() + + try: + assert alluxio_data == local_data + except AssertionError: + md5_alluxio_data = _get_md5(alluxio_data) + md5_local_data = _get_md5(local_data) + error_message = ( + f"Data mismatch between Alluxio and local file\n" + f"Alluxio file path: {alluxio_file_path}\n" + f"Local file path: {local_file_path}\n" + f"Alluxio data md5: {md5_alluxio_data}, length:{len(alluxio_data)}\n" + f"Local data md5: {md5_local_data}, length:{len(local_data)}" + ) + raise AssertionError(error_message) + def test_alluxio_client(alluxio_client: AlluxioClient): file_size = os.path.getsize(LOCAL_FILE_PATH) @@ -111,6 +142,12 @@ def test_alluxio_client(alluxio_client: AlluxioClient): ) LOGGER.debug("Passed corner test cases") + # test full data read + + validate_full_read(alluxio_client, + ALLUXIO_FILE_PATH, + LOCAL_FILE_PATH) + def test_etcd_alluxio_client(etcd_alluxio_client: AlluxioClient): test_alluxio_client(etcd_alluxio_client) diff --git a/tests/conftest.py b/tests/conftest.py index e2446c5..8777961 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -189,6 +189,26 @@ def alluxio_file_system(docker_alluxio): ) yield alluxio_file_system +@pytest.fixture +def alluxio_file_system_alluxiocommon(docker_alluxio): + LOGGER.debug( + f"get AlluxioFileSystem connect to {docker_alluxio}" + f" with alluxiocommon enabled." + ) + parsed_url = urlparse(docker_alluxio) + host = parsed_url.hostname + fsspec.register_implementation("alluxio", AlluxioFileSystem, clobber=True) + alluxio_options = { + "alluxio.common.extension.enable": "True"} + alluxio_file_system = fsspec.filesystem( + "alluxio", + worker_hosts=host, + target_protocol="file", + options=alluxio_options, + preload_path=ALLUXIO_FILE_PATH, + ) + yield alluxio_file_system + @pytest.fixture def etcd_alluxio_file_system(docker_alluxio_with_etcd): diff --git a/tests/fs/test_docker_alluxio_fsspec.py b/tests/fs/test_docker_alluxio_fsspec.py index 5445974..163aa8f 100644 --- a/tests/fs/test_docker_alluxio_fsspec.py +++ b/tests/fs/test_docker_alluxio_fsspec.py @@ -4,6 +4,8 @@ def test_simple_fsspec(alluxio_file_system): alluxio_file_system.ls(TEST_ROOT) # no error +def test_simple_fsspec_alluxiocommon(alluxio_file_system_alluxiocommon): + alluxio_file_system_alluxiocommon.ls(TEST_ROOT) def test_simple_etcd_fsspec(etcd_alluxio_file_system): etcd_alluxio_file_system.ls(TEST_ROOT) # no error From 9cda7cf74724791371599988c7058cada0db9006 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Thu, 25 Apr 2024 16:15:44 -0700 Subject: [PATCH 10/34] remove unnecessary files --- rust/alluxiocommon/tests/tests.py | 63 ------------------------------- rust/ray-e2e-report.md | 12 ------ 2 files changed, 75 deletions(-) delete mode 100644 rust/alluxiocommon/tests/tests.py delete mode 100644 rust/ray-e2e-report.md diff --git a/rust/alluxiocommon/tests/tests.py b/rust/alluxiocommon/tests/tests.py deleted file mode 100644 index 0fd42f7..0000000 --- a/rust/alluxiocommon/tests/tests.py +++ /dev/null @@ -1,63 +0,0 @@ -from alluxiocommon import _DataManager -from hashlib import md5 -import time - -KB = 1024 -MB = 1024 * KB - -FULL_PAGE_URL_FORMAT = ( - "http://{worker_host}:{http_port}/v1/file/{path_id}/page/{page_index}" -) -PAGE_URL_FORMAT = "http://{worker_host}:{http_port}/v1/file/{path_id}/page/{page_index}?offset={page_offset}&length={page_length}" - - -class TestSuite: - - def __init__(self): - self.page_size = 32 * MB - self.worker_host = "172.31.45.56" - self.worker_http_port = 28080 - self.dm = _DataManager(4, ondemand_pool=False) - - def multi_http_requests(self): - offset, length = 0,40 * MB - read_urls = [] - path_id = "33b86e86ae983c17d23ac116a983c15ac9ed4254d56d288df891df4e1101adae" - start = offset - while start < offset + length: - page_index = start // self.page_size - inpage_off = start % self.page_size - inpage_read_len = min(self.page_size - inpage_off, length - start) - page_url = None - if inpage_off == 0 and inpage_read_len == self.page_size: - page_url = FULL_PAGE_URL_FORMAT.format( - worker_host=self.worker_host, - http_port=self.worker_http_port, - path_id=path_id, - page_index=page_index, - ) - else: - page_url = PAGE_URL_FORMAT.format( - worker_host=self.worker_host, - http_port=self.worker_http_port, - path_id=path_id, - page_index=page_index, - page_offset=inpage_off, - page_length=inpage_read_len, - ) - read_urls.append(page_url) - start += inpage_read_len - data = self.dm.make_multi_http_req(read_urls) - print(f"type:{type(data)}") - md5_instance = md5() - md5_instance.update(data) - print(f"data len:{len(data)}, md5:{md5_instance.hexdigest()}") - time.sleep(30) - -def main(): - ts = TestSuite() - for _ in range(1): - ts.multi_http_requests() - -if __name__ == "__main__": - main() diff --git a/rust/ray-e2e-report.md b/rust/ray-e2e-report.md deleted file mode 100644 index 07e5115..0000000 --- a/rust/ray-e2e-report.md +++ /dev/null @@ -1,12 +0,0 @@ -Before optimization - -| Test | Benchmark | Local FileSystem(nvme) | Local region S3 | AlluxioFsSpec | AlluxioFsSpec PyO3 optimized | -|------------------------|-----------|-------------------------|----------------|-----------------|------------------------------| -| Ray + XgBoost(10G) | | | | PyTorch | -| Ray + PyTorch(parquet) | | | | CosmoFlow | Tensorflow | -| Ray + PyTorch(image) | | | Forces mean absolute error 0.036 | DimeNet++ | PyTorch | -| | | | 0.8 Local Distance Difference Test (lDDT) | AlphaFold2 (PyTorch) | PyTorch | - - -After optimization - From 9d3c31a0a479414d049c22ed02fa929eb76531a6 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 14:10:30 -0700 Subject: [PATCH 11/34] try adding a ci yaml file with alluxiocommon enabled as dependency --- .github/workflows/ci-withalluxiocommon.yml | 116 +++++++++++++++++++++ alluxiofs/client/core.py | 5 +- 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-withalluxiocommon.yml diff --git a/.github/workflows/ci-withalluxiocommon.yml b/.github/workflows/ci-withalluxiocommon.yml new file mode 100644 index 0000000..9cf717a --- /dev/null +++ b/.github/workflows/ci-withalluxiocommon.yml @@ -0,0 +1,116 @@ +name: CI + +on: [pull_request, workflow_dispatch] + +defaults: + run: + shell: bash -l -eo pipefail {0} + +jobs: + alluxiocommon: + runs-on: ubuntu-latest + strategy: + fail-fase: false +# matrix: +# platform: +# - runner: ubuntu-latest +# target: x86_64 +# - runner: ubuntu-latest +# target: x86 +# - runner: ubuntu-latest +# target: aarch64 +# - runner: ubuntu-latest +# target: armv7 +# - runner: ubuntu-latest +# target: s390x +# - runner: ubuntu-latest +# target: ppc64le + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml + sccache: 'true' + manylinux: auto +# - name: Upload wheels +# uses: actions/upload-artifact@v4 +# with: +# name: wheels-linux-${{ matrix.platform.target }} +# path: dist +# - name: pytest +# if: ${{ startsWith(matrix.platform.target, 'x86_64') }} +# shell: bash +# run: | +# set -e +# pip install alluxiocommon --find-links dist --force-reinstall +# pip install pytest +# cd /root/github/alluxiofs/rust/alluxiocommon && pytest +# - name: pytest +# if: ${{ !startsWith(matrix.platform.target, 'x86') && matrix.platform.target != 'ppc64' }} +# uses: uraimo/run-on-arch-action@v2.5.0 +# with: +# arch: ${{ matrix.platform.target }} +# distro: ubuntu22.04 +# githubToken: ${{ github.token }} +# install: | +# apt-get update +# apt-get install -y --no-install-recommends python3 python3-pip +# pip3 install -U pip pytest +# run: | +# set -e +# pip3 install alluxiocommon --find-links dist --force-reinstall +# cd /root/github/alluxiofs/rust/alluxiocommon && pytest + + test: + needs: alluxiocommon + name: Test + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + python-version: ["3.8","3.9"] + + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Setup Conda Environment + uses: mamba-org/setup-micromamba@v1 + with: + cache-downloads: true + environment-file: environment_conda.yaml + environment-name: alluxio_test + create-args: >- + python=${{ matrix.python-version }} + + - name: Conda info + run: | + conda list + conda --version + + - name: Install dependencies + run: | + pip install .[tests] + + - name: Run Alluxio FileSystem tests + run: | + pytest -vv \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%H:%M:%S" \ + tests/ + + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + - uses: pre-commit/action@v3.0.0 diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py index 275f0fe..fbf7c4b 100644 --- a/alluxiofs/client/core.py +++ b/alluxiofs/client/core.py @@ -14,7 +14,10 @@ import humanfriendly import requests from requests.adapters import HTTPAdapter -from alluxiocommon import _DataManager +try: + from alluxiocommon import _DataManager +except ModuleNotFoundError: + print("[WARNING]pkg 'alluxiocommon' not installed, relative modules unable to invoke.") from .const import ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE from .const import ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE From c585c8a35578bf56dbd3516e9b9e3ffaf7531283 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 14:19:20 -0700 Subject: [PATCH 12/34] cleanup fixes --- alluxiofs/client/core.py | 46 +++++++++++++++++--------- benchmark/AbstractBench.py | 3 +- rust/README.md | 3 +- rust/alluxiocommon/Cargo.toml | 1 - rust/alluxiocommon/setup.py | 3 +- tests/client/test_read_docker.py | 13 +++++--- tests/client/test_read_range.py | 10 +++--- tests/conftest.py | 18 ++++++---- tests/fs/test_docker_alluxio_fsspec.py | 2 ++ 9 files changed, 62 insertions(+), 37 deletions(-) diff --git a/alluxiofs/client/core.py b/alluxiofs/client/core.py index fbf7c4b..26b6b12 100644 --- a/alluxiofs/client/core.py +++ b/alluxiofs/client/core.py @@ -14,10 +14,13 @@ import humanfriendly import requests from requests.adapters import HTTPAdapter + try: from alluxiocommon import _DataManager except ModuleNotFoundError: - print("[WARNING]pkg 'alluxiocommon' not installed, relative modules unable to invoke.") + print( + "[WARNING]pkg 'alluxiocommon' not installed, relative modules unable to invoke." + ) from .const import ALLUXIO_HASH_NODE_PER_WORKER_DEFAULT_VALUE from .const import ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE @@ -202,16 +205,20 @@ def __init__( self.logger.debug( f"Hash node per worker is set to {hash_node_per_worker}" ) - if ALLUXIO_COMMON_EXTENSION_ENABLE in options \ - and options[ALLUXIO_COMMON_EXTENSION_ENABLE].lower() == "true": - print(f"Using alluxiocommon extension..") - self.logger.debug( - "alluxiocommon extension enabled." + if ( + ALLUXIO_COMMON_EXTENSION_ENABLE in options + and options[ALLUXIO_COMMON_EXTENSION_ENABLE].lower() == "true" + ): + print("Using alluxiocommon extension..") + self.logger.debug("alluxiocommon extension enabled.") + ondemand_pool_disabled = ( + ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE in options + and options[ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE].lower() + == "true" + ) + self.data_manager = _DataManager( + concurrency, ondemand_pool_disabled=ondemand_pool_disabled ) - ondemand_pool_disabled = ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE in options \ - and options[ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE].lower() == "true" - self.data_manager = _DataManager(concurrency, - ondemand_pool_disabled=ondemand_pool_disabled) if ( not isinstance(hash_node_per_worker, int) or hash_node_per_worker <= 0 @@ -604,7 +611,9 @@ def write_page(self, file_path, page_index, page_bytes): f"Error writing to file {file_path} at page {page_index}: {e}" ) - def _all_page_generator_alluxiocommon(self, worker_host, worker_http_port, path_id): + def _all_page_generator_alluxiocommon( + self, worker_host, worker_http_port, path_id + ): page_index = 0 fetching_pages_num_each_round = 4 while True: @@ -619,9 +628,14 @@ def _all_page_generator_alluxiocommon(self, worker_host, worker_http_port, path_ ) read_urls.append(page_url) page_index += 1 - pages_content = self.data_manager.make_multi_http_req(read_urls) + pages_content = self.data_manager.make_multi_http_req( + read_urls + ) yield pages_content - if len(pages_content) < fetching_pages_num_each_round * self.page_size: + if ( + len(pages_content) + < fetching_pages_num_each_round * self.page_size + ): break except Exception as e: # data_manager won't throw exception if there are any first few content retrieved @@ -653,14 +667,16 @@ def _all_page_generator(self, worker_host, worker_http_port, path_id): page_index += 1 def _range_page_generator_alluxiocommon( - self, worker_host, worker_http_port, path_id, offset, length + self, worker_host, worker_http_port, path_id, offset, length ): read_urls = [] start = offset while start < offset + length: page_index = start // self.page_size inpage_off = start % self.page_size - inpage_read_len = min(self.page_size - inpage_off, offset + length - start) + inpage_read_len = min( + self.page_size - inpage_off, offset + length - start + ) page_url = None if inpage_off == 0 and inpage_read_len == self.page_size: page_url = FULL_PAGE_URL_FORMAT.format( diff --git a/benchmark/AbstractBench.py b/benchmark/AbstractBench.py index 76246b4..99f3ae3 100644 --- a/benchmark/AbstractBench.py +++ b/benchmark/AbstractBench.py @@ -81,7 +81,8 @@ def init(self): # protocol = self.get_protocol(self.args.path) alluxio_options = { # "alluxio.common.extension.enable": "True", - "alluxio.worker.page.store.page.size" : "1MB"} + "alluxio.worker.page.store.page.size": "1MB" + } self.alluxio_fs = AlluxioFileSystem( etcd_hosts=self.args.etcd_hosts, worker_hosts=self.args.worker_hosts, diff --git a/rust/README.md b/rust/README.md index 2bf7a04..1cab1f6 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,7 +1,7 @@ ## A common native lib for alluxio python client to build and play: -create virtualenv, (a tool used to create isolated Python environments): +create virtualenv, (a tool used to create isolated Python environments): python3 -m venv .env source .env/bin/activate @@ -12,4 +12,3 @@ then can do: python3 >>> import alluxiocommon >>> alluxiocommon.multi_http_requests(["http://google.com"],[(0,0)]) - diff --git a/rust/alluxiocommon/Cargo.toml b/rust/alluxiocommon/Cargo.toml index d20d22e..ce4baad 100644 --- a/rust/alluxiocommon/Cargo.toml +++ b/rust/alluxiocommon/Cargo.toml @@ -23,4 +23,3 @@ log = "0.4.21" [dependencies.pyo3] version = "0.21.1" features = ["extension-module"] - diff --git a/rust/alluxiocommon/setup.py b/rust/alluxiocommon/setup.py index 9d8c5d5..2585310 100644 --- a/rust/alluxiocommon/setup.py +++ b/rust/alluxiocommon/setup.py @@ -1,5 +1,6 @@ from setuptools import setup -from setuptools_rust import Binding, RustExtension +from setuptools_rust import Binding +from setuptools_rust import RustExtension setup( diff --git a/tests/client/test_read_docker.py b/tests/client/test_read_docker.py index 6b7becb..1e4bf25 100644 --- a/tests/client/test_read_docker.py +++ b/tests/client/test_read_docker.py @@ -1,10 +1,10 @@ import os import random +from hashlib import md5 from alluxiofs import AlluxioClient from tests.conftest import ALLUXIO_FILE_PATH from tests.conftest import LOCAL_FILE_PATH -from hashlib import md5 NUM_TESTS = 10 @@ -19,6 +19,7 @@ def _get_md5(payload): m.update(payload) return m.hexdigest() + def validate_read_range( alluxio_client: AlluxioClient, alluxio_file_path, @@ -70,6 +71,7 @@ def validate_invalid_read_range( "Expected an exception from local file read but none occurred." ) + def validate_full_read( alluxio_client: AlluxioClient, alluxio_file_path, @@ -144,13 +146,14 @@ def test_alluxio_client(alluxio_client: AlluxioClient): # test full data read - validate_full_read(alluxio_client, - ALLUXIO_FILE_PATH, - LOCAL_FILE_PATH) + validate_full_read(alluxio_client, ALLUXIO_FILE_PATH, LOCAL_FILE_PATH) def test_etcd_alluxio_client(etcd_alluxio_client: AlluxioClient): test_alluxio_client(etcd_alluxio_client) -def test_alluxio_client_alluxiocommon(alluxio_client_alluxiocommon: AlluxioClient): + +def test_alluxio_client_alluxiocommon( + alluxio_client_alluxiocommon: AlluxioClient, +): test_alluxio_client(alluxio_client_alluxiocommon) diff --git a/tests/client/test_read_range.py b/tests/client/test_read_range.py index e149811..55c6e8c 100644 --- a/tests/client/test_read_range.py +++ b/tests/client/test_read_range.py @@ -6,6 +6,7 @@ from alluxiofs.client.const import ALLUXIO_COMMON_EXTENSION_ENABLE from alluxiofs.client.const import ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE + def parse_args(): parser = argparse.ArgumentParser( description="Validate Alluxio read_range with local file." @@ -40,14 +41,14 @@ def parse_args(): "--enable-alluxiocommon", type=bool, default=False, - help="To enable alluxiocommon extension" + help="To enable alluxiocommon extension", ) parser.add_argument( "--disable-alluxiocommon-ondemandpool", type=bool, default=False, help="To disable alluxiocommon ondemand pool, " - "effective when --enable-alluxiocommon is enabeld" + "effective when --enable-alluxiocommon is enabeld", ) return parser.parse_args() @@ -101,14 +102,13 @@ def manual_test_invalid_read_range( def main(args): - options={} + options = {} if args.enable_alluxiocommon: options[ALLUXIO_COMMON_EXTENSION_ENABLE] = "True" if args.disable_alluxiocommon_ondemandpool: options[ALLUXIO_COMMON_ONDEMANDPOOL_DISABLE] = "True" - alluxio_client = AlluxioClient(etcd_hosts=args.etcd_hosts, - options=options) + alluxio_client = AlluxioClient(etcd_hosts=args.etcd_hosts, options=options) file_size = os.path.getsize(args.local_file_path) invalid_test_cases = [(-1, 100), (file_size - 1, -2)] diff --git a/tests/conftest.py b/tests/conftest.py index 8777961..a34b32e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -152,18 +152,22 @@ def alluxio_client(docker_alluxio): alluxio_client = AlluxioClient(worker_hosts=host, worker_http_port=port) yield alluxio_client + @pytest.fixture def alluxio_client_alluxiocommon(docker_alluxio): - LOGGER.debug(f"get AlluxioClient with alluxiocommon connect to {docker_alluxio}") + LOGGER.debug( + f"get AlluxioClient with alluxiocommon connect to {docker_alluxio}" + ) parsed_url = urlparse(docker_alluxio) host = parsed_url.hostname port = parsed_url.port - alluxio_options = { - "alluxio.common.extension.enable": "True"} - alluxio_client = AlluxioClient(worker_hosts=host, worker_http_port=port, - options=alluxio_options) + alluxio_options = {"alluxio.common.extension.enable": "True"} + alluxio_client = AlluxioClient( + worker_hosts=host, worker_http_port=port, options=alluxio_options + ) yield alluxio_client + @pytest.fixture def etcd_alluxio_client(docker_alluxio_with_etcd): LOGGER.debug( @@ -189,6 +193,7 @@ def alluxio_file_system(docker_alluxio): ) yield alluxio_file_system + @pytest.fixture def alluxio_file_system_alluxiocommon(docker_alluxio): LOGGER.debug( @@ -198,8 +203,7 @@ def alluxio_file_system_alluxiocommon(docker_alluxio): parsed_url = urlparse(docker_alluxio) host = parsed_url.hostname fsspec.register_implementation("alluxio", AlluxioFileSystem, clobber=True) - alluxio_options = { - "alluxio.common.extension.enable": "True"} + alluxio_options = {"alluxio.common.extension.enable": "True"} alluxio_file_system = fsspec.filesystem( "alluxio", worker_hosts=host, diff --git a/tests/fs/test_docker_alluxio_fsspec.py b/tests/fs/test_docker_alluxio_fsspec.py index 163aa8f..b1c27e5 100644 --- a/tests/fs/test_docker_alluxio_fsspec.py +++ b/tests/fs/test_docker_alluxio_fsspec.py @@ -4,8 +4,10 @@ def test_simple_fsspec(alluxio_file_system): alluxio_file_system.ls(TEST_ROOT) # no error + def test_simple_fsspec_alluxiocommon(alluxio_file_system_alluxiocommon): alluxio_file_system_alluxiocommon.ls(TEST_ROOT) + def test_simple_etcd_fsspec(etcd_alluxio_file_system): etcd_alluxio_file_system.ls(TEST_ROOT) # no error From b5171620c4a24ec4a7bc73e6c0f83f4330c1a16e Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 14:20:57 -0700 Subject: [PATCH 13/34] add to ci.yml of alluxiocommon dependency --- .github/workflows/ci-withalluxiocommon.yml | 116 --------------------- .github/workflows/ci.yml | 60 +++++++++++ 2 files changed, 60 insertions(+), 116 deletions(-) delete mode 100644 .github/workflows/ci-withalluxiocommon.yml diff --git a/.github/workflows/ci-withalluxiocommon.yml b/.github/workflows/ci-withalluxiocommon.yml deleted file mode 100644 index 9cf717a..0000000 --- a/.github/workflows/ci-withalluxiocommon.yml +++ /dev/null @@ -1,116 +0,0 @@ -name: CI - -on: [pull_request, workflow_dispatch] - -defaults: - run: - shell: bash -l -eo pipefail {0} - -jobs: - alluxiocommon: - runs-on: ubuntu-latest - strategy: - fail-fase: false -# matrix: -# platform: -# - runner: ubuntu-latest -# target: x86_64 -# - runner: ubuntu-latest -# target: x86 -# - runner: ubuntu-latest -# target: aarch64 -# - runner: ubuntu-latest -# target: armv7 -# - runner: ubuntu-latest -# target: s390x -# - runner: ubuntu-latest -# target: ppc64le - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml - sccache: 'true' - manylinux: auto -# - name: Upload wheels -# uses: actions/upload-artifact@v4 -# with: -# name: wheels-linux-${{ matrix.platform.target }} -# path: dist -# - name: pytest -# if: ${{ startsWith(matrix.platform.target, 'x86_64') }} -# shell: bash -# run: | -# set -e -# pip install alluxiocommon --find-links dist --force-reinstall -# pip install pytest -# cd /root/github/alluxiofs/rust/alluxiocommon && pytest -# - name: pytest -# if: ${{ !startsWith(matrix.platform.target, 'x86') && matrix.platform.target != 'ppc64' }} -# uses: uraimo/run-on-arch-action@v2.5.0 -# with: -# arch: ${{ matrix.platform.target }} -# distro: ubuntu22.04 -# githubToken: ${{ github.token }} -# install: | -# apt-get update -# apt-get install -y --no-install-recommends python3 python3-pip -# pip3 install -U pip pytest -# run: | -# set -e -# pip3 install alluxiocommon --find-links dist --force-reinstall -# cd /root/github/alluxiofs/rust/alluxiocommon && pytest - - test: - needs: alluxiocommon - name: Test - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - python-version: ["3.8","3.9"] - - steps: - - name: Checkout source - uses: actions/checkout@v4 - - - name: Setup Conda Environment - uses: mamba-org/setup-micromamba@v1 - with: - cache-downloads: true - environment-file: environment_conda.yaml - environment-name: alluxio_test - create-args: >- - python=${{ matrix.python-version }} - - - name: Conda info - run: | - conda list - conda --version - - - name: Install dependencies - run: | - pip install .[tests] - - - name: Run Alluxio FileSystem tests - run: | - pytest -vv \ - --log-format="%(asctime)s %(levelname)s %(message)s" \ - --log-date-format="%H:%M:%S" \ - tests/ - - lint: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f1ec1b..9cf717a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,67 @@ defaults: shell: bash -l -eo pipefail {0} jobs: + alluxiocommon: + runs-on: ubuntu-latest + strategy: + fail-fase: false +# matrix: +# platform: +# - runner: ubuntu-latest +# target: x86_64 +# - runner: ubuntu-latest +# target: x86 +# - runner: ubuntu-latest +# target: aarch64 +# - runner: ubuntu-latest +# target: armv7 +# - runner: ubuntu-latest +# target: s390x +# - runner: ubuntu-latest +# target: ppc64le + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml + sccache: 'true' + manylinux: auto +# - name: Upload wheels +# uses: actions/upload-artifact@v4 +# with: +# name: wheels-linux-${{ matrix.platform.target }} +# path: dist +# - name: pytest +# if: ${{ startsWith(matrix.platform.target, 'x86_64') }} +# shell: bash +# run: | +# set -e +# pip install alluxiocommon --find-links dist --force-reinstall +# pip install pytest +# cd /root/github/alluxiofs/rust/alluxiocommon && pytest +# - name: pytest +# if: ${{ !startsWith(matrix.platform.target, 'x86') && matrix.platform.target != 'ppc64' }} +# uses: uraimo/run-on-arch-action@v2.5.0 +# with: +# arch: ${{ matrix.platform.target }} +# distro: ubuntu22.04 +# githubToken: ${{ github.token }} +# install: | +# apt-get update +# apt-get install -y --no-install-recommends python3 python3-pip +# pip3 install -U pip pytest +# run: | +# set -e +# pip3 install alluxiocommon --find-links dist --force-reinstall +# cd /root/github/alluxiofs/rust/alluxiocommon && pytest + test: + needs: alluxiocommon name: Test runs-on: ubuntu-latest timeout-minutes: 10 From b5b5b8b6c829279b002653ec062c13a0fabb53cf Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 14:51:38 -0700 Subject: [PATCH 14/34] add rust/alluxiocommon/tests/ --- .github/workflows/ci.yml | 40 +++++++------------- rust/alluxiocommon/tests/test_datamanager.py | 10 +++++ 2 files changed, 23 insertions(+), 27 deletions(-) create mode 100644 rust/alluxiocommon/tests/test_datamanager.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9cf717a..da9f8a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,23 +8,11 @@ defaults: jobs: alluxiocommon: + name: Build alluxiocommon runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fase: false -# matrix: -# platform: -# - runner: ubuntu-latest -# target: x86_64 -# - runner: ubuntu-latest -# target: x86 -# - runner: ubuntu-latest -# target: aarch64 -# - runner: ubuntu-latest -# target: armv7 -# - runner: ubuntu-latest -# target: s390x -# - runner: ubuntu-latest -# target: ppc64le steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -37,19 +25,17 @@ jobs: args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml sccache: 'true' manylinux: auto -# - name: Upload wheels -# uses: actions/upload-artifact@v4 -# with: -# name: wheels-linux-${{ matrix.platform.target }} -# path: dist -# - name: pytest -# if: ${{ startsWith(matrix.platform.target, 'x86_64') }} -# shell: bash -# run: | -# set -e -# pip install alluxiocommon --find-links dist --force-reinstall -# pip install pytest -# cd /root/github/alluxiofs/rust/alluxiocommon && pytest + - name: pytest + if: ${{ startsWith(matrix.platform.target, 'x86_64') }} + shell: bash + run: | + set -e + pip install alluxiocommon --find-links dist --force-reinstall + pip install pytest + pytest -vv \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%H:%M:%S" \ + rust/alluxiocommon/tests/ # - name: pytest # if: ${{ !startsWith(matrix.platform.target, 'x86') && matrix.platform.target != 'ppc64' }} # uses: uraimo/run-on-arch-action@v2.5.0 diff --git a/rust/alluxiocommon/tests/test_datamanager.py b/rust/alluxiocommon/tests/test_datamanager.py new file mode 100644 index 0000000..2adc094 --- /dev/null +++ b/rust/alluxiocommon/tests/test_datamanager.py @@ -0,0 +1,10 @@ +def test_import_alluxiocommon(): + try: + from alluxiocommon import _DataManager + + dm = _DataManager() + print(f"alluxiocommon._DataManager:{dm} instantiated!") + except Exception as e: + assert ( + False + ), f"Unexpected exception when importing alluxiocommon package:{e}" From a24652f261981904db96d0a5bbf3375debe238c0 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 21:52:42 +0000 Subject: [PATCH 15/34] fixes --- rust/alluxiocommon/tests/test_datamanager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/alluxiocommon/tests/test_datamanager.py b/rust/alluxiocommon/tests/test_datamanager.py index 2adc094..abfec01 100644 --- a/rust/alluxiocommon/tests/test_datamanager.py +++ b/rust/alluxiocommon/tests/test_datamanager.py @@ -1,8 +1,7 @@ def test_import_alluxiocommon(): try: from alluxiocommon import _DataManager - - dm = _DataManager() + dm = _DataManager(4) print(f"alluxiocommon._DataManager:{dm} instantiated!") except Exception as e: assert ( From 647e3eae779efede61370359db93008a7fe97a48 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 14:54:32 -0700 Subject: [PATCH 16/34] remove changes in ci.yml as it wont run --- .github/workflows/ci.yml | 60 ++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da9f8a2..6926e37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,35 +7,35 @@ defaults: shell: bash -l -eo pipefail {0} jobs: - alluxiocommon: - name: Build alluxiocommon - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fase: false - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml - sccache: 'true' - manylinux: auto - - name: pytest - if: ${{ startsWith(matrix.platform.target, 'x86_64') }} - shell: bash - run: | - set -e - pip install alluxiocommon --find-links dist --force-reinstall - pip install pytest - pytest -vv \ - --log-format="%(asctime)s %(levelname)s %(message)s" \ - --log-date-format="%H:%M:%S" \ - rust/alluxiocommon/tests/ +# alluxiocommon: +# name: Build alluxiocommon +# runs-on: ubuntu-latest +# timeout-minutes: 10 +# strategy: +# fail-fase: false +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-python@v5 +# with: +# python-version: '3.10' +# - name: Build wheels +# uses: PyO3/maturin-action@v1 +# with: +# target: ${{ matrix.platform.target }} +# args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml +# sccache: 'true' +# manylinux: auto +# - name: pytest +# if: ${{ startsWith(matrix.platform.target, 'x86_64') }} +# shell: bash +# run: | +# set -e +# pip install alluxiocommon --find-links dist --force-reinstall +# pip install pytest +# pytest -vv \ +# --log-format="%(asctime)s %(levelname)s %(message)s" \ +# --log-date-format="%H:%M:%S" \ +# rust/alluxiocommon/tests/ # - name: pytest # if: ${{ !startsWith(matrix.platform.target, 'x86') && matrix.platform.target != 'ppc64' }} # uses: uraimo/run-on-arch-action@v2.5.0 @@ -53,7 +53,7 @@ jobs: # cd /root/github/alluxiofs/rust/alluxiocommon && pytest test: - needs: alluxiocommon +# needs: alluxiocommon name: Test runs-on: ubuntu-latest timeout-minutes: 10 From 083a2a5becc88a4fa2773df3857f38b3a8fd444e Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 15:32:23 -0700 Subject: [PATCH 17/34] try add alluxiocommon pyo3 build --- .github/workflows/ci.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6926e37..7cfffd3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,6 @@ jobs: # cd /root/github/alluxiofs/rust/alluxiocommon && pytest test: -# needs: alluxiocommon name: Test runs-on: ubuntu-latest timeout-minutes: 10 @@ -80,10 +79,30 @@ jobs: conda list conda --version + # alluxiocommon related + - name: Build alluxiocommon wheels + uses: PyO3/maturin-action@v1 + with: + target: x86_64 + args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml + sccache: 'true' + manylinux: auto + - name: Install dependencies run: | pip install .[tests] +# - name: Run AlluxioCommon tests +# shell: bash +# run: | +# set -e +# pip install alluxiocommon --find-links dist --force-reinstall +# pip install pytest +# pytest -vv \ +# --log-format="%(asctime)s %(levelname)s %(message)s" \ +# --log-date-format="%H:%M:%S" \ +# rust/alluxiocommon/tests/ + - name: Run Alluxio FileSystem tests run: | pytest -vv \ From 5fe1c1dab260ea0288aec0c7c19a1c20cdb0754f Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 16:20:21 -0700 Subject: [PATCH 18/34] fix pyo3 related --- .github/workflows/ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cfffd3..8438af8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,10 +60,14 @@ jobs: fail-fast: false matrix: python-version: ["3.8","3.9"] + target: [x86_64] steps: - - name: Checkout source - uses: actions/checkout@v4 +# - name: Checkout source + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - name: Setup Conda Environment uses: mamba-org/setup-micromamba@v1 @@ -83,9 +87,9 @@ jobs: - name: Build alluxiocommon wheels uses: PyO3/maturin-action@v1 with: - target: x86_64 - args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml - sccache: 'true' + target: ${{ matrix.target }} + args: --release --out dist -m rust/alluxiocommon/Cargo.toml +# sccache: 'true' manylinux: auto - name: Install dependencies From 3eccac5e1deb9660bb967c308817eee6d72d6573 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 23:26:22 -0700 Subject: [PATCH 19/34] pre-commit style changes + manually install openssl --- .github/workflows/ci.yml | 3 +++ rust/alluxiocommon/tests/test_datamanager.py | 1 + 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8438af8..4a49d6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,9 @@ jobs: conda list conda --version + - name: Install openssl for AlluxioCommon Build + run: sudo apt-get install pkg-config libssl-dev + # alluxiocommon related - name: Build alluxiocommon wheels uses: PyO3/maturin-action@v1 diff --git a/rust/alluxiocommon/tests/test_datamanager.py b/rust/alluxiocommon/tests/test_datamanager.py index abfec01..bac153a 100644 --- a/rust/alluxiocommon/tests/test_datamanager.py +++ b/rust/alluxiocommon/tests/test_datamanager.py @@ -1,6 +1,7 @@ def test_import_alluxiocommon(): try: from alluxiocommon import _DataManager + dm = _DataManager(4) print(f"alluxiocommon._DataManager:{dm} instantiated!") except Exception as e: From c9c1dbcff3bd87d6662aea423e2091b7611a8f34 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Fri, 26 Apr 2024 23:26:55 -0700 Subject: [PATCH 20/34] chanegs --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a49d6e..001188f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,7 +84,7 @@ jobs: conda --version - name: Install openssl for AlluxioCommon Build - run: sudo apt-get install pkg-config libssl-dev + run: sudo apt-get install libssl-dev # alluxiocommon related - name: Build alluxiocommon wheels From f44d55037394f33e1a1831a4cebbe6211b48d135 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sat, 27 Apr 2024 21:58:00 -0700 Subject: [PATCH 21/34] try to use rustls instead of rust-native-tls as reqwest dependency as rust-native-tls requires dynamic link of tls in runtime env --- .github/workflows/ci.yml | 4 ++-- rust/alluxiocommon/Cargo.toml | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 001188f..3eec3f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,8 +83,8 @@ jobs: conda list conda --version - - name: Install openssl for AlluxioCommon Build - run: sudo apt-get install libssl-dev +# - name: Install openssl for AlluxioCommon Build +# run: sudo apt-get install pkg-config libssl-dev # alluxiocommon related - name: Build alluxiocommon wheels diff --git a/rust/alluxiocommon/Cargo.toml b/rust/alluxiocommon/Cargo.toml index ce4baad..aad1ea7 100644 --- a/rust/alluxiocommon/Cargo.toml +++ b/rust/alluxiocommon/Cargo.toml @@ -13,7 +13,8 @@ name = "alluxiocommon" crate-type = ["cdylib"] [dependencies] -reqwest = { version = "0.11", features = ["blocking"] } +#reqwest = { version = "0.12", features = ["blocking"] } +reqwest = { version = "0.11.16", features = ["blocking", "json", "rustls-tls"], default-features = false } tokio = { version = "1", features = ["full"] } bytes = "1.5.0" rayon = "1.7.0" @@ -23,3 +24,7 @@ log = "0.4.21" [dependencies.pyo3] version = "0.21.1" features = ["extension-module"] + +#[dependencies.openssl] +#version = "0.10" +#features = ["vendored"] From b93e580ba042601cac4fbeb4f9c811f8cc53070a Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sat, 27 Apr 2024 22:05:40 -0700 Subject: [PATCH 22/34] enable run alluxiocommon tests job in ci.yml --- .github/workflows/ci.yml | 24 ++++++++++-------------- rust/alluxiocommon/Cargo.toml | 5 ----- 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3eec3f1..d5394c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,32 +83,28 @@ jobs: conda list conda --version -# - name: Install openssl for AlluxioCommon Build -# run: sudo apt-get install pkg-config libssl-dev - # alluxiocommon related - name: Build alluxiocommon wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} args: --release --out dist -m rust/alluxiocommon/Cargo.toml -# sccache: 'true' manylinux: auto - name: Install dependencies run: | pip install .[tests] + pip install alluxiocommon --find-links dist --force-reinstall -# - name: Run AlluxioCommon tests -# shell: bash -# run: | -# set -e -# pip install alluxiocommon --find-links dist --force-reinstall -# pip install pytest -# pytest -vv \ -# --log-format="%(asctime)s %(levelname)s %(message)s" \ -# --log-date-format="%H:%M:%S" \ -# rust/alluxiocommon/tests/ + - name: Run AlluxioCommon tests + shell: bash + run: | + set -e + pip install pytest + pytest -vv \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%H:%M:%S" \ + rust/alluxiocommon/tests/ - name: Run Alluxio FileSystem tests run: | diff --git a/rust/alluxiocommon/Cargo.toml b/rust/alluxiocommon/Cargo.toml index aad1ea7..b0d7671 100644 --- a/rust/alluxiocommon/Cargo.toml +++ b/rust/alluxiocommon/Cargo.toml @@ -13,7 +13,6 @@ name = "alluxiocommon" crate-type = ["cdylib"] [dependencies] -#reqwest = { version = "0.12", features = ["blocking"] } reqwest = { version = "0.11.16", features = ["blocking", "json", "rustls-tls"], default-features = false } tokio = { version = "1", features = ["full"] } bytes = "1.5.0" @@ -24,7 +23,3 @@ log = "0.4.21" [dependencies.pyo3] version = "0.21.1" features = ["extension-module"] - -#[dependencies.openssl] -#version = "0.10" -#features = ["vendored"] From 0b23d38d7ad6e5429dbb41ef390e11b13277e515 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sat, 27 Apr 2024 22:17:46 -0700 Subject: [PATCH 23/34] reorder to have alluxiocommon install after alluxiocommon wheel building --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5394c7..5bc610b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,10 @@ jobs: conda list conda --version + - name: Install dependencies + run: | + pip install .[tests] + # alluxiocommon related - name: Build alluxiocommon wheels uses: PyO3/maturin-action@v1 @@ -91,15 +95,11 @@ jobs: args: --release --out dist -m rust/alluxiocommon/Cargo.toml manylinux: auto - - name: Install dependencies - run: | - pip install .[tests] - pip install alluxiocommon --find-links dist --force-reinstall - - name: Run AlluxioCommon tests shell: bash run: | set -e + pip install alluxiocommon --find-links dist --force-reinstall pip install pytest pytest -vv \ --log-format="%(asctime)s %(levelname)s %(message)s" \ From e509554b78d51ad3b987646578ec02ac094c301c Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sat, 27 Apr 2024 22:36:39 -0700 Subject: [PATCH 24/34] try fix pip install alluxiocommon from dist/ --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5bc610b..8b3ec24 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: shell: bash run: | set -e - pip install alluxiocommon --find-links dist --force-reinstall + pip install alluxiocommon --find-links=dist/ --force-reinstall pip install pytest pytest -vv \ --log-format="%(asctime)s %(levelname)s %(message)s" \ From 006ffd88481895d234bf6060af4679ee1ae7d1d2 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sat, 27 Apr 2024 22:41:11 -0700 Subject: [PATCH 25/34] add --no-index --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b3ec24..9c4e05a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: shell: bash run: | set -e - pip install alluxiocommon --find-links=dist/ --force-reinstall + pip install alluxiocommon --no-index --find-links=dist/ --force-reinstall pip install pytest pytest -vv \ --log-format="%(asctime)s %(levelname)s %(message)s" \ From e0a62b49787d47fef8b138ad1bef6d80a40e87fe Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sun, 28 Apr 2024 14:09:16 -0700 Subject: [PATCH 26/34] add ls dist/ to debug --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c4e05a..4a8e546 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,10 @@ jobs: args: --release --out dist -m rust/alluxiocommon/Cargo.toml manylinux: auto + + - name: List dist contents + run: ls -l dist/ + - name: Run AlluxioCommon tests shell: bash run: | From b50cd321db22e2b62119b7850edd6948efb1a5ad Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sun, 28 Apr 2024 14:13:07 -0700 Subject: [PATCH 27/34] add pyo3 with python version when buildoing --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a8e546..6874167 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} - args: --release --out dist -m rust/alluxiocommon/Cargo.toml + args: --release -i python${{ matrix.python-version }} --out dist -m rust/alluxiocommon/Cargo.toml manylinux: auto From 7571b50a4504957e389f6bb418326bc5a36a42bf Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sun, 28 Apr 2024 14:19:50 -0700 Subject: [PATCH 28/34] merge pytest for alluxiocommon and alluxiofs together --- .github/workflows/ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6874167..6664a7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,10 +95,6 @@ jobs: args: --release -i python${{ matrix.python-version }} --out dist -m rust/alluxiocommon/Cargo.toml manylinux: auto - - - name: List dist contents - run: ls -l dist/ - - name: Run AlluxioCommon tests shell: bash run: | @@ -109,9 +105,8 @@ jobs: --log-format="%(asctime)s %(levelname)s %(message)s" \ --log-date-format="%H:%M:%S" \ rust/alluxiocommon/tests/ - - - name: Run Alluxio FileSystem tests - run: | +# - name: Run Alluxio FileSystem tests +# run: | pytest -vv \ --log-format="%(asctime)s %(levelname)s %(message)s" \ --log-date-format="%H:%M:%S" \ From 06fd44695db78f41462da003f21f7894187d5bc0 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sun, 28 Apr 2024 14:21:48 -0700 Subject: [PATCH 29/34] syntax fix --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6664a7f..70af213 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,8 +108,8 @@ jobs: # - name: Run Alluxio FileSystem tests # run: | pytest -vv \ - --log-format="%(asctime)s %(levelname)s %(message)s" \ - --log-date-format="%H:%M:%S" \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%H:%M:%S" \ tests/ lint: From c8d9743a495a58f46bc0e54306fed0291d445fca Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sun, 28 Apr 2024 14:23:42 -0700 Subject: [PATCH 30/34] syntax fix --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70af213..636e371 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,6 +95,9 @@ jobs: args: --release -i python${{ matrix.python-version }} --out dist -m rust/alluxiocommon/Cargo.toml manylinux: auto + - name: List dist contents + run: ls -l dist/ + - name: Run AlluxioCommon tests shell: bash run: | @@ -105,8 +108,6 @@ jobs: --log-format="%(asctime)s %(levelname)s %(message)s" \ --log-date-format="%H:%M:%S" \ rust/alluxiocommon/tests/ -# - name: Run Alluxio FileSystem tests -# run: | pytest -vv \ --log-format="%(asctime)s %(levelname)s %(message)s" \ --log-date-format="%H:%M:%S" \ From b2deaff50ddb0cb62de1236aba63a66f63e35361 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sun, 28 Apr 2024 21:14:53 -0700 Subject: [PATCH 31/34] try reorder --- .github/workflows/ci.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 636e371..6ae8467 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,10 +83,6 @@ jobs: conda list conda --version - - name: Install dependencies - run: | - pip install .[tests] - # alluxiocommon related - name: Build alluxiocommon wheels uses: PyO3/maturin-action@v1 @@ -95,13 +91,11 @@ jobs: args: --release -i python${{ matrix.python-version }} --out dist -m rust/alluxiocommon/Cargo.toml manylinux: auto - - name: List dist contents - run: ls -l dist/ - - name: Run AlluxioCommon tests shell: bash run: | set -e + pip install .[tests] pip install alluxiocommon --no-index --find-links=dist/ --force-reinstall pip install pytest pytest -vv \ From 03a16dace402f8e5b87516ed0fa71d5bfb5d5019 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Sun, 28 Apr 2024 21:45:26 -0700 Subject: [PATCH 32/34] name change --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ae8467..79fa19e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: args: --release -i python${{ matrix.python-version }} --out dist -m rust/alluxiocommon/Cargo.toml manylinux: auto - - name: Run AlluxioCommon tests + - name: Install all packages and Run AlluxioCommon tests shell: bash run: | set -e From 1de780880243c842c0c89fb166c8335b60996acc Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Mon, 29 Apr 2024 12:31:20 -0700 Subject: [PATCH 33/34] modify rust/alluxiocommon/README.md and cleanup ci.yml --- .github/workflows/ci.yml | 45 ---------------------------------------- rust/README.md | 34 ++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79fa19e..d57d120 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,51 +7,6 @@ defaults: shell: bash -l -eo pipefail {0} jobs: -# alluxiocommon: -# name: Build alluxiocommon -# runs-on: ubuntu-latest -# timeout-minutes: 10 -# strategy: -# fail-fase: false -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-python@v5 -# with: -# python-version: '3.10' -# - name: Build wheels -# uses: PyO3/maturin-action@v1 -# with: -# target: ${{ matrix.platform.target }} -# args: --release --out dist --find-interpreter --manifest-path rust/alluxiocommon/Cargo.toml -# sccache: 'true' -# manylinux: auto -# - name: pytest -# if: ${{ startsWith(matrix.platform.target, 'x86_64') }} -# shell: bash -# run: | -# set -e -# pip install alluxiocommon --find-links dist --force-reinstall -# pip install pytest -# pytest -vv \ -# --log-format="%(asctime)s %(levelname)s %(message)s" \ -# --log-date-format="%H:%M:%S" \ -# rust/alluxiocommon/tests/ -# - name: pytest -# if: ${{ !startsWith(matrix.platform.target, 'x86') && matrix.platform.target != 'ppc64' }} -# uses: uraimo/run-on-arch-action@v2.5.0 -# with: -# arch: ${{ matrix.platform.target }} -# distro: ubuntu22.04 -# githubToken: ${{ github.token }} -# install: | -# apt-get update -# apt-get install -y --no-install-recommends python3 python3-pip -# pip3 install -U pip pytest -# run: | -# set -e -# pip3 install alluxiocommon --find-links dist --force-reinstall -# cd /root/github/alluxiofs/rust/alluxiocommon && pytest - test: name: Test runs-on: ubuntu-latest diff --git a/rust/README.md b/rust/README.md index 1cab1f6..e2bba63 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,14 +1,36 @@ -## A common native lib for alluxio python client -to build and play: +## A PyO3 based common native extension lib for alluxio python client + +### Developer Prerequisites: +- Install Rust: +https://www.rust-lang.org/tools/install +- Install maturin: +https://www.maturin.rs/installation + + +### To build developer version locally and play: + +1) create virtualenv, (a tool used to create isolated Python environments): -create virtualenv, (a tool used to create isolated Python environments): python3 -m venv .env source .env/bin/activate maturin develop -then can do: +2) then can start using: + python3 - >>> import alluxiocommon - >>> alluxiocommon.multi_http_requests(["http://google.com"],[(0,0)]) + >>> from alluxiocommon import _DataManager + >>> dm = _DataManager(4) + >>> # do something with dm... + +### To build wheel package and install with pip: + + #in rust/alluxiocommon dir: + $ maturin build --out /dist -m /rust/alluxiocommon/Cargo.toml -i python (python version such as 3.8) + #then find .whl package in /dist: + [root@ip-XXX-XX-XX-XX alluxiofs]# ls -l dist/ + total 21848 + -rw-r--r--. 1 root root 22318133 Apr 28 05:31 alluxiocommon-0.1.0-cp38-cp38-linux_x86_64.whl + #install with pip + $ pip install dist/alluxiocommon-0.1.0-cp38-cp38-linux_x86_64.whl --force-reinstall From 8f73b590a58d681fb56d724e9e2acd56e2838347 Mon Sep 17 00:00:00 2001 From: Lucy Ge Date: Mon, 29 Apr 2024 12:33:58 -0700 Subject: [PATCH 34/34] remove comments for debugging purpose before --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index a34b32e..c29c53b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,7 +129,7 @@ def docker_alluxio(): return launch_alluxio_dockers() yield yield_url() - # stop_alluxio_dockers() + stop_alluxio_dockers() @pytest.fixture(scope="session")