diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..434c79fa3 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --package xtask --target-dir target/xtask -- " diff --git a/.config/hakari.toml b/.config/hakari.toml index 3d049eb80..900c2bc75 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -25,3 +25,12 @@ platforms = [ # Write out exact versions rather than a semver range. (Defaults to false.) # exact-versions = true +[traversal-excludes] +workspace-members = [ + "xtask", +] + +[final-excludes] +workspace-members = [ + "xtask", +] diff --git a/.config/nextest.toml b/.config/nextest.toml index 76fd74b5d..6a109e171 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,2 +1,5 @@ +[profile.ci] +fail-fast = false + [profile.ci.junit] path = "junit.xml" diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml new file mode 100644 index 000000000..402f3e470 --- /dev/null +++ b/.github/workflows/audit.yaml @@ -0,0 +1,22 @@ +name: Security audit +on: + push: + paths: + - '.github/workflows/audit.yaml' + - '**/Cargo.toml' + - '**/Cargo.lock' + - 'deny.toml' + schedule: + - cron: '0 0 * * *' +jobs: + audit: + runs-on: ubuntu-latest + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@v4 + + - uses: EmbarkStudios/cargo-deny-action@v1 + with: + rust-version: "stable" diff --git a/.github/workflows/cargo.yaml b/.github/workflows/ci.yaml similarity index 80% rename from .github/workflows/cargo.yaml rename to .github/workflows/ci.yaml index a421c01f7..b93334c0f 100644 --- a/.github/workflows/cargo.yaml +++ b/.github/workflows/ci.yaml @@ -1,10 +1,17 @@ -name: cargo +name: ci on: push: branches: - main - pull_request: {} + + pull_request: + workflow_dispatch: + merge_group: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true jobs: pre-job: @@ -30,8 +37,6 @@ jobs: if: ${{ needs.pre-job.outputs.should_skip != 'true' }} steps: - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - uses: awalsh128/cache-apt-pkgs-action@latest with: @@ -48,14 +53,8 @@ jobs: prefix-key: "v0-rust-${{ steps.setup-rust.outputs.cachekey }}" key: clippy - - uses: cargo-bins/cargo-binstall@main - - - uses: taiki-e/install-action@v2 - with: - tool: just - - name: Make sure code is linted - run: just ci clippy + run: cargo +nightly xtask powerset clippy fmt: name: Fmt @@ -66,8 +65,6 @@ jobs: checks: write steps: - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - uses: dtolnay/rust-toolchain@stable id: setup-rust @@ -75,14 +72,8 @@ jobs: toolchain: nightly components: rustfmt - - uses: cargo-bins/cargo-binstall@main - - - uses: taiki-e/install-action@v2 - with: - tool: just - - name: Make sure code is formatted - run: just ci fmt + run: cargo +nightly fmt --check hakari: name: Hakari @@ -91,8 +82,6 @@ jobs: if: ${{ needs.pre-job.outputs.should_skip != 'true' }} steps: - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - uses: dtolnay/rust-toolchain@stable id: setup-rust @@ -103,10 +92,15 @@ jobs: - uses: taiki-e/install-action@v2 with: - tool: cargo-hakari,just + tool: cargo-hakari - name: Make sure Hakari is up-to-date - run: just ci hakari + run: | + set -xeo pipefail + + cargo hakari manage-deps --dry-run + cargo hakari generate --diff + cargo hakari verify test: name: Test @@ -115,8 +109,6 @@ jobs: if: ${{ needs.pre-job.outputs.should_skip != 'true' }} steps: - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - uses: awalsh128/cache-apt-pkgs-action@latest with: @@ -137,9 +129,12 @@ jobs: - uses: taiki-e/install-action@v2 with: - tool: cargo-nextest,cargo-llvm-cov,just + tool: cargo-nextest,cargo-llvm-cov - - run: just ci test + # Note; we don't run the powerset here because it's very slow on CI + # Perhaps we should consider it at some point. + - name: Run tests + run: cargo +nightly llvm-cov nextest --branch --no-fail-fast --all-features --lcov --output-path ./lcov.info --profile ci - uses: codecov/codecov-action@v5 with: diff --git a/.github/workflows/gh-cache.yaml b/.github/workflows/gh-cache.yaml new file mode 100644 index 000000000..1d082437d --- /dev/null +++ b/.github/workflows/gh-cache.yaml @@ -0,0 +1,28 @@ +name: gh-cache + +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Cleanup + run: | + echo "Fetching list of cache key" + cacheKeysForPR=$(gh cache list --ref $BRANCH --limit 100 --json id --jq '.[].id') + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh cache delete $cacheKey + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + BRANCH: refs/pull/${{ github.event.pull_request.number }}/merge diff --git a/.gitignore b/.gitignore index c8c6b5e7a..36565efdc 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ node_modules/ lcov.info local/ coverage/ +benchmark.txt +.aliases diff --git a/Cargo.lock b/Cargo.lock index b7881b71e..093ed09db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,13 +76,10 @@ dependencies = [ ] [[package]] -name = "ansi_term" -version = "0.11.0" +name = "anes" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" -dependencies = [ - "winapi", -] +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" @@ -135,9 +132,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arbitrary" @@ -202,7 +199,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", "tokio-websockets", "tracing", @@ -282,7 +279,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -339,9 +336,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.3" +version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -365,9 +362,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.64.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35fe5e7f71b1cc6274e905d3bcc7daf94099ac2d4cba83447ffb959b5b27b3c1" +checksum = "d3ba2c5c0f2618937ce3d4a5ad574b86775576fa24006bcb3128c6e2cbf3c34e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -376,7 +373,7 @@ dependencies = [ "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -399,15 +396,15 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09677244a9da92172c8dc60109b4a9658597d4d298b188dd0018b6a66b410ca4" +checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -421,15 +418,15 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.50.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fea2f3a8bb3bd10932ae7ad59cc59f65f270fc9183a7e91f501dc5efbef7ee" +checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -443,15 +440,15 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.50.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ada54e5f26ac246dc79727def52f7f8ed38915cb47781e2a72213957dc3a7d5" +checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -466,9 +463,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.5" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -481,7 +478,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "once_cell", "p256", "percent-encoding", @@ -566,6 +563,15 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +dependencies = [ + "aws-smithy-types", +] + [[package]] name = "aws-smithy-query" version = "0.60.7" @@ -578,9 +584,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.3" +version = "1.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be28bd063fa91fd871d131fc8b68d7cd4c5fa0869bea68daca50dcb1cbd76be2" +checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -613,7 +619,7 @@ dependencies = [ "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "pin-project-lite", "tokio", "tracing", @@ -631,7 +637,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.12", - "http 1.1.0", + "http 1.2.0", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -679,7 +685,7 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -712,7 +718,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", @@ -956,14 +962,20 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.3", + "thiserror 2.0.5", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", @@ -991,7 +1003,7 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" dependencies = [ - "smallvec 1.13.2", + "smallvec", "target-lexicon", ] @@ -1022,6 +1034,33 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -1035,18 +1074,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -1054,11 +1094,23 @@ dependencies = [ "strsim", ] +[[package]] +name = "clap_derive" +version = "4.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "clap_lex" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" @@ -1208,6 +1260,44 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "futures", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -1621,9 +1711,9 @@ checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "fdeflate" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c6f4c64c1d33a3111c4466f7365ebdcc37c5bd1ea0d62aae2e3d722aacbedb" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" dependencies = [ "simd-adler32", ] @@ -1724,9 +1814,9 @@ dependencies = [ [[package]] name = "fred" -version = "10.0.0" +version = "10.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfdd0e46d87d0e8fc1e6636a5dd3b9a5c708722e9662df289ba62a8c198a721" +checksum = "0f5fbcd7118f15ce0ed032105c91137efa563996788a76a770e2fd928ddb243a" dependencies = [ "arc-swap", "async-trait", @@ -1989,7 +2079,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.2.0", "indexmap 2.7.0", "slab", "tokio", @@ -2006,7 +2096,7 @@ dependencies = [ "bytes", "fastrand", "futures-util", - "http 1.1.0", + "http 1.2.0", "pin-project-lite", "tokio", ] @@ -2025,6 +2115,16 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2073,6 +2173,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2118,7 +2224,7 @@ dependencies = [ "parking_lot", "rand", "resolv-conf", - "smallvec 1.13.2", + "smallvec", "thiserror 1.0.69", "tokio", "tracing", @@ -2166,9 +2272,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -2193,7 +2299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -2204,7 +2310,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "pin-project-lite", ] @@ -2271,13 +2377,13 @@ dependencies = [ "futures-channel", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "httparse", "httpdate", "itoa", "pin-project-lite", - "smallvec 1.13.2", + "smallvec", "tokio", "want", ] @@ -2305,13 +2411,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http 1.1.0", + "http 1.2.0", "hyper 1.5.1", "hyper-util", "rustls 0.23.19", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tower-service", "webpki-roots 0.26.7", ] @@ -2338,7 +2444,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "hyper 1.5.1", "pin-project-lite", @@ -2427,7 +2533,7 @@ dependencies = [ "icu_normalizer_data", "icu_properties", "icu_provider", - "smallvec 1.13.2", + "smallvec", "utf16_iter", "utf8_iter", "write16", @@ -2512,7 +2618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ "idna_adapter", - "smallvec 1.13.2", + "smallvec", "utf8_iter", ] @@ -2596,12 +2702,32 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi 0.4.0", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -2657,9 +2783,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -2821,15 +2947,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" -[[package]] -name = "matchers" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" -dependencies = [ - "regex-automata 0.1.10", -] - [[package]] name = "matchers" version = "0.1.0" @@ -2854,12 +2971,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md-5" version = "0.10.6" @@ -3133,7 +3244,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -3152,6 +3263,12 @@ version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + [[package]] name = "openssl-probe" version = "0.1.5" @@ -3181,7 +3298,7 @@ dependencies = [ "opentelemetry", "tracing", "tracing-core", - "tracing-subscriber 0.3.19", + "tracing-subscriber", ] [[package]] @@ -3192,11 +3309,11 @@ checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", - "http 1.1.0", + "http 1.2.0", "opentelemetry", "opentelemetry-proto", "opentelemetry_sdk", - "prost 0.13.3", + "prost 0.13.4", "thiserror 1.0.69", "tokio", "tonic", @@ -3211,7 +3328,7 @@ checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost 0.13.3", + "prost 0.13.4", "tonic", ] @@ -3293,15 +3410,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "owning_ref" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "p256" version = "0.11.1" @@ -3338,7 +3446,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "smallvec 1.13.2", + "smallvec", "windows-targets 0.52.6", ] @@ -3372,8 +3480,8 @@ checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" dependencies = [ "heck", "itertools 0.13.0", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.4", + "prost-types 0.13.4", ] [[package]] @@ -3402,20 +3510,20 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 1.0.69", + "thiserror 2.0.5", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd" +checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" dependencies = [ "pest", "pest_generator", @@ -3423,9 +3531,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e" +checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" dependencies = [ "pest", "pest_meta", @@ -3436,9 +3544,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.14" +version = "2.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d" +checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" dependencies = [ "once_cell", "pest", @@ -3513,11 +3621,39 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "png" -version = "0.17.14" +version = "0.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0" +checksum = "b67582bd5b65bdff614270e2ea89a1cf15bef71245cc1e5f7ea126977144211d" dependencies = [ "bitflags 1.3.2", "crc32fast", @@ -3572,7 +3708,7 @@ dependencies = [ "prost-build 0.12.6", "prost-derive 0.12.6", "sha2", - "smallvec 1.13.2", + "smallvec", "symbolic-demangle", "tempfile", "thiserror 1.0.69", @@ -3660,12 +3796,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", - "prost-derive 0.13.3", + "prost-derive 0.13.4", ] [[package]] @@ -3691,11 +3827,10 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" +checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" dependencies = [ - "bytes", "heck", "itertools 0.13.0", "log", @@ -3703,8 +3838,8 @@ dependencies = [ "once_cell", "petgraph", "prettyplease", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.4", + "prost-types 0.13.4", "regex", "syn 2.0.90", "tempfile", @@ -3725,9 +3860,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", "itertools 0.13.0", @@ -3747,11 +3882,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" dependencies = [ - "prost 0.13.3", + "prost 0.13.4", ] [[package]] @@ -3786,7 +3921,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls 0.23.19", "socket2", - "thiserror 2.0.3", + "thiserror 2.0.5", "tokio", "tracing", ] @@ -3806,7 +3941,7 @@ dependencies = [ "rustls-pki-types", "rustls-platform-verifier", "slab", - "thiserror 2.0.3", + "thiserror 2.0.5", "tinyvec", "tracing", "web-time", @@ -4011,7 +4146,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -4033,7 +4168,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.2", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tower-service", "url", "wasm-bindgen", @@ -4367,6 +4502,8 @@ dependencies = [ name = "scuffle-batching" version = "0.0.2" dependencies = [ + "criterion", + "futures", "scuffle-workspace-hack", "tokio", "tokio-util", @@ -4419,7 +4556,7 @@ dependencies = [ "serde_derive", "smart-default", "tracing", - "tracing-subscriber 0.3.19", + "tracing-subscriber", ] [[package]] @@ -4428,7 +4565,7 @@ version = "0.0.2" dependencies = [ "anyhow", "bytes", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "opentelemetry", @@ -4441,7 +4578,7 @@ dependencies = [ "scuffle-http", "scuffle-pprof", "scuffle-workspace-hack", - "thiserror 2.0.3", + "thiserror 2.0.5", "tokio", "tracing", "tracing-opentelemetry", @@ -4479,7 +4616,7 @@ dependencies = [ "bytes", "futures-util", "h3", - "http 1.1.0", + "http 1.2.0", "pin-project-lite", "scuffle-workspace-hack", "tokio", @@ -4490,14 +4627,13 @@ name = "scuffle-http" version = "0.0.2" dependencies = [ "async-trait", - "axum", "axum-core", "bytes", "derive_more 1.0.0", "futures", "h3", "h3-quinn", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "httpdate", "hyper 1.5.1", @@ -4511,16 +4647,13 @@ dependencies = [ "scuffle-context", "scuffle-h3-webtransport", "scuffle-workspace-hack", - "smallvec 1.13.2", + "smallvec", "spin", - "thiserror 2.0.3", + "thiserror 2.0.5", "tokio", - "tokio-rustls 0.26.0", - "tokio-stream", + "tokio-rustls 0.26.1", "tower-service", "tracing", - "tracing-fmt", - "tracing-subscriber 0.3.19", ] [[package]] @@ -4528,14 +4661,14 @@ name = "scuffle-http-examples" version = "0.0.0" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "scuffle-http", "scuffle-workspace-hack", "tokio", "tracing", - "tracing-subscriber 0.3.19", + "tracing-subscriber", ] [[package]] @@ -4557,7 +4690,7 @@ dependencies = [ "file-format", "fred", "gifski", - "http 1.1.0", + "http 1.2.0", "humantime-serde", "imgref", "libavif-sys", @@ -4566,7 +4699,7 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry_sdk", "png", - "prost 0.13.3", + "prost 0.13.4", "reqwest", "rgb", "scuffle-bootstrap", @@ -4583,11 +4716,11 @@ dependencies = [ "serde_json", "smart-default", "strfmt", - "thiserror 2.0.3", + "thiserror 2.0.5", "tokio", "tonic", "tracing", - "tracing-subscriber 0.3.19", + "tracing-subscriber", "url", ] @@ -4608,8 +4741,8 @@ version = "0.0.1" dependencies = [ "pbjson", "pbjson-build", - "prost 0.13.3", - "prost-build 0.13.3", + "prost 0.13.4", + "prost-build 0.13.4", "scuffle-workspace-hack", "serde", "tonic", @@ -4660,7 +4793,7 @@ dependencies = [ "flate2", "pprof", "scuffle-workspace-hack", - "thiserror 2.0.3", + "thiserror 2.0.5", ] [[package]] @@ -4684,10 +4817,7 @@ dependencies = [ "scuffle-bootstrap", "scuffle-workspace-hack", "serde", - "serde_derive", - "thiserror 2.0.3", - "tracing", - "tracing-subscriber 0.3.19", + "thiserror 2.0.5", ] [[package]] @@ -4726,6 +4856,8 @@ dependencies = [ "bytes", "cc", "chrono", + "clap", + "clap_builder", "config", "crossbeam-utils", "either", @@ -4756,7 +4888,7 @@ dependencies = [ "opentelemetry", "opentelemetry_sdk", "proc-macro2", - "prost 0.13.3", + "prost 0.13.4", "quote", "rand", "regex", @@ -4770,21 +4902,21 @@ dependencies = [ "serde", "serde_json", "simd-adler32", - "smallvec 1.13.2", + "smallvec", "socket2", "spin", "subtle", "syn 2.0.90", "sync_wrapper 1.0.2", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-stream", "tokio-util", "tower 0.5.1", "tracing", "tracing-core", - "tracing-log 0.2.0", - "tracing-subscriber 0.3.19", + "tracing-log", + "tracing-subscriber", "uuid", "zerocopy", ] @@ -5085,15 +5217,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "smallvec" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - [[package]] name = "smallvec" version = "1.13.2" @@ -5317,11 +5440,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "643caef17e3128658ff44d85923ef2d28af81bb71e0d67bbfe1d76f19a73e053" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.5", ] [[package]] @@ -5337,9 +5460,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "995d0bbc9995d1f19d28b7215a9352b0fc3cd3a2d2ec95c2cadc485cdedbcdde" dependencies = [ "proc-macro2", "quote", @@ -5358,9 +5481,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -5379,9 +5502,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -5406,6 +5529,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -5423,9 +5556,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -5462,20 +5595,19 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ "rustls 0.23.19", - "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -5484,9 +5616,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -5506,14 +5638,14 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.2.0", "httparse", "rand", "ring", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", ] @@ -5563,7 +5695,7 @@ dependencies = [ "base64 0.22.1", "bytes", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -5571,7 +5703,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.3", + "prost 0.13.4", "socket2", "tokio", "tokio-stream", @@ -5589,8 +5721,8 @@ checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" dependencies = [ "prettyplease", "proc-macro2", - "prost-build 0.13.3", - "prost-types 0.13.3", + "prost-build 0.13.4", + "prost-types 0.13.4", "quote", "syn 2.0.90", ] @@ -5676,26 +5808,6 @@ dependencies = [ "valuable", ] -[[package]] -name = "tracing-fmt" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "880547feb88739526e322a366be2411c41c797f0dabcddfe99a3216e5a664f71" -dependencies = [ - "tracing-subscriber 0.1.6", -] - -[[package]] -name = "tracing-log" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-log" version = "0.2.0" @@ -5717,11 +5829,11 @@ dependencies = [ "once_cell", "opentelemetry", "opentelemetry_sdk", - "smallvec 1.13.2", + "smallvec", "tracing", "tracing-core", - "tracing-log 0.2.0", - "tracing-subscriber 0.3.19", + "tracing-log", + "tracing-subscriber", "web-time", ] @@ -5735,41 +5847,24 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-subscriber" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "192ca16595cdd0661ce319e8eede9c975f227cdaabc4faaefdc256f43d852e45" -dependencies = [ - "ansi_term", - "chrono", - "lazy_static", - "matchers 0.0.1", - "owning_ref", - "regex", - "smallvec 0.6.14", - "tracing-core", - "tracing-log 0.1.4", -] - [[package]] name = "tracing-subscriber" version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ - "matchers 0.1.0", + "matchers", "nu-ansi-term", "once_cell", "regex", "serde", "serde_json", "sharded-slab", - "smallvec 1.13.2", + "smallvec", "thread_local", "tracing", "tracing-core", - "tracing-log 0.2.0", + "tracing-log", "tracing-serde", ] @@ -5966,9 +6061,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -5977,13 +6072,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", "syn 2.0.90", @@ -5992,9 +6086,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.47" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", @@ -6005,9 +6099,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6015,9 +6109,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", @@ -6028,15 +6122,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "web-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -6364,6 +6458,18 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xtask" +version = "0.1.0" +dependencies = [ + "anyhow", + "cargo_metadata", + "clap", + "serde", + "serde_derive", + "serde_json", +] + [[package]] name = "yaml-rust2" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index f348b1eb4..217a8c2fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "apps/image-processor/proto", "apps/image-processor/examples", "crates/workspace-hack", + "dev-tools/xtask", ] resolver = "2" diff --git a/Justfile b/Justfile index 38ea2e733..569859062 100644 --- a/Justfile +++ b/Justfile @@ -1,4 +1,13 @@ -mod ci 'just/ci.just' +# An alias for cargo +nightly xtask check +powerset *args: + cargo +nightly xtask powerset {{args}} + +# An alias for cargo +nightly fmt --all +fmt *args: + cargo +nightly fmt --all {{args}} + +lint *args: + cargo +nightly clippy --fix --allow-dirty --all-targets --all-features --allow-staged {{args}} test *args: #!/bin/bash @@ -15,12 +24,9 @@ test *args: cargo llvm-cov report --html cargo llvm-cov report --lcov --output-path ./lcov.info -hakari: - cargo +nightly hakari generate - cargo +nightly hakari manage-deps - -clippy: - cargo +nightly clippy --fix --allow-dirty --all-targets --all-features +deny *args: + cargo deny {{args}} --all-features check -fmt: - cargo +nightly fmt --all +workspace-hack: + cargo hakari manage-deps + cargo hakari generate diff --git a/README.md b/README.md index b813fc5f6..4b702a7c2 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,18 @@

- Twitter -   - Bluesky -   - Discord -   - LinkedIn + + Twitter + + + Bluesky + + + Discord + + + LinkedIn +

diff --git a/apps/image-processor/Cargo.toml b/apps/image-processor/Cargo.toml index 614fa302c..f516b8648 100644 --- a/apps/image-processor/Cargo.toml +++ b/apps/image-processor/Cargo.toml @@ -26,17 +26,17 @@ file-format = "0.26.0" rgb = "0.8" imgref = "1.10" libavif-sys = { version = "0.17.0", features = ["codec-dav1d", "codec-rav1e"], default-features = false } -libwebp-sys2 = { version = "0.1", features = ["1_2", "demux", "mux", "static"] } +libwebp-sys2 = { version = "0.1.9", features = ["1_2", "demux", "mux", "static"] } gifski = { version = "1.13", default-features = false, features = ["gifsicle"] } png = "0.17" -bytes = "1.0" +bytes = "1" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } fast_image_resize = "5.0.0" chrono = { version = "0.4", features = ["serde"] } url = { version = "2", features = ["serde"] } http = "1" humantime-serde = "1" -smart-default = "0.7.1" +smart-default = "0.7" axum = { version = "0.7" } tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } mongodb = { version = "3" } diff --git a/apps/image-processor/proto/Cargo.toml b/apps/image-processor/proto/Cargo.toml index 578f077eb..96bd803ce 100644 --- a/apps/image-processor/proto/Cargo.toml +++ b/apps/image-processor/proto/Cargo.toml @@ -31,3 +31,11 @@ serde = [ "pbjson", ] +[package.metadata.xtask] +# Even though these features effect the build.rs they are additive because they do not +# effect each other. +addative-features = [ + "server", + "client", + "serde", +] diff --git a/codecov.yml b/codecov.yml index 349530758..d513377bc 100644 --- a/codecov.yml +++ b/codecov.yml @@ -20,6 +20,9 @@ comment: require_head: no require_base: no +ignore: + - "dev-tools/**" + component_management: individual_components: - component_id: scuffle-batching diff --git a/crates/batching/Cargo.toml b/crates/batching/Cargo.toml index fe1725a68..fbcacb890 100644 --- a/crates/batching/Cargo.toml +++ b/crates/batching/Cargo.toml @@ -10,7 +10,24 @@ license = "MIT OR Apache-2.0" description = "Optimized batching and dataloading for external services." keywords = ["batching", "dataloading", "external", "services", "async"] +[[bench]] +name = "scuffle-batching-batcher" +harness = false +path = "benchmarks/batcher.rs" + +[[bench]] +name = "scuffle-batching-dataloader" +harness = false +path = "benchmarks/dataloader.rs" + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage_nightly)'] } + [dependencies] tokio = { version = "1", default-features = false, features = ["time", "sync", "rt"] } tokio-util = "0.7" scuffle-workspace-hack.workspace = true + +[dev-dependencies] +criterion = { version = "0.5.1", features = ["async_tokio"] } +futures = "0.3" diff --git a/crates/batching/benchmarks/batcher.rs b/crates/batching/benchmarks/batcher.rs new file mode 100644 index 000000000..a749ad611 --- /dev/null +++ b/crates/batching/benchmarks/batcher.rs @@ -0,0 +1,109 @@ +use std::future::Future; +use std::marker::PhantomData; +use std::sync::Arc; + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use scuffle_batching::batch::BatchResponse; + +struct DataloaderImpl(F, PhantomData

); + +impl DataloaderImpl { + fn new(f: F) -> Self { + Self(f, PhantomData) + } +} + +impl scuffle_batching::BatchExecutor for DataloaderImpl +where + F: Fn(Vec<(usize, BatchResponse)>) -> Fut + Send + Sync, + Fut: Future + Send, + Self: Send + Sync, +{ + type Request = usize; + type Response = usize; + + async fn execute(&self, keys: Vec<(Self::Request, BatchResponse)>) { + (self.0)(keys).await; + } +} + +async fn run_scuffle_batcher_many( + size: usize, + loader: impl scuffle_batching::BatchExecutor + Send + Sync + 'static, +) { + let batcher = Arc::new(scuffle_batching::Batcher::new( + loader, + size / 2, + 100, + std::time::Duration::from_millis(5), + )); + + let spawn = || { + let batcher = batcher.clone(); + tokio::spawn(async move { batcher.execute_many(0..size / 4).await }) + }; + + futures::future::join_all([spawn(), spawn(), spawn(), spawn()]).await; +} + +async fn run_scuffle_batcher_single( + size: usize, + loader: impl scuffle_batching::BatchExecutor + Send + Sync + 'static, +) { + let batcher = Arc::new(scuffle_batching::Batcher::new( + loader, + size / 2, + 100, + std::time::Duration::from_millis(5), + )); + + let spawn = |i| { + let batcher = batcher.clone(); + tokio::spawn(async move { batcher.execute(i).await }) + }; + + futures::future::join_all((0..size / 4).cycle().take(size).map(spawn)).await; +} + +fn delay(c: &mut Criterion) { + let size: usize = 1000; + + let mut group = c.benchmark_group("batcher - delay"); + + let runtime = || tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); + + group.bench_with_input(BenchmarkId::new("many", size), &size, |b, &s| { + b.to_async(runtime()).iter(|| async move { + run_scuffle_batcher_many( + s, + DataloaderImpl::new(|keys: Vec<(usize, BatchResponse)>| async move { + black_box(tokio::time::sleep(std::time::Duration::from_millis(1))).await; + for (key, resp) in keys { + resp.send(black_box(key)); + } + }), + ) + .await; + }); + }); + + group.bench_with_input(BenchmarkId::new("single", size), &size, |b, &s| { + b.to_async(runtime()).iter(|| async move { + run_scuffle_batcher_single( + s, + DataloaderImpl::new(|keys: Vec<(usize, BatchResponse)>| async move { + black_box(tokio::time::sleep(std::time::Duration::from_millis(1))).await; + for (key, resp) in keys { + resp.send(black_box(key)); + } + }), + ) + .await; + }); + }); + + group.finish(); +} + +criterion_group!(benches, delay); +criterion_main!(benches); diff --git a/crates/batching/benchmarks/dataloader.rs b/crates/batching/benchmarks/dataloader.rs new file mode 100644 index 000000000..9fc819436 --- /dev/null +++ b/crates/batching/benchmarks/dataloader.rs @@ -0,0 +1,105 @@ +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::marker::PhantomData; +use std::sync::Arc; + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; + +struct DataloaderImpl(F, PhantomData

); + +impl DataloaderImpl { + fn new(f: F) -> Self { + Self(f, PhantomData) + } +} + +impl scuffle_batching::DataLoaderFetcher for DataloaderImpl +where + F: Fn(HashSet) -> Fut + Send + Sync, + Fut: Future>> + Send, + Self: Send + Sync, +{ + type Key = usize; + type Value = usize; + + async fn load(&self, keys: HashSet) -> Option> { + (self.0)(keys).await + } +} + +async fn run_scuffle_dataloader_many( + size: usize, + loader: impl scuffle_batching::DataLoaderFetcher + Send + Sync + 'static, +) { + let dataloader = Arc::new(scuffle_batching::DataLoader::new( + loader, + size / 2, + 100, + std::time::Duration::from_millis(5), + )); + + let spawn = || { + let dataloader = dataloader.clone(); + tokio::spawn(async move { dataloader.load_many(0..size).await }) + }; + + futures::future::join_all([spawn(), spawn(), spawn(), spawn()]).await; +} + +async fn run_scuffle_dataloader_single( + size: usize, + loader: impl scuffle_batching::DataLoaderFetcher + Send + Sync + 'static, +) { + let dataloader = Arc::new(scuffle_batching::DataLoader::new( + loader, + size / 2, + 100, + std::time::Duration::from_millis(5), + )); + + let spawn = |i| { + let dataloader = dataloader.clone(); + tokio::spawn(async move { dataloader.load(i).await }) + }; + + futures::future::join_all((0..size).cycle().take(size * 4).map(spawn)).await; +} + +fn delay(c: &mut Criterion) { + let size: usize = 1000; + + let mut group = c.benchmark_group("dataloader - delay"); + + let runtime = || tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); + + group.bench_with_input(BenchmarkId::new("many", size), &size, |b, &s| { + b.to_async(runtime()).iter(|| async move { + run_scuffle_dataloader_many( + s, + DataloaderImpl::new(|keys: HashSet| async move { + black_box(tokio::time::sleep(std::time::Duration::from_millis(1))).await; + black_box(Some(keys.into_iter().map(black_box).map(|k| (k, k)).collect())) + }), + ) + .await; + }); + }); + + group.bench_with_input(BenchmarkId::new("single", size), &size, |b, &s| { + b.to_async(runtime()).iter(|| async move { + run_scuffle_dataloader_single( + s, + DataloaderImpl::new(|keys: HashSet| async move { + black_box(tokio::time::sleep(std::time::Duration::from_millis(1))).await; + black_box(Some(keys.into_iter().map(black_box).map(|k| (k, k)).collect())) + }), + ) + .await; + }); + }); + + group.finish(); +} + +criterion_group!(benches, delay); +criterion_main!(benches); diff --git a/crates/batching/examples/Cargo.toml b/crates/batching/examples/Cargo.toml index 5e570d161..14dfba8ae 100644 --- a/crates/batching/examples/Cargo.toml +++ b/crates/batching/examples/Cargo.toml @@ -2,6 +2,10 @@ name = "scuffle-batching-examples" version = "0.1.0" edition = "2021" +repository = "https://github.com/scufflecloud/scuffle" +authors = ["Scuffle "] +readme = "README.md" +license = "MIT OR Apache-2.0" [[example]] name = "scuffle-batching-dataloader" diff --git a/crates/batching/examples/LICENSE.Apache-2.0 b/crates/batching/examples/LICENSE.Apache-2.0 new file mode 120000 index 000000000..1c5185b65 --- /dev/null +++ b/crates/batching/examples/LICENSE.Apache-2.0 @@ -0,0 +1 @@ +../../../LICENSE.Apache-2.0 \ No newline at end of file diff --git a/crates/batching/examples/LICENSE.MIT b/crates/batching/examples/LICENSE.MIT new file mode 120000 index 000000000..4cf3365c6 --- /dev/null +++ b/crates/batching/examples/LICENSE.MIT @@ -0,0 +1 @@ +../../../LICENSE.MIT \ No newline at end of file diff --git a/crates/batching/examples/README.md b/crates/batching/examples/README.md new file mode 100644 index 000000000..5cda147b8 --- /dev/null +++ b/crates/batching/examples/README.md @@ -0,0 +1,29 @@ +# scuffle-batching-examples + +> [!WARNING] +> This crate is under active development and may not be stable. + + [![crates.io](https://img.shields.io/crates/v/scuffle-batching-examples.svg)](https://crates.io/crates/scuffle-batching-examples) [![docs.rs](https://img.shields.io/docsrs/scuffle-batching-examples)](https://docs.rs/scuffle-batching-examples) + +--- + +A collection of examples for scuffle-batching. + +For more information checkout the [scuffle-batching](../README.md) crate. + +## Examples + +- [dataloader](./dataloader) - A basic example of a scuffle-batching application. + +## Status + +This crate is currently under development and is not yet stable, unit tests are not yet fully implemented. + +Unit tests are not yet fully implemented. Use at your own risk. + +## License + +This project is licensed under the [MIT](./LICENSE.MIT) or [Apache-2.0](./LICENSE.Apache-2.0) license. +You can choose between one of them if you use this work. + +`SPDX-License-Identifier: MIT OR Apache-2.0` diff --git a/crates/batching/src/batch.rs b/crates/batching/src/batch.rs index cc74cae44..e6d7d6ab1 100644 --- a/crates/batching/src/batch.rs +++ b/crates/batching/src/batch.rs @@ -1,5 +1,4 @@ use std::future::Future; -use std::sync::atomic::AtomicU64; use std::sync::Arc; use tokio::sync::oneshot; @@ -39,6 +38,24 @@ impl BatchResponse { { self.send(Err(error).into()) } + + /// Send a `None` response back to the requestor + #[inline(always)] + pub fn send_none(self) + where + Resp: From>, + { + self.send(None.into()) + } + + /// Send a value response back to the requestor + #[inline(always)] + pub fn send_some(self, value: T) + where + Resp: From>, + { + self.send(Some(value).into()) + } } /// A trait for executing batches @@ -57,66 +74,75 @@ pub trait BatchExecutor { /// A builder for a [`Batcher`] #[derive(Clone, Copy, Debug)] #[must_use = "builders must be used to create a batcher"] -pub struct BatcherBuilder { +pub struct BatcherBuilder { batch_size: usize, concurrency: usize, delay: std::time::Duration, + _marker: std::marker::PhantomData, } -impl Default for BatcherBuilder { +impl Default for BatcherBuilder { fn default() -> Self { Self::new() } } -impl BatcherBuilder { +impl BatcherBuilder { /// Create a new builder - pub fn new() -> Self { + pub const fn new() -> Self { Self { batch_size: 1000, concurrency: 50, delay: std::time::Duration::from_millis(5), + _marker: std::marker::PhantomData, } } /// Set the batch size - pub fn batch_size(mut self, batch_size: usize) -> Self { - self.batch_size = batch_size; + #[inline] + pub const fn batch_size(mut self, batch_size: usize) -> Self { + self.with_batch_size(batch_size); self } /// Set the delay - pub fn delay(mut self, delay: std::time::Duration) -> Self { - self.delay = delay; + #[inline] + pub const fn delay(mut self, delay: std::time::Duration) -> Self { + self.with_delay(delay); self } /// Set the concurrency to 1 - pub fn concurrency(mut self, concurrency: usize) -> Self { - self.concurrency = concurrency; + #[inline] + pub const fn concurrency(mut self, concurrency: usize) -> Self { + self.with_concurrency(concurrency); self } /// Set the concurrency - pub fn with_concurrency(&mut self, concurrency: usize) -> &mut Self { + #[inline] + pub const fn with_concurrency(&mut self, concurrency: usize) -> &mut Self { self.concurrency = concurrency; self } /// Set the batch size - pub fn with_batch_size(&mut self, batch_size: usize) -> &mut Self { + #[inline] + pub const fn with_batch_size(&mut self, batch_size: usize) -> &mut Self { self.batch_size = batch_size; self } /// Set the delay - pub fn with_delay(&mut self, delay: std::time::Duration) -> &mut Self { + #[inline] + pub const fn with_delay(&mut self, delay: std::time::Duration) -> &mut Self { self.delay = delay; self } /// Build the batcher - pub fn build(self, executor: E) -> Batcher + #[inline] + pub fn build(self, executor: E) -> Batcher where E: BatchExecutor + Send + Sync + 'static, { @@ -135,16 +161,15 @@ where semaphore: Arc, current_batch: Arc>>>, batch_size: usize, - batch_id: AtomicU64, } struct Batch where E: BatchExecutor + Send + Sync + 'static, { - id: u64, items: Vec<(E::Request, BatchResponse)>, semaphore: Arc, + created_at: std::time::Instant, } impl Batcher @@ -165,12 +190,11 @@ where semaphore, current_batch, batch_size: batch_size.max(1), - batch_id: AtomicU64::new(0), } } /// Create a builder for a [`Batcher`] - pub fn builder() -> BatcherBuilder { + pub const fn builder() -> BatcherBuilder { BatcherBuilder::new() } @@ -191,10 +215,7 @@ where for item in items { if batch.is_none() { - batch.replace(Batch::new( - self.batch_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed), - self.semaphore.clone(), - )); + batch.replace(Batch::new(self.semaphore.clone())); } let batch_mut = batch.as_mut().unwrap(); @@ -224,22 +245,23 @@ async fn batch_loop( ) where E: BatchExecutor + Send + Sync + 'static, { - let mut pending_id = None; + let mut delay_delta = delay; loop { - tokio::time::sleep(delay).await; + tokio::time::sleep(delay_delta).await; let mut batch = current_batch.lock().await; - let Some(batch_id) = batch.as_ref().map(|b| b.id) else { - pending_id = None; + let Some(created_at) = batch.as_ref().map(|b| b.created_at) else { + delay_delta = delay; continue; }; - if pending_id != Some(batch_id) || batch.as_ref().unwrap().items.is_empty() { - pending_id = Some(batch_id); - continue; + let remaining = delay.saturating_sub(created_at.elapsed()); + if remaining == std::time::Duration::ZERO { + tokio::spawn(batch.take().unwrap().spawn(executor.clone())); + delay_delta = delay; + } else { + delay_delta = remaining; } - - tokio::spawn(batch.take().unwrap().spawn(executor.clone())); } } @@ -247,9 +269,9 @@ impl Batch where E: BatchExecutor + Send + Sync + 'static, { - fn new(id: u64, semaphore: Arc) -> Self { + fn new(semaphore: Arc) -> Self { Self { - id, + created_at: std::time::Instant::now(), items: Vec::new(), semaphore, } @@ -260,3 +282,355 @@ where executor.execute(self.items).await; } } + +#[cfg_attr(all(coverage_nightly, test), coverage(off))] +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::atomic::AtomicUsize; + + use super::*; + + struct TestExecutor { + values: HashMap, + delay: std::time::Duration, + requests: Arc, + capacity: usize, + } + + impl BatchExecutor for TestExecutor + where + K: Clone + Eq + std::hash::Hash + Send + Sync + 'static, + V: Clone + Send + Sync + 'static, + { + type Request = K; + type Response = V; + + async fn execute(&self, requests: Vec<(Self::Request, BatchResponse)>) { + tokio::time::sleep(self.delay).await; + + assert!(requests.len() <= self.capacity); + + self.requests.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + for (request, response) in requests { + if let Some(value) = self.values.get(&request) { + response.send(value.clone()); + } + } + } + } + + #[tokio::test] + async fn basic() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestExecutor { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = Batcher::builder().batch_size(2).concurrency(1).build(fetcher); + + let start = std::time::Instant::now(); + let a = loader.execute("a").await; + assert_eq!(a, Some(1)); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 1); + + let start = std::time::Instant::now(); + let b = loader.execute("b").await; + assert_eq!(b, Some(2)); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 2); + let start = std::time::Instant::now(); + let c = loader.execute("c").await; + assert_eq!(c, Some(3)); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 3); + + let start = std::time::Instant::now(); + let ab = loader.execute_many(vec!["a", "b"]).await; + assert_eq!(ab, vec![Some(1), Some(2)]); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 4); + + let start = std::time::Instant::now(); + let unknown = loader.execute("unknown").await; + assert_eq!(unknown, None); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 5); + } + + #[tokio::test] + async fn concurrency_high() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestExecutor { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = Batcher::builder().batch_size(2).concurrency(10).build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader + .execute_many(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + .await; + assert_eq!(ab, vec![Some(1), Some(2), Some(3), None, None, None, None, None, None, None]); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 5); + } + + #[tokio::test] + async fn delay_low() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestExecutor { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = Batcher::builder() + .batch_size(2) + .concurrency(1) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader + .execute_many(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + .await; + assert_eq!(ab, vec![Some(1), Some(2), Some(3), None, None, None, None, None, None, None]); + assert!(start.elapsed() < std::time::Duration::from_millis(35)); + assert!(start.elapsed() >= std::time::Duration::from_millis(25)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 5); + } + + #[tokio::test] + async fn batch_size() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestExecutor { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 100, + }; + + let loader = BatcherBuilder::default() + .batch_size(100) + .concurrency(1) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader + .execute_many(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + .await; + assert_eq!(ab, vec![Some(1), Some(2), Some(3), None, None, None, None, None, None, None]); + assert!(start.elapsed() >= std::time::Duration::from_millis(10)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn high_concurrency() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestExecutor { + values: HashMap::from_iter((0..1134).map(|i| (i, i * 2 + 5))), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 100, + }; + + let loader = BatcherBuilder::default() + .batch_size(100) + .concurrency(10) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader.execute_many(0..1134).await; + assert_eq!(ab, (0..1134).map(|i| Some(i * 2 + 5)).collect::>()); + assert!(start.elapsed() >= std::time::Duration::from_millis(15)); + assert!(start.elapsed() < std::time::Duration::from_millis(25)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 1134 / 100 + 1); + } + + #[tokio::test] + async fn delayed_start() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestExecutor { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = BatcherBuilder::default() + .batch_size(2) + .concurrency(100) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + + let start = std::time::Instant::now(); + let ab = loader + .execute_many(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + .await; + assert_eq!(ab, vec![Some(1), Some(2), Some(3), None, None, None, None, None, None, None]); + assert!(start.elapsed() >= std::time::Duration::from_millis(5)); + assert!(start.elapsed() < std::time::Duration::from_millis(25)); + } + + #[tokio::test] + async fn delayed_start_single() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestExecutor { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = BatcherBuilder::default() + .batch_size(2) + .concurrency(100) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + + let start = std::time::Instant::now(); + let ab = loader.execute_many(vec!["a"]).await; + assert_eq!(ab, vec![Some(1)]); + assert!(start.elapsed() >= std::time::Duration::from_millis(15)); + assert!(start.elapsed() < std::time::Duration::from_millis(20)); + } + + #[tokio::test] + async fn no_deduplication() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestExecutor { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 4, + }; + + let loader = BatcherBuilder::default() + .batch_size(4) + .concurrency(1) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader.execute_many(vec!["a", "a", "b", "b", "c", "c"]).await; + assert_eq!(ab, vec![Some(1), Some(1), Some(2), Some(2), Some(3), Some(3)]); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 2); + assert!(start.elapsed() >= std::time::Duration::from_millis(5)); + assert!(start.elapsed() < std::time::Duration::from_millis(20)); + } + + #[tokio::test] + async fn result() { + let requests = Arc::new(AtomicUsize::new(0)); + + struct TestExecutor(Arc); + + impl BatchExecutor for TestExecutor { + type Request = &'static str; + type Response = Result; + + async fn execute(&self, requests: Vec<(Self::Request, BatchResponse)>) { + self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + for (request, response) in requests { + match request.parse() { + Ok(value) => response.send_ok(value), + Err(_) => response.send_err(()), + } + } + } + } + + let loader = BatcherBuilder::default() + .batch_size(4) + .concurrency(1) + .delay(std::time::Duration::from_millis(10)) + .build(TestExecutor(requests.clone())); + + let start = std::time::Instant::now(); + let ab = loader.execute_many(vec!["1", "1", "2", "2", "3", "3", "hello"]).await; + assert_eq!( + ab, + vec![ + Some(Ok(1)), + Some(Ok(1)), + Some(Ok(2)), + Some(Ok(2)), + Some(Ok(3)), + Some(Ok(3)), + Some(Err(())) + ] + ); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 2); + assert!(start.elapsed() >= std::time::Duration::from_millis(5)); + assert!(start.elapsed() < std::time::Duration::from_millis(20)); + } + + #[tokio::test] + async fn option() { + let requests = Arc::new(AtomicUsize::new(0)); + + struct TestExecutor(Arc); + + impl BatchExecutor for TestExecutor { + type Request = &'static str; + type Response = Option; + + async fn execute(&self, requests: Vec<(Self::Request, BatchResponse)>) { + self.0.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + for (request, response) in requests { + match request.parse() { + Ok(value) => response.send_some(value), + Err(_) => response.send_none(), + } + } + } + } + + let loader = BatcherBuilder::default() + .batch_size(4) + .concurrency(1) + .delay(std::time::Duration::from_millis(10)) + .build(TestExecutor(requests.clone())); + + let start = std::time::Instant::now(); + let ab = loader.execute_many(vec!["1", "1", "2", "2", "3", "3", "hello"]).await; + assert_eq!( + ab, + vec![ + Some(Some(1)), + Some(Some(1)), + Some(Some(2)), + Some(Some(2)), + Some(Some(3)), + Some(Some(3)), + Some(None) + ] + ); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 2); + assert!(start.elapsed() >= std::time::Duration::from_millis(5)); + assert!(start.elapsed() < std::time::Duration::from_millis(20)); + } +} diff --git a/crates/batching/src/dataloader.rs b/crates/batching/src/dataloader.rs index e518e91b4..352aaa594 100644 --- a/crates/batching/src/dataloader.rs +++ b/crates/batching/src/dataloader.rs @@ -1,6 +1,5 @@ use std::collections::{HashMap, HashSet}; use std::future::Future; -use std::sync::atomic::AtomicU64; use std::sync::Arc; /// A trait for fetching data in batches @@ -17,48 +16,75 @@ pub trait DataLoaderFetcher { /// A builder for a [`DataLoader`] #[derive(Clone, Copy, Debug)] #[must_use = "builders must be used to create a dataloader"] -pub struct DataLoaderBuilder { +pub struct DataLoaderBuilder { batch_size: usize, concurrency: usize, delay: std::time::Duration, + _phantom: std::marker::PhantomData, } -impl Default for DataLoaderBuilder { +impl Default for DataLoaderBuilder { fn default() -> Self { Self::new() } } -impl DataLoaderBuilder { +impl DataLoaderBuilder { /// Create a new builder - pub fn new() -> Self { + pub const fn new() -> Self { Self { batch_size: 1000, concurrency: 50, delay: std::time::Duration::from_millis(5), + _phantom: std::marker::PhantomData, } } /// Set the batch size - pub fn batch_size(mut self, batch_size: usize) -> Self { + #[inline] + pub const fn batch_size(mut self, batch_size: usize) -> Self { + self.with_batch_size(batch_size); + self + } + + /// Set the delay + #[inline] + pub const fn delay(mut self, delay: std::time::Duration) -> Self { + self.with_delay(delay); + self + } + + /// Set the concurrency + #[inline] + pub const fn concurrency(mut self, concurrency: usize) -> Self { + self.with_concurrency(concurrency); + self + } + + /// Set the batch size + #[inline] + pub const fn with_batch_size(&mut self, batch_size: usize) -> &mut Self { self.batch_size = batch_size; self } /// Set the delay - pub fn delay(mut self, delay: std::time::Duration) -> Self { + #[inline] + pub const fn with_delay(&mut self, delay: std::time::Duration) -> &mut Self { self.delay = delay; self } /// Set the concurrency - pub fn concurrency(mut self, concurrency: usize) -> Self { + #[inline] + pub const fn with_concurrency(&mut self, concurrency: usize) -> &mut Self { self.concurrency = concurrency; self } /// Build the dataloader - pub fn build(self, executor: E) -> DataLoader + #[inline] + pub fn build(self, executor: E) -> DataLoader where E: DataLoaderFetcher + Send + Sync + 'static, { @@ -77,7 +103,6 @@ where semaphore: Arc, current_batch: Arc>>>, batch_size: usize, - batch_id: AtomicU64, } impl DataLoader @@ -98,12 +123,12 @@ where semaphore, current_batch, batch_size: batch_size.max(1), - batch_id: AtomicU64::new(0), } } /// Create a builder for a [`DataLoader`] - pub fn builder() -> DataLoaderBuilder { + #[inline] + pub const fn builder() -> DataLoaderBuilder { DataLoaderBuilder::new() } @@ -127,7 +152,6 @@ where I: IntoIterator + Send, { struct BatchWaiting { - id: u64, keys: HashSet, result: Arc>, } @@ -137,22 +161,21 @@ where let mut count = 0; { + let mut new_batch = false; let mut batch = self.current_batch.lock().await; for item in items { if batch.is_none() { - batch.replace(Batch::new( - self.batch_id.fetch_add(1, std::sync::atomic::Ordering::Relaxed), - self.semaphore.clone(), - )); + batch.replace(Batch::new(self.semaphore.clone())); + new_batch = true; } let batch_mut = batch.as_mut().unwrap(); batch_mut.items.insert(item.clone()); - if waiters.is_empty() || waiters.last().unwrap().id != batch_mut.id { + if new_batch { + new_batch = false; waiters.push(BatchWaiting { - id: batch_mut.id, keys: HashSet::new(), result: batch_mut.result.clone(), }); @@ -189,22 +212,23 @@ async fn batch_loop( ) where E: DataLoaderFetcher + Send + Sync + 'static, { - let mut pending_id = None; + let mut delay_delta = delay; loop { - tokio::time::sleep(delay).await; + tokio::time::sleep(delay_delta).await; let mut batch = current_batch.lock().await; - let Some(batch_id) = batch.as_ref().map(|b| b.id) else { - pending_id = None; + let Some(created_at) = batch.as_ref().map(|b| b.created_at) else { + delay_delta = delay; continue; }; - if pending_id != Some(batch_id) || batch.as_ref().unwrap().items.is_empty() { - pending_id = Some(batch_id); - continue; + let remaining = delay.saturating_sub(created_at.elapsed()); + if remaining == std::time::Duration::ZERO { + tokio::spawn(batch.take().unwrap().spawn(executor.clone())); + delay_delta = delay; + } else { + delay_delta = remaining; } - - tokio::spawn(batch.take().unwrap().spawn(executor.clone())); } } @@ -234,22 +258,22 @@ struct Batch where E: DataLoaderFetcher + Send + Sync + 'static, { - id: u64, items: HashSet, result: Arc>, semaphore: Arc, + created_at: std::time::Instant, } impl Batch where E: DataLoaderFetcher + Send + Sync + 'static, { - fn new(id: u64, semaphore: Arc) -> Self { + fn new(semaphore: Arc) -> Self { Self { - id, items: HashSet::new(), result: Arc::new(BatchResult::new()), semaphore, + created_at: std::time::Instant::now(), } } @@ -257,11 +281,278 @@ where let _drop_guard = self.result.token.clone().drop_guard(); let _ticket = self.semaphore.acquire_owned().await.unwrap(); let result = executor.load(self.items).await; - match self.result.values.set(result) { - Ok(()) => {} - Err(_) => unreachable!( + + #[cfg_attr(all(coverage_nightly, test), coverage(off))] + fn unknwown_error(_: E) -> ! { + unreachable!( "batch result already set, this is a bug please report it https://github.com/scufflecloud/scuffle/issues" - ), + ) } + + self.result.values.set(result).map_err(unknwown_error).unwrap(); + } +} + +#[cfg_attr(all(coverage_nightly, test), coverage(off))] +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicUsize; + + use super::*; + + struct TestFetcher { + values: HashMap, + delay: std::time::Duration, + requests: Arc, + capacity: usize, + } + + impl DataLoaderFetcher for TestFetcher + where + K: Clone + Eq + std::hash::Hash + Send + Sync, + V: Clone + Send + Sync, + { + type Key = K; + type Value = V; + + async fn load(&self, keys: HashSet) -> Option> { + assert!(keys.len() <= self.capacity); + tokio::time::sleep(self.delay).await; + self.requests.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Some( + keys.into_iter() + .filter_map(|k| { + let value = self.values.get(&k)?.clone(); + Some((k, value)) + }) + .collect(), + ) + } + } + + #[tokio::test] + async fn basic() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestFetcher { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = DataLoader::builder().batch_size(2).concurrency(1).build(fetcher); + + let start = std::time::Instant::now(); + let a = loader.load("a").await.unwrap(); + assert_eq!(a, Some(1)); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 1); + + let start = std::time::Instant::now(); + let b = loader.load("b").await.unwrap(); + assert_eq!(b, Some(2)); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 2); + let start = std::time::Instant::now(); + let c = loader.load("c").await.unwrap(); + assert_eq!(c, Some(3)); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 3); + + let start = std::time::Instant::now(); + let ab = loader.load_many(vec!["a", "b"]).await.unwrap(); + assert_eq!(ab, HashMap::from_iter(vec![("a", 1), ("b", 2)])); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 4); + + let start = std::time::Instant::now(); + let unknown = loader.load("unknown").await.unwrap(); + assert_eq!(unknown, None); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 5); + } + + #[tokio::test] + async fn concurrency_high() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestFetcher { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = DataLoader::builder().batch_size(2).concurrency(10).build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader + .load_many(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + .await + .unwrap(); + assert_eq!(ab, HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)])); + assert!(start.elapsed() < std::time::Duration::from_millis(15)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 5); + } + + #[tokio::test] + async fn delay_low() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestFetcher { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = DataLoader::builder() + .batch_size(2) + .concurrency(1) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader + .load_many(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + .await + .unwrap(); + assert_eq!(ab, HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)])); + assert!(start.elapsed() < std::time::Duration::from_millis(35)); + assert!(start.elapsed() >= std::time::Duration::from_millis(25)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 5); + } + + #[tokio::test] + async fn batch_size() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestFetcher { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 100, + }; + + let loader = DataLoaderBuilder::default() + .batch_size(100) + .concurrency(1) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader + .load_many(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + .await + .unwrap(); + assert_eq!(ab, HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)])); + assert!(start.elapsed() >= std::time::Duration::from_millis(10)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn high_concurrency() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestFetcher { + values: HashMap::from_iter((0..1134).map(|i| (i, i * 2 + 5))), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 100, + }; + + let loader = DataLoaderBuilder::default() + .batch_size(100) + .concurrency(10) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader.load_many(0..1134).await.unwrap(); + assert_eq!(ab, HashMap::from_iter((0..1134).map(|i| (i, i * 2 + 5)))); + assert!(start.elapsed() >= std::time::Duration::from_millis(15)); + assert!(start.elapsed() < std::time::Duration::from_millis(25)); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 1134 / 100 + 1); + } + + #[tokio::test] + async fn delayed_start() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestFetcher { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = DataLoader::builder() + .batch_size(2) + .concurrency(100) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + + let start = std::time::Instant::now(); + let ab = loader + .load_many(vec!["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + .await + .unwrap(); + assert_eq!(ab, HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)])); + assert!(start.elapsed() >= std::time::Duration::from_millis(5)); + assert!(start.elapsed() < std::time::Duration::from_millis(25)); + } + + #[tokio::test] + async fn delayed_start_single() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestFetcher { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 2, + }; + + let loader = DataLoader::builder() + .batch_size(2) + .concurrency(100) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + + let start = std::time::Instant::now(); + let ab = loader.load_many(vec!["a"]).await.unwrap(); + assert_eq!(ab, HashMap::from_iter(vec![("a", 1)])); + assert!(start.elapsed() >= std::time::Duration::from_millis(15)); + assert!(start.elapsed() < std::time::Duration::from_millis(20)); + } + + #[tokio::test] + async fn deduplication() { + let requests = Arc::new(AtomicUsize::new(0)); + + let fetcher = TestFetcher { + values: HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)]), + delay: std::time::Duration::from_millis(5), + requests: requests.clone(), + capacity: 4, + }; + + let loader = DataLoader::builder() + .batch_size(4) + .concurrency(1) + .delay(std::time::Duration::from_millis(10)) + .build(fetcher); + + let start = std::time::Instant::now(); + let ab = loader.load_many(vec!["a", "a", "b", "b", "c", "c"]).await.unwrap(); + assert_eq!(ab, HashMap::from_iter(vec![("a", 1), ("b", 2), ("c", 3)])); + assert_eq!(requests.load(std::sync::atomic::Ordering::Relaxed), 1); + assert!(start.elapsed() >= std::time::Duration::from_millis(5)); + assert!(start.elapsed() < std::time::Duration::from_millis(20)); } } diff --git a/crates/batching/src/lib.rs b/crates/batching/src/lib.rs index d2cfc91ea..c99f5e1aa 100644 --- a/crates/batching/src/lib.rs +++ b/crates/batching/src/lib.rs @@ -1,4 +1,5 @@ #![doc = include_str!("../README.md")] +#![cfg_attr(all(coverage_nightly, test), feature(coverage_attribute))] pub mod batch; pub mod dataloader; diff --git a/crates/bootstrap/Cargo.toml b/crates/bootstrap/Cargo.toml index 6be25dc73..64c64dc9b 100644 --- a/crates/bootstrap/Cargo.toml +++ b/crates/bootstrap/Cargo.toml @@ -19,6 +19,3 @@ pin-project-lite = "0.2" scuffle-context = { version = "0.0.1", path = "../context" } scuffle-bootstrap-derive = { version = "0.0.1", path = "./derive" } scuffle-workspace-hack.workspace = true - -[features] -default = [] diff --git a/crates/bootstrap/examples/Cargo.toml b/crates/bootstrap/examples/Cargo.toml index 783b1c3b3..4e18bff41 100644 --- a/crates/bootstrap/examples/Cargo.toml +++ b/crates/bootstrap/examples/Cargo.toml @@ -20,12 +20,12 @@ name = "scuffle-bootstrap-tracing" path = "src/tracing.rs" [dependencies] -serde_derive = "1.0" -serde = "1.0" +serde_derive = "1" +serde = "1" smart-default = "0.7" tracing = "0.1" tracing-subscriber = "0.3" -anyhow = "1.0" +anyhow = "1" scuffle-bootstrap = { version = "0.0.1", path = ".."} scuffle-settings = { version = "0.0.1", path = "../../settings", features = ["bootstrap"] } diff --git a/crates/bootstrap/telemetry/Cargo.toml b/crates/bootstrap/telemetry/Cargo.toml index f4ba706b4..842204183 100644 --- a/crates/bootstrap/telemetry/Cargo.toml +++ b/crates/bootstrap/telemetry/Cargo.toml @@ -15,11 +15,11 @@ tracing = "0.1" anyhow = "1" prometheus-client = { version = "0.22.3", optional = true } http = "1" -http-body = "1.0.1" -http-body-util = "0.1" -bytes = "1.6.0" +http-body = "1" +http-body-util = "0.1.2" +bytes = "1" querystring = { version = "1", optional = true } -tokio = { version = "1.36.0", optional = true } +tokio = { version = "1", optional = true, features = ["rt"], default-features = false} thiserror = { version = "2", optional = true } opentelemetry = { version = "0.27", optional = true } @@ -37,7 +37,17 @@ scuffle-workspace-hack.workspace = true default = ["prometheus", "pprof", "opentelemetry-metrics", "opentelemetry-traces", "opentelemetry-logs"] prometheus = ["prometheus-client", "opentelemetry"] pprof = ["scuffle-pprof", "querystring", "tokio"] -opentelemetry = ["dep:opentelemetry", "dep:opentelemetry_sdk", "thiserror"] +opentelemetry = ["dep:opentelemetry", "dep:opentelemetry_sdk", "thiserror", "tokio"] opentelemetry-metrics = ["opentelemetry"] opentelemetry-traces = ["opentelemetry", "tracing-opentelemetry"] opentelemetry-logs = ["opentelemetry", "opentelemetry-appender-tracing"] + +[package.metadata.xtask] +addative-features = [ + "default", + "prometheus", + "pprof", + "opentelemetry-metrics", + "opentelemetry-traces", + "opentelemetry-logs", +] diff --git a/crates/bootstrap/telemetry/src/lib.rs b/crates/bootstrap/telemetry/src/lib.rs index 597794b09..478e31539 100644 --- a/crates/bootstrap/telemetry/src/lib.rs +++ b/crates/bootstrap/telemetry/src/lib.rs @@ -1,5 +1,9 @@ use anyhow::Context; use bytes::Bytes; +#[cfg(feature = "opentelemetry-logs")] +pub use opentelemetry_appender_tracing; +#[cfg(feature = "opentelemetry")] +pub use opentelemetry_sdk; #[cfg(feature = "prometheus")] pub use prometheus_client; use scuffle_bootstrap::global::Global; @@ -7,8 +11,8 @@ use scuffle_bootstrap::service::Service; use scuffle_context::ContextFutExt; use scuffle_http::backend::HttpServer; use scuffle_http::body::IncomingBody; -#[cfg(feature = "opentelemetry")] -pub use {opentelemetry_appender_tracing, opentelemetry_sdk, tracing_opentelemetry}; +#[cfg(feature = "opentelemetry-traces")] +pub use tracing_opentelemetry; pub struct TelemetrySvc; @@ -272,14 +276,38 @@ pub mod opentelemetry { } pub fn is_enabled(&self) -> bool { - self.metrics.is_some() || self.traces.is_some() || self.logs.is_some() + #[cfg_attr( + not(any( + feature = "opentelemetry-metrics", + feature = "opentelemetry-traces", + feature = "opentelemetry-logs" + )), + allow(unused_mut) + )] + let mut enabled = false; + #[cfg(feature = "opentelemetry-metrics")] + { + enabled |= self.metrics.is_some(); + } + #[cfg(feature = "opentelemetry-traces")] + { + enabled |= self.traces.is_some(); + } + #[cfg(feature = "opentelemetry-logs")] + { + enabled |= self.logs.is_some(); + } + enabled } #[cfg(feature = "opentelemetry-metrics")] pub fn with_metrics(self, metrics: impl Into>) -> Self { Self { metrics: metrics.into(), - ..self + #[cfg(feature = "opentelemetry-traces")] + traces: self.traces, + #[cfg(feature = "opentelemetry-logs")] + logs: self.logs, } } @@ -287,7 +315,10 @@ pub mod opentelemetry { pub fn with_traces(self, traces: impl Into>) -> Self { Self { traces: traces.into(), - ..self + #[cfg(feature = "opentelemetry-metrics")] + metrics: self.metrics, + #[cfg(feature = "opentelemetry-logs")] + logs: self.logs, } } @@ -295,7 +326,10 @@ pub mod opentelemetry { pub fn with_logs(self, logs: impl Into>) -> Self { Self { logs: logs.into(), - ..self + #[cfg(feature = "opentelemetry-traces")] + traces: self.traces, + #[cfg(feature = "opentelemetry-metrics")] + metrics: self.metrics, } } diff --git a/crates/ffmpeg/Cargo.toml b/crates/ffmpeg/Cargo.toml index 018ab7caf..af7e831e3 100644 --- a/crates/ffmpeg/Cargo.toml +++ b/crates/ffmpeg/Cargo.toml @@ -11,19 +11,28 @@ description = "FFmpeg bindings for Rust." keywords = ["ffmpeg", "video", "audio", "media"] [dependencies] -ffmpeg-sys-next = "7.1.0" +ffmpeg-sys-next = "7.1" libc = "0.2" -bytes = { optional = true, version = "1.8.0" } -tokio = { optional = true, version = "1.41.1" } +bytes = { optional = true, version = "1" } +tokio = { optional = true, version = "1" } crossbeam-channel = { optional = true, version = "0.5.13" } -tracing = { optional = true, version = "0.1.40" } -arc-swap = { version = "1.7.1" } +tracing = { optional = true, version = "0.1" } +arc-swap = { version = "1.7" } scuffle-workspace-hack.workspace = true [features] -default = [] channel = ["dep:bytes"] tokio-channel = ["channel", "dep:tokio"] crossbeam-channel = ["channel", "dep:crossbeam-channel"] tracing = ["dep:tracing"] build = ["ffmpeg-sys-next/build"] + +[package.metadata.xtask] +# Note: build is not an addative feature because it changes the build.rs and therefore +# requires a full rebuild of the crate. +addative-features = [ + "channel", + "tokio-channel", + "crossbeam-channel", + "tracing", +] diff --git a/crates/h3-webtransport/Cargo.toml b/crates/h3-webtransport/Cargo.toml index 2b580e553..9acfc687f 100644 --- a/crates/h3-webtransport/Cargo.toml +++ b/crates/h3-webtransport/Cargo.toml @@ -15,9 +15,9 @@ bytes = "1" futures-util = { version = "0.3", default-features = false } http = "1" pin-project-lite = { version = "0.2", default-features = false } -tokio = { version = "1.28", default-features = false } +tokio = { version = "1", default-features = false } scuffle-workspace-hack.workspace = true [dependencies.h3] -version = "0" +version = "0.0.6" features = ["i-implement-a-third-party-backend-and-opt-into-breaking-changes"] diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index ba9ed7396..9521f27ea 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -53,19 +53,14 @@ hyper-util = { version = "0.1.10", optional = true, features = ["server", "tokio scuffle-context = { version = "0.0.1", path = "../context" } scuffle-workspace-hack.workspace = true -[dev-dependencies] -tokio = { version = "1", features = ["macros"] } -tracing-fmt = { version = "0.1" } -tracing-subscriber = { version = "0.3" } -axum = { version = "0.7" } -tokio-stream = { version = "0.1" } - [features] _tcp = [ "hyper", "hyper-util", ] -_quic = [] +_quic = [ + "h3", +] error-backtrace = [] @@ -137,3 +132,23 @@ default = [ "http2", "tracing", ] + +[package.metadata.xtask] +addative-features = [ + "_tcp", + "_quic", + "error-backtrace", + "http1", + "http2", + "http3", + "http3-webtransport", + "quic-quinn", + "_tls", + "tls-rustls", + "tls-rustls-pem", + "tracing", + "tower", + "axum", + "http3-default", + "default", +] diff --git a/crates/http/examples/Cargo.toml b/crates/http/examples/Cargo.toml index 6fcc36e8e..9b10adb5e 100644 --- a/crates/http/examples/Cargo.toml +++ b/crates/http/examples/Cargo.toml @@ -9,11 +9,11 @@ path = "src/tcp.rs" [dependencies] scuffle-http = { path = "..", features = ["http1", "http2", "tracing"] } -tokio = { version = "1.41.1", features = ["full"] } +tokio = { version = "1", features = ["full"] } tracing = "0.1" tracing-subscriber = "0.3" -http-body = "1.0.1" +http-body = "1" http-body-util = "0.1.2" -http = "1.1.0" -bytes = "1.6.0" +http = "1" +bytes = "1" scuffle-workspace-hack.workspace = true diff --git a/crates/http/src/backend/mod.rs b/crates/http/src/backend/mod.rs index 21f8f55d8..a764cecc0 100644 --- a/crates/http/src/backend/mod.rs +++ b/crates/http/src/backend/mod.rs @@ -1,6 +1,6 @@ use crate::svc::ConnectionAcceptor; -#[cfg(feature = "h3")] +#[cfg(feature = "_quic")] pub mod quic; #[cfg(feature = "_tcp")] pub mod tcp; diff --git a/crates/http/src/backend/quic/body.rs b/crates/http/src/backend/quic/body.rs index 839deda1a..2b5710fb7 100644 --- a/crates/http/src/backend/quic/body.rs +++ b/crates/http/src/backend/quic/body.rs @@ -1,3 +1,5 @@ +#![cfg_attr(not(feature = "quic-quinn"), allow(dead_code))] + use std::future::Future; use std::pin::Pin; use std::task::{Context, Poll}; @@ -25,6 +27,7 @@ pub(crate) struct QuicIncomingBodyInner> { } impl> QuicIncomingBodyInner { + #[cfg(feature = "quic-quinn")] pub(crate) fn new(stream: RequestStream, size_hint: Option) -> Self { Self { stream, @@ -44,6 +47,11 @@ impl http_body::Body for QuicIncomingBody { match self.get_mut() { #[cfg(feature = "quic-quinn")] QuicIncomingBody::Quinn(inner) => Pin::new(inner).poll_frame(cx), + #[cfg(not(feature = "quic-quinn"))] + _ => { + let _ = cx; + unreachable!("impossible to construct QuicIncomingBody with no transport") + } } } @@ -51,6 +59,8 @@ impl http_body::Body for QuicIncomingBody { match self { #[cfg(feature = "quic-quinn")] QuicIncomingBody::Quinn(inner) => Pin::new(inner).size_hint(), + #[cfg(not(feature = "quic-quinn"))] + _ => unreachable!("impossible to construct QuicIncomingBody with no transport"), } } @@ -58,6 +68,8 @@ impl http_body::Body for QuicIncomingBody { match self { #[cfg(feature = "quic-quinn")] QuicIncomingBody::Quinn(inner) => inner.is_end_stream(), + #[cfg(not(feature = "quic-quinn"))] + _ => unreachable!("impossible to construct QuicIncomingBody with no transport"), } } } @@ -115,6 +127,7 @@ impl> http_body::Body for QuicIncomingBodyInner { Poll::Ready(Ok(Some(trailers))) => Poll::Ready(Some(Ok(http_body::Frame::trailers(trailers)))), // We will only poll the recv_trailers once so if pending is returned we are done. Poll::Pending => { + #[cfg(feature = "tracing")] tracing::warn!("recv_trailers is pending"); Poll::Ready(None) } diff --git a/crates/http/src/backend/quic/mod.rs b/crates/http/src/backend/quic/mod.rs index 2b0481014..ade9ff73f 100644 --- a/crates/http/src/backend/quic/mod.rs +++ b/crates/http/src/backend/quic/mod.rs @@ -1,11 +1,8 @@ -#[cfg(not(any(feature = "quic-quinn")))] -compile_error!("http3 feature requires a transport feature to be enabled: quic-quinn"); - mod body; pub(crate) use body::QuicIncomingBody; -#[cfg(feature = "h3-quinn")] +#[cfg(feature = "quic-quinn")] pub mod quinn; #[cfg(feature = "http3-webtransport")] @@ -16,14 +13,14 @@ use crate::svc::ConnectionAcceptor; #[derive(derive_more::From, derive_more::Debug)] pub enum QuicServer { - #[cfg(feature = "h3-quinn")] + #[cfg(feature = "quic-quinn")] #[debug("Quinn")] Quinn(quinn::QuinnServer), } #[derive(Debug, thiserror::Error)] pub enum QuicBackendError { - #[cfg(feature = "h3-quinn")] + #[cfg(feature = "quic-quinn")] #[error("quinn: {0}")] Quinn(#[from] quinn::QuinnServerError), } @@ -37,8 +34,13 @@ impl HttpServer for QuicServer { workers: usize, ) -> Result<(), Self::Error> { match self { - #[cfg(feature = "h3-quinn")] + #[cfg(feature = "quic-quinn")] QuicServer::Quinn(server) => Ok(server.start(service, workers).await?), + #[cfg(not(any(feature = "quic-quinn")))] + _ => { + let _ = (service, workers); + unreachable!("impossible to construct QuicServer with no transports") + } } } @@ -46,6 +48,8 @@ impl HttpServer for QuicServer { match self { #[cfg(feature = "h3-quinn")] QuicServer::Quinn(server) => Ok(server.shutdown().await?), + #[cfg(not(any(feature = "h3-quinn")))] + _ => unreachable!("impossible to construct QuicServer with no transports"), } } @@ -53,6 +57,8 @@ impl HttpServer for QuicServer { match self { #[cfg(feature = "h3-quinn")] QuicServer::Quinn(server) => Ok(server.local_addr()?), + #[cfg(not(any(feature = "h3-quinn")))] + _ => unreachable!("impossible to construct QuicServer with no transports"), } } @@ -60,6 +66,8 @@ impl HttpServer for QuicServer { match self { #[cfg(feature = "h3-quinn")] QuicServer::Quinn(server) => Ok(server.wait().await?), + #[cfg(not(any(feature = "h3-quinn")))] + _ => unreachable!("impossible to construct QuicServer with no transports"), } } } diff --git a/crates/http/src/backend/quic/quinn/serve.rs b/crates/http/src/backend/quic/quinn/serve.rs index b2e6351f6..5baa06fae 100644 --- a/crates/http/src/backend/quic/quinn/serve.rs +++ b/crates/http/src/backend/quic/quinn/serve.rs @@ -61,7 +61,9 @@ async fn serve_handle( config: Arc, ctx: scuffle_context::Context, ) { + #[cfg(feature = "tracing")] tracing::debug!("serving quinn connection: {:?}", conn.remote_address()); + let handle = Arc::new(handle); let (ctx, ctx_handler) = ctx.new_child(); @@ -212,6 +214,7 @@ async fn serve_handle_inner( }; let Some((request, stream)) = conn.with_context("quinn accept")? else { + #[cfg(feature = "tracing")] tracing::debug!("no request, closing connection"); return Ok(()); }; diff --git a/crates/http/src/backend/tcp/mod.rs b/crates/http/src/backend/tcp/mod.rs index 5e4d6e561..46735a1a6 100644 --- a/crates/http/src/backend/tcp/mod.rs +++ b/crates/http/src/backend/tcp/mod.rs @@ -84,7 +84,7 @@ impl HttpServer for TcpServer { #[cfg(all(feature = "http2", feature = "tls-rustls"))] let allow_http2 = config.allow_upgrades || config.only_http.is_none_or(|v| v == config::HttpVersion::Http2); - #[cfg(feature = "tls-rustls")] + #[cfg(all(feature = "tls-rustls", any(feature = "http1", feature = "http2")))] if let Some(acceptor) = &mut config.acceptor { let mut alpn = Vec::new(); #[cfg(feature = "http2")] diff --git a/crates/http/src/backend/tcp/serve.rs b/crates/http/src/backend/tcp/serve.rs index 16fdd558e..fd28b1741 100644 --- a/crates/http/src/backend/tcp/serve.rs +++ b/crates/http/src/backend/tcp/serve.rs @@ -202,6 +202,7 @@ async fn serve_handle( config: TcpServerConfigInner, ctx: &scuffle_context::Context, ) -> Result<(), crate::Error> { + #[cfg(feature = "tracing")] tracing::debug!("serving connection: {:?}", addr); let io = hyper_util::rt::TokioIo::new(stream); diff --git a/crates/http/src/body.rs b/crates/http/src/body.rs index 4fc3a220b..4da0bf6ae 100644 --- a/crates/http/src/body.rs +++ b/crates/http/src/body.rs @@ -12,12 +12,12 @@ pub struct IncomingBody { } impl IncomingBody { - #[cfg_attr(not(any(feature = "_quic", feature = "_tcp")), allow(dead_code))] + #[cfg_attr(not(any(feature = "http3", feature = "http1", feature = "http2")), allow(dead_code))] pub(crate) fn new(inner: impl Into) -> Self { Self { inner: inner.into() } } - #[cfg_attr(not(any(feature = "_quic", feature = "_tcp")), allow(dead_code))] + #[cfg_attr(not(any(feature = "http3", feature = "http1", feature = "http2")), allow(dead_code))] pub(crate) fn empty() -> Self { Self { inner: IncomingBodyInner::Empty, @@ -37,7 +37,7 @@ pub(crate) enum IncomingBodyInner { Tcp(#[from] hyper::body::Incoming), #[cfg(feature = "_quic")] Quic(#[from] QuicIncomingBody), - #[cfg_attr(not(any(feature = "_quic", feature = "_tcp")), allow(dead_code))] + #[cfg_attr(not(any(feature = "http3", feature = "http1", feature = "http2")), allow(dead_code))] Empty, } @@ -46,7 +46,7 @@ impl http_body::Body for IncomingBody { type Error = crate::Error; fn poll_frame(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll, Self::Error>>> { - #[cfg(not(any(feature = "_quic", feature = "_tcp")))] + #[cfg(not(any(feature = "http3", feature = "http1", feature = "http2")))] let _ = cx; match &mut self.inner { @@ -140,7 +140,7 @@ where } } -#[cfg_attr(not(any(feature = "_quic", feature = "_tcp")), allow(dead_code))] +#[cfg_attr(not(any(feature = "http3", feature = "http1", feature = "http2")), allow(dead_code))] pub(crate) fn has_body(method: &http::Method) -> bool { !matches!( method, diff --git a/crates/http/src/error.rs b/crates/http/src/error.rs index 03fa2910f..eeeeab64c 100644 --- a/crates/http/src/error.rs +++ b/crates/http/src/error.rs @@ -1,5 +1,4 @@ use std::convert::Infallible; -use std::error::Error as StdError; #[derive(Debug)] pub struct Error { @@ -212,7 +211,7 @@ pub(crate) fn downcast(error: Box Error::with_kind(ErrorKind::Unknown(error)) } -#[cfg(any(feature = "http1", feature = "http2"))] +#[cfg(any(feature = "_tcp", feature = "_quic"))] pub(crate) fn find_source(mut error: &(dyn std::error::Error + 'static)) -> Option { loop { if let Some(err) = error.downcast_ref::() { @@ -301,10 +300,10 @@ impl std::fmt::Display for Error { pub enum ErrorKind { #[error("http: {0}")] Http(#[from] http::Error), - #[cfg(feature = "http3")] + #[cfg(feature = "_quic")] #[error("h3: {0}")] H3(#[from] h3::Error), - #[cfg(any(feature = "http1", feature = "http2"))] + #[cfg(feature = "_tcp")] #[error("hyper: {0}")] Hyper(#[from] hyper::Error), #[error("closed")] @@ -337,7 +336,7 @@ impl ErrorKindExt for http::Error { } } -#[cfg(feature = "http3")] +#[cfg(feature = "_quic")] impl ErrorKindExt for h3::Error { fn severity(&self) -> ErrorSeverity { match self.kind() { @@ -348,6 +347,8 @@ impl ErrorKindExt for h3::Error { _ => ErrorSeverity::Error, }, _ => { + use std::error::Error as StdError; + if let Some(severity) = self.source().and_then(find_source) { severity } else { @@ -358,9 +359,11 @@ impl ErrorKindExt for h3::Error { } } -#[cfg(any(feature = "http1", feature = "http2"))] +#[cfg(feature = "_tcp")] impl ErrorKindExt for hyper::Error { fn severity(&self) -> ErrorSeverity { + use std::error::Error as StdError; + if self.is_incomplete_message() { ErrorSeverity::Debug } else { @@ -426,9 +429,9 @@ impl ErrorKind { Self::Configuration => ErrorSeverity::Error, Self::Closed => ErrorSeverity::Debug, Self::Unknown(_) => ErrorSeverity::Error, - #[cfg(feature = "http3")] + #[cfg(feature = "_quic")] Self::H3(err) => err.severity(), - #[cfg(any(feature = "http1", feature = "http2"))] + #[cfg(feature = "_tcp")] Self::Hyper(err) => err.severity(), #[cfg(feature = "axum")] Self::Axum(err) => err.severity(), diff --git a/crates/http/src/util.rs b/crates/http/src/util.rs index 195a151f9..ab6a2018a 100644 --- a/crates/http/src/util.rs +++ b/crates/http/src/util.rs @@ -1,12 +1,12 @@ +#![cfg_attr(not(any(feature = "http1", feature = "http2", feature = "http3")), allow(dead_code))] + pub struct AbortOnDrop(Option>); impl AbortOnDrop { - #[cfg_attr(not(any(feature = "_tcp", feature = "_quic")), allow(dead_code))] pub fn new(handle: tokio::task::JoinHandle) -> Self { Self(Some(handle)) } - #[cfg_attr(not(any(feature = "_tcp", feature = "_quic")), allow(dead_code))] pub fn disarm(mut self) -> tokio::task::JoinHandle { self.0.take().expect("disarmed twice") } diff --git a/crates/metrics/Cargo.toml b/crates/metrics/Cargo.toml index 9d40f2f53..c15911b24 100644 --- a/crates/metrics/Cargo.toml +++ b/crates/metrics/Cargo.toml @@ -25,3 +25,12 @@ prometheus = ["dep:prometheus-client"] default = ["prometheus"] tracing = ["internal-logs", "dep:tracing"] extended-numbers = [] + +[package.metadata.xtask] +addative-features = [ + "prometheus", + "tracing", + "extended-numbers", + "internal-logs", + "default", +] diff --git a/crates/settings/Cargo.toml b/crates/settings/Cargo.toml index 60bd6826c..d0333a248 100644 --- a/crates/settings/Cargo.toml +++ b/crates/settings/Cargo.toml @@ -33,9 +33,18 @@ templates = ["minijinja"] all-formats = ["toml", "json", "yaml", "json5", "ini", "ron"] full = ["all-formats", "templates", "cli", "bootstrap"] bootstrap = ["scuffle-bootstrap", "anyhow", "cli"] -default = [] -[dev-dependencies] -serde_derive = "1" -tracing = "0.1" -tracing-subscriber = "0.3.18" +[package.metadata.xtask] +addative-features = [ + "cli", + "ron", + "toml", + "yaml", + "json", + "json5", + "ini", + "templates", + "all-formats", + "full", + "bootstrap", +] diff --git a/crates/settings/examples/Cargo.toml b/crates/settings/examples/Cargo.toml index eef4b4ef3..9242f8548 100644 --- a/crates/settings/examples/Cargo.toml +++ b/crates/settings/examples/Cargo.toml @@ -15,5 +15,5 @@ path = "src/cli.rs" scuffle-settings = { path = "../", features = ["cli"] } serde = "1" serde_derive = "1" -smart-default = "0.7.1" +smart-default = "0.7" scuffle-workspace-hack.workspace = true diff --git a/crates/settings/src/lib.rs b/crates/settings/src/lib.rs index 2eae7b9a9..e23cefea7 100644 --- a/crates/settings/src/lib.rs +++ b/crates/settings/src/lib.rs @@ -94,59 +94,47 @@ impl config::Format for FormatWrapper { #[cfg(feature = "toml")] Some("toml") => config::FileFormat::Toml.parse(uri, template_text(text, &config::FileFormat::Toml)?.as_ref()), #[cfg(not(feature = "toml"))] - Some("toml") => { - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("toml support is not enabled, consider building with the `toml` feature enabled"), - ))) - } + Some("toml") => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "toml support is not enabled, consider building with the `toml` feature enabled", + ))), #[cfg(feature = "json")] Some("json") => config::FileFormat::Json.parse(uri, template_text(text, &config::FileFormat::Json)?.as_ref()), #[cfg(not(feature = "json"))] - Some("json") => { - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("json support is not enabled, consider building with the `json` feature enabled"), - ))) - } + Some("json") => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "json support is not enabled, consider building with the `json` feature enabled", + ))), #[cfg(feature = "yaml")] Some("yaml") | Some("yml") => { config::FileFormat::Yaml.parse(uri, template_text(text, &config::FileFormat::Yaml)?.as_ref()) } #[cfg(not(feature = "yaml"))] - Some("yaml") | Some("yml") => { - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("yaml support is not enabled, consider building with the `yaml` feature enabled"), - ))) - } + Some("yaml") | Some("yml") => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "yaml support is not enabled, consider building with the `yaml` feature enabled", + ))), #[cfg(feature = "json5")] Some("json5") => config::FileFormat::Json5.parse(uri, template_text(text, &config::FileFormat::Json5)?.as_ref()), #[cfg(not(feature = "json5"))] - Some("json5") => { - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("json5 support is not enabled, consider building with the `json5` feature enabled"), - ))) - } + Some("json5") => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "json5 support is not enabled, consider building with the `json5` feature enabled", + ))), #[cfg(feature = "ini")] Some("ini") => config::FileFormat::Ini.parse(uri, template_text(text, &config::FileFormat::Ini)?.as_ref()), #[cfg(not(feature = "ini"))] - Some("ini") => { - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("ini support is not enabled, consider building with the `ini` feature enabled"), - ))) - } + Some("ini") => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "ini support is not enabled, consider building with the `ini` feature enabled", + ))), #[cfg(feature = "ron")] Some("ron") => config::FileFormat::Ron.parse(uri, template_text(text, &config::FileFormat::Ron)?.as_ref()), #[cfg(not(feature = "ron"))] - Some("ron") => { - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("ron support is not enabled, consider building with the `ron` feature enabled"), - ))) - } + Some("ron") => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "ron support is not enabled, consider building with the `ron` feature enabled", + ))), _ => { let formats: &[config::FileFormat] = &[ #[cfg(feature = "toml")] diff --git a/crates/signal/Cargo.toml b/crates/signal/Cargo.toml index 405ae0024..9669f2a9a 100644 --- a/crates/signal/Cargo.toml +++ b/crates/signal/Cargo.toml @@ -11,10 +11,10 @@ description = "Ergonomic async signal handling." keywords = ["signal", "async"] [dependencies] -tokio = { version = "1.41.1", default-features = false, features = ["signal"] } +tokio = { version = "1", default-features = false, features = ["signal"] } scuffle-bootstrap = { version = "0.0.1", path = "../bootstrap", optional = true } scuffle-context = { version = "0.0.1", path = "../context", optional = true } -anyhow = { version = "1.0", optional = true } +anyhow = { version = "1", optional = true } scuffle-workspace-hack.workspace = true [dev-dependencies] @@ -24,4 +24,3 @@ futures = "0.3" [features] bootstrap = ["scuffle-bootstrap", "scuffle-context", "anyhow", "tokio/macros"] -default = [] diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml index 53b509d1e..87b99878f 100644 --- a/crates/workspace-hack/Cargo.toml +++ b/crates/workspace-hack/Cargo.toml @@ -21,6 +21,8 @@ axum-core = { version = "0.4", default-features = false, features = ["tracing"] bitflags = { version = "2", default-features = false, features = ["serde"] } bytes = { version = "1", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] } +clap = { version = "4" } +clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "suggestions", "usage"] } config = { version = "0.14", default-features = false, features = ["ini", "json", "json5", "ron", "toml", "yaml"] } crossbeam-utils = { version = "0.8" } either = { version = "1", default-features = false, features = ["use_std"] } @@ -85,6 +87,8 @@ bitflags = { version = "2", default-features = false, features = ["serde"] } bytes = { version = "1", features = ["serde"] } cc = { version = "1", default-features = false, features = ["parallel"] } chrono = { version = "0.4", features = ["serde"] } +clap = { version = "4" } +clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "suggestions", "usage"] } config = { version = "0.14", default-features = false, features = ["ini", "json", "json5", "ron", "toml", "yaml"] } crossbeam-utils = { version = "0.8" } either = { version = "1", default-features = false, features = ["use_std"] } diff --git a/deny.toml b/deny.toml new file mode 100644 index 000000000..9bf852a36 --- /dev/null +++ b/deny.toml @@ -0,0 +1,33 @@ +[licenses] +allow = [ + "MIT", + "Apache-2.0", + "BSD-2-Clause", + "BSD-3-Clause", + "Unicode-3.0", + "GPL-3.0", + "AGPL-3.0", + "Apache-2.0 WITH LLVM-exception", + "MPL-2.0", + "ISC", + "Zlib", + "WTFPL", + "OpenSSL", + "CC0-1.0", +] + +unused-allowed-license = "warn" +confidence-threshold = 0.95 + +[[licenses.clarify]] +name = "ring" +expression = "MIT AND ISC AND OpenSSL" +license-files = [ + { path = "LICENSE", hash = 0xbd0eed23 } +] + +[advisories] +# TODO: update when mongodb updates their version or we remove mongodb +ignore = [ + "RUSTSEC-2024-0388", +] diff --git a/dev-tools/xtask/Cargo.toml b/dev-tools/xtask/Cargo.toml new file mode 100644 index 000000000..f56d06ed9 --- /dev/null +++ b/dev-tools/xtask/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "xtask" +version = "0.1.0" +edition = "2021" +publish = false +license = "MIT OR Apache-2.0" + +# This package is a special package as its used by developers to run commands in the workspace. +# Therefore we should try to keep the dependencies to a minimum. If you need to add a command that is +# particularly heavy that includes a lot of dependencies consider adding it to a separate specialized +# package. + +[dependencies] +clap = { version = "4.5.23", features = ["derive", "env"] } +cargo_metadata = "0.19.1" +anyhow = "1.0" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" diff --git a/dev-tools/xtask/LICENSE.Apache-2.0 b/dev-tools/xtask/LICENSE.Apache-2.0 new file mode 120000 index 000000000..5a4558f07 --- /dev/null +++ b/dev-tools/xtask/LICENSE.Apache-2.0 @@ -0,0 +1 @@ +../../LICENSE.Apache-2.0 \ No newline at end of file diff --git a/dev-tools/xtask/LICENSE.MIT b/dev-tools/xtask/LICENSE.MIT new file mode 120000 index 000000000..244dbbf0b --- /dev/null +++ b/dev-tools/xtask/LICENSE.MIT @@ -0,0 +1 @@ +../../LICENSE.MIT \ No newline at end of file diff --git a/dev-tools/xtask/src/cmd/mod.rs b/dev-tools/xtask/src/cmd/mod.rs new file mode 100644 index 000000000..8c60521aa --- /dev/null +++ b/dev-tools/xtask/src/cmd/mod.rs @@ -0,0 +1,17 @@ +use anyhow::Context; + +mod power_set; + +#[derive(Debug, Clone, clap::Subcommand)] +pub enum Commands { + #[clap(alias = "powerset")] + PowerSet(power_set::PowerSet), +} + +impl Commands { + pub fn run(self) -> anyhow::Result<()> { + match self { + Commands::PowerSet(cmd) => cmd.run().context("power set"), + } + } +} diff --git a/dev-tools/xtask/src/cmd/power_set.rs b/dev-tools/xtask/src/cmd/power_set.rs new file mode 100644 index 000000000..6669802a3 --- /dev/null +++ b/dev-tools/xtask/src/cmd/power_set.rs @@ -0,0 +1,162 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::Context; + +use crate::utils::{cargo_cmd, comma_delimited, parse_features, test_package_features, XTaskMetadata}; + +#[derive(Debug, Clone, clap::Parser)] +pub struct PowerSet { + #[clap(long, value_delimiter = ',')] + #[clap(alias = "feature")] + /// Features to test + features: Vec, + #[clap(long, value_delimiter = ',')] + #[clap(alias = "exclude-feature")] + /// Features to exclude from testing + exclude_features: Vec, + #[clap(long, short, value_delimiter = ',')] + #[clap(alias = "package")] + /// Packages to test + packages: Vec, + #[clap(long, short, value_delimiter = ',')] + #[clap(alias = "exclude-package")] + /// Packages to exclude from testing + exclude_packages: Vec, + #[clap(long, default_value = "0")] + /// Number of tests to skip + skip: usize, + #[clap(long, default_value = "true")] + /// Fail fast + fail_fast: bool, + #[clap(long, default_value = "target/power-set")] + /// Target directory + target_dir: String, + #[clap(long, action = clap::ArgAction::SetTrue)] + /// Override target directory + no_override_target_dir: bool, + #[clap(name = "command", default_value = "clippy")] + /// Command to run + command: String, + #[clap(last = true)] + /// Additional arguments to pass to the command + args: Vec, +} + +const IGNORED_PACKAGES: &[&str] = &["scuffle-workspace-hack", "xtask"]; + +impl PowerSet { + pub fn run(self) -> anyhow::Result<()> { + let start = std::time::Instant::now(); + + let metadata = crate::utils::metadata()?; + + let mut tests = BTreeMap::new(); + + let features = self.features.into_iter().map(|f| f.to_lowercase()).collect::>(); + + let (added_global_features, added_package_features) = parse_features(features.iter().map(|f| f.as_str())); + let (excluded_global_features, excluded_package_features) = + parse_features(self.exclude_features.iter().map(|f| f.as_str())); + + let ignored_packages = self + .exclude_packages + .into_iter() + .chain(IGNORED_PACKAGES.iter().map(|p| p.to_string())) + .map(|p| p.to_lowercase()) + .collect::>(); + let packages = self.packages.into_iter().map(|p| p.to_lowercase()).collect::>(); + + let xtask_metadata = metadata + .workspace_packages() + .iter() + .map(|p| { + XTaskMetadata::from_package(p).with_context(|| format!("failed to get metadata for package {}", p.name)) + }) + .collect::>>()?; + + // For each package in the workspace, run tests + for (package, xtask_metadata) in metadata.workspace_packages().iter().zip(xtask_metadata.iter()) { + if ignored_packages.contains(&package.name.to_lowercase()) + || !(packages.is_empty() || packages.contains(&package.name.to_lowercase())) + { + continue; + } + + let added_features = added_package_features + .get(package.name.as_str()) + .into_iter() + .flatten() + .chain(added_global_features.iter()) + .copied() + .filter(|s| package.features.contains_key(*s)); + let excluded_features = excluded_package_features + .get(package.name.as_str()) + .into_iter() + .flatten() + .chain(excluded_global_features.iter()) + .copied() + .filter(|s| package.features.contains_key(*s)); + + let features = test_package_features(package, added_features, excluded_features, xtask_metadata) + .with_context(|| package.name.clone())?; + + tests.insert(package.name.as_str(), features); + } + + let mut i = 0; + let total = tests.values().map(|s| s.len()).sum::(); + + let mut failed = Vec::new(); + + for (package, power_set) in tests.iter() { + for features in power_set.iter() { + if i < self.skip { + i += 1; + continue; + } + + let mut cmd = cargo_cmd(); + cmd.arg(&self.command); + cmd.args(&self.args); + cmd.arg("--no-default-features"); + if !features.is_empty() { + cmd.arg("--features").arg(comma_delimited(features.iter())); + } + cmd.arg("--package").arg(package); + + if !self.no_override_target_dir { + cmd.arg("--target-dir").arg(&self.target_dir); + } + + println!("executing {:?} ({}/{})", cmd, i, total); + + if !cmd.status()?.success() { + failed.push((*package, features)); + if self.fail_fast { + anyhow::bail!( + "failed to execute command for package {} with features {:?} after {:?}", + package, + features, + start.elapsed() + ); + } + } + + i += 1; + } + } + + if !failed.is_empty() { + eprintln!("failed to execute command for the following:"); + for (package, features) in failed { + eprintln!(" {} with features {:?}", package, features); + } + + anyhow::bail!("failed to execute command for some packages after {:?}", start.elapsed()); + } + + println!("all commands executed successfully after {:?}", start.elapsed()); + + Ok(()) + } +} diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs new file mode 100644 index 000000000..76af43a0f --- /dev/null +++ b/dev-tools/xtask/src/main.rs @@ -0,0 +1,20 @@ +use clap::Parser; +use cmd::Commands; + +mod cmd; +mod utils; + +#[derive(Debug, clap::Parser)] +#[command( + name = "cargo xtask", + bin_name = "cargo xtask", + about = "A utility for running commands in the workspace" +)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +fn main() -> anyhow::Result<()> { + Cli::parse().command.run() +} diff --git a/dev-tools/xtask/src/utils.rs b/dev-tools/xtask/src/utils.rs new file mode 100644 index 000000000..586c6b82c --- /dev/null +++ b/dev-tools/xtask/src/utils.rs @@ -0,0 +1,313 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::Context; + +pub fn metadata() -> anyhow::Result { + cargo_metadata::MetadataCommand::new().exec().context("cargo metadata") +} + +#[derive(Debug, Clone, serde_derive::Deserialize)] +#[serde(default)] +pub struct XTaskMetadata { + #[serde(alias = "skip-feature-sets")] + pub skip_feature_sets: BTreeSet>, + #[serde(alias = "skip-optional-dependencies")] + pub skip_optional_dependencies: bool, + #[serde(alias = "extra-features")] + pub extra_features: BTreeSet, + #[serde(alias = "deny-list")] + pub deny_list: BTreeSet, + #[serde(alias = "always-include-features")] + pub always_include_features: BTreeSet, + #[serde(alias = "max-combination-size")] + pub max_combination_size: Option, + #[serde(alias = "allow-list")] + pub allow_list: BTreeSet, + #[serde(alias = "addative-features")] + pub addative_features: BTreeSet, +} + +impl Default for XTaskMetadata { + fn default() -> Self { + Self { + skip_feature_sets: Default::default(), + skip_optional_dependencies: true, + extra_features: Default::default(), + deny_list: Default::default(), + always_include_features: Default::default(), + max_combination_size: None, + allow_list: Default::default(), + addative_features: Default::default(), + } + } +} + +impl XTaskMetadata { + pub fn from_package(package: &cargo_metadata::Package) -> anyhow::Result { + let Some(metadata) = package.metadata.get("xtask") else { + return Ok(Self::default()); + }; + + serde_json::from_value(metadata.clone()).context("xtask") + } +} + +fn find_permutations<'a>( + initial_start: BTreeSet<&'a str>, + remaining: usize, + permutations: &mut BTreeSet>, + viable_features: &BTreeMap<&'a str, BTreeSet<&'a str>>, + skip_feature_sets: &BTreeSet>, +) { + let mut stack = vec![(initial_start, remaining)]; + + while let Some((start, rem)) = stack.pop() { + if skip_feature_sets.iter().any(|s| s.is_subset(&start)) || !permutations.insert(start.clone()) || rem == 0 { + continue; + } + + let flattened: BTreeSet<_> = start + .iter() + .flat_map(|f| viable_features[f].iter().chain(std::iter::once(f))) + .collect(); + + for (feature, deps) in viable_features.iter() { + if flattened.contains(feature) { + continue; + } + + let mut new_start = start.clone(); + new_start.insert(feature); + for dep in deps { + new_start.remove(dep); + } + + if permutations.contains(&new_start) || skip_feature_sets.contains(&new_start) { + continue; + } + + stack.push((new_start, rem.saturating_sub(1))); + } + } +} + +fn flatten_features<'a>(deps: &[&'a str], package_features: &BTreeMap<&'a str, Vec<&'a str>>) -> BTreeSet<&'a str> { + let mut next: Vec<_> = deps.iter().collect(); + + let mut features = BTreeSet::new(); + while let Some(dep) = next.pop() { + if let Some(deps) = package_features.get(dep) { + if features.insert(*dep) { + next.extend(deps); + } + } + } + + features +} + +pub fn test_package_features<'a>( + package: &'a cargo_metadata::Package, + added_features: impl IntoIterator, + excluded_features: impl IntoIterator, + xtask_metadata: &'a XTaskMetadata, +) -> anyhow::Result>> { + if package.features.is_empty() { + return Ok(BTreeSet::new()); + } + + let mut always_included_features: BTreeSet<_> = xtask_metadata + .always_include_features + .iter() + .map(|f| f.as_str()) + .chain(added_features) + .collect(); + let skip_feature_sets: BTreeSet<_> = excluded_features + .into_iter() + .map(|f| BTreeSet::from_iter(std::iter::once(f))) + .chain( + xtask_metadata + .skip_feature_sets + .iter() + .map(|s| s.iter().map(|f| f.as_str()).collect()), + ) + .collect(); + + let mut package_features: BTreeMap<&str, Vec<&str>> = package + .features + .iter() + .map(|(k, v)| (k.as_str(), v.iter().map(|f| f.as_str()).collect())) + .collect(); + + if xtask_metadata.skip_optional_dependencies { + let mut implicit_features = BTreeSet::new(); + let mut used_deps = BTreeSet::new(); + + for (feature, deps) in package.features.iter() { + for dep in deps.iter().filter_map(|f| f.strip_prefix("dep:")) { + if dep == feature && deps.len() == 1 { + implicit_features.insert(feature.as_str()); + } else { + used_deps.insert(dep); + } + } + } + + for feature in implicit_features { + if used_deps.contains(&feature) || xtask_metadata.extra_features.contains(feature) { + continue; + } + + package_features.remove(feature); + } + } + + let use_allow_list = !xtask_metadata.allow_list.is_empty(); + let use_deny_list = !xtask_metadata.deny_list.is_empty(); + + if use_allow_list && use_deny_list { + anyhow::bail!("Cannot specify both allow and deny lists, please specify only one."); + } + + let mut viable_features = BTreeMap::new(); + + let mut addative_features = BTreeMap::new(); + + for (feature, deps) in package_features.iter() { + // If we are using an allow list, only include features that are in the allow + // list If we are using a deny list, skip features that are in the deny list + if (use_allow_list && !xtask_metadata.allow_list.contains(*feature)) + || (use_deny_list && xtask_metadata.deny_list.contains(*feature)) + { + continue; + } + + let flattened = flatten_features(deps, &package_features); + + if !xtask_metadata.addative_features.contains(*feature) { + viable_features.insert(*feature, flattened); + } else { + addative_features.insert(*feature, flattened); + } + } + + // Remove features that are not in the package + always_included_features.retain(|f| package_features.contains_key(f)); + + // Non addative permutations are permutations that we need to find every + // combination of + let mut non_addative_permutations = BTreeSet::new(); + + // This finds all the combinations of features that are not addative + find_permutations( + always_included_features.clone(), + xtask_metadata.max_combination_size.unwrap_or(viable_features.len() + 1), + &mut non_addative_permutations, + &viable_features, + &skip_feature_sets, + ); + + // This finds all the combinations of features that are addative + // With addative features we do not need to find every combination, we just need + // to add the addative features to the non addative permutations + + // This loop adds the addative features to the non addative permutations + // Example: + // - NON_ADDATIVE = [(A), (B), (A, B), ()] + // - ADDATIVE = [(C), (D), (E)] + // Result: [ + // (), + // (A), + // (B), + // (A, B), + // (A, C), + // (A, C, D), + // (A, C, D, E), + // (B, C), + // (B, C, D), + // (B, C, D, E), + // (A, B, C), + // (A, B, C, D), + // (A, B, C, D, E), + // (C), + // (D), + // (E), + // (C, D), + // (C, D, E), + // ] + // To note: we do not test for combinations of the addative features. Such as + // (A, D, E). + + let mut permutations = BTreeSet::new(); + for mut permutation in non_addative_permutations { + let flattened: BTreeSet<_> = permutation + .clone() + .into_iter() + .flat_map(|f| viable_features[&f].iter().copied().chain(std::iter::once(f))) + .collect(); + + permutations.insert(permutation.clone()); + for (feature, deps) in addative_features.iter() { + if flattened.contains(feature) { + continue; + } + + permutation.insert(feature); + for dep in deps { + permutation.remove(dep); + } + permutations.insert(permutation.clone()); + } + } + + let flattened: BTreeSet<_> = always_included_features + .iter() + .flat_map(|f| viable_features[f].iter().chain(std::iter::once(f))) + .collect(); + + for feature in addative_features.keys() { + if flattened.contains(feature) { + continue; + } + + let mut permutation = always_included_features.clone(); + permutation.insert(feature); + permutations.insert(permutation); + } + + Ok(permutations) +} + +pub fn parse_features<'a>( + features: impl IntoIterator, +) -> (BTreeSet<&'a str>, BTreeMap<&'a str, BTreeSet<&'a str>>) { + let mut generic_features = BTreeSet::new(); + let mut crate_features = BTreeMap::new(); + + for feature in features { + let mut splits = feature.splitn(2, '/'); + let first = splits.next().unwrap(); + if let Some(second) = splits.next() { + crate_features.entry(first).or_insert_with(BTreeSet::new).insert(second); + } else { + generic_features.insert(first); + } + } + + (generic_features, crate_features) +} + +pub fn cargo_cmd() -> std::process::Command { + std::process::Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string())) +} + +pub fn comma_delimited(features: impl IntoIterator>) -> String { + let mut string = String::new(); + for feature in features { + if !string.is_empty() { + string.push(','); + } + string.push_str(feature.as_ref()); + } + string +} diff --git a/just/ci.just b/just/ci.just deleted file mode 100644 index 1bad8733a..000000000 --- a/just/ci.just +++ /dev/null @@ -1,15 +0,0 @@ -export CI := "1" -export RUSTUP_TOOLCHAIN := "nightly" - -test: - cargo llvm-cov nextest --branch --lcov --profile ci --output-path ./lcov.info --features scuffle-ffmpeg/build - -fmt: - cargo fmt --check --all -- --color=always - -clippy: - cargo clippy --all-targets --all-features -- -D warnings - -hakari: - cargo hakari generate --diff - cargo hakari manage-deps --dry-run