diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml deleted file mode 100644 index ce3709ddd..000000000 --- a/.github/workflows/regression.yml +++ /dev/null @@ -1,177 +0,0 @@ -# Regression Detection -# -# This workflow runs our regression detection experiments, which are relative -# evaluations of the base SHA for the PR to whatever SHA was just pushed into -# the project (unless that SHA happens to be master branch HEAD). The goal is to -# give quick-ish feedback on all-up lading for a variety of configs as to -# whether throughput performance has gone down, gotten more variable in the -# pushed SHA. -# -# Regression detection is always done relative to the pushed SHA, meaning any -# changes you introduce to the experiment will be picked up both for the base -# SHA variant and your current SHA. Tags are SHA-SHA. The first SHA is the one -# that triggered this workflow, the second is the one of the lading being -# tested. For comparison images the two SHAs are identical. - -name: Regression Detector (untrusted) - -on: - pull_request: - -jobs: - cancel-previous: - runs-on: ubuntu-22.04 - timeout-minutes: 3 - steps: - - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ secrets.GITHUB_TOKEN }} - all_but_latest: true # can cancel workflows scheduled later - - compute-metadata: - name: Compute metadata for regression experiments - runs-on: ubuntu-22.04 - outputs: - pr-number: ${{ steps.pr-metadata.outputs.PR_NUMBER }} - - comparison: ${{ steps.comparison.outputs.COMPARISON }} - comparison-tag: ${{ steps.comparison.outputs.COMPARISON_TAG }} - baseline: ${{ steps.baseline.outputs.BASELINE }} - baseline-tag: ${{ steps.baseline.outputs.BASELINE_TAG }} - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.base_ref }} - path: baseline-lading - - - name: Setup PR metadata - id: pr-metadata - run: | - echo "PR_NUMBER=${{ github.event.number }}" >> $GITHUB_OUTPUT - - - name: Setup baseline variables - id: baseline - run: | - pushd baseline-lading - export BASELINE_SHA=$(git rev-parse HEAD) - popd - - export BASELINE_TAG="${{ github.event.pull_request.head.sha }}-${BASELINE_SHA}" - echo "baseline sha is: ${BASELINE_SHA}" - echo "baseline tag is: ${BASELINE_TAG}" - - echo "BASELINE=${BASELINE_SHA}" >> $GITHUB_OUTPUT - echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT - - - name: Setup comparison variables - id: comparison - run: | - export COMPARISON_SHA=${{ github.event.pull_request.head.sha }} - export COMPARISON_TAG="${{ github.event.pull_request.head.sha }}-${{ github.event.pull_request.head.sha }}" - - echo "comparison sha is: ${COMPARISON_SHA}" - echo "comparison tag is: ${COMPARISON_TAG}" - - echo "COMPARISON=${COMPARISON_SHA}" >> $GITHUB_OUTPUT - echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT - - ## - ## BUILD - ## - - build-baseline: - name: Build baseline Lading container - runs-on: ubuntu-22.04 - needs: - - compute-metadata - steps: - - uses: colpal/actions-clean@v1 - - - uses: actions/checkout@v3 - - - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-soak-meta.outputs.baseline-sha }} - path: baseline-lading - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2.2.1 - - - name: Build 'lading' target image - uses: docker/build-push-action@v3 - with: - context: baseline-lading/ - cache-from: type=gha - cache-to: type=gha,mode=max - file: regression/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar - tags: | - lading:${{ needs.compute-metadata.outputs.pr-number }}-${{ needs.compute-metadata.outputs.baseline-tag }} - - - name: Upload image as artifact - uses: actions/upload-artifact@v3 - with: - name: baseline-image - path: "${{ runner.temp }}/baseline-image.tar" - - build-comparison: - name: Build comparison lading container - runs-on: ubuntu-22.04 - needs: - - compute-metadata - steps: - - uses: colpal/actions-clean@v1 - - - uses: actions/checkout@v3 - - - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-soak-meta.outputs.comparison-sha }} - path: comparison-lading - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2.2.1 - - - name: Build 'lading' target image - uses: docker/build-push-action@v3 - with: - context: comparison-lading/ - cache-from: type=gha - cache-to: type=gha,mode=max - file: regression/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar - tags: | - lading:${{ needs.compute-metadata.outputs.pr-number }}-${{ needs.compute-metadata.outputs.comparison-tag }} - - - name: Upload image as artifact - uses: actions/upload-artifact@v3 - with: - name: comparison-image - path: "${{ runner.temp }}/comparison-image.tar" - - transmit-metadata: - name: Transmit metadata to trusted workflow - runs-on: ubuntu-22.04 - needs: - - compute-metadata - steps: - - name: Write out metadata - run: | - echo "COMPARISON_TAG=${{ needs.compute-metadata.outputs.pr-number }}-${{ needs.compute-metadata.outputs.comparison-tag }}" > ${{ runner.temp }}/meta - echo "COMPARISON_SHA=${{ needs.compute-metadata.outputs.comparison }}" >> ${{ runner.temp }}/meta - echo "BASELINE_TAG=${{ needs.compute-metadata.outputs.pr-number }}-${{ needs.compute-metadata.outputs.baseline-tag }}" >> ${{ runner.temp }}/meta - echo "BASELINE_SHA=${{ needs.compute-metadata.outputs.baseline }}" >> ${{ runner.temp }}/meta - echo "CHECKOUT_SHA=${{ github.sha }}" >> ${{ runner.temp }}/meta - echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> ${{ runner.temp }}/meta - echo "BASE_SHA=${{ github.event.pull_request.base.sha }}" >> ${{ runner.temp }}/meta - echo "GITHUB_EVENT_NUMBER=${{ github.event.number }}" >> ${{ runner.temp }}/meta - - - name: Upload metadata - uses: actions/upload-artifact@v3 - with: - name: meta - path: "${{ runner.temp }}/meta" diff --git a/.github/workflows/regression_trusted.yml b/.github/workflows/regression_trusted.yml deleted file mode 100644 index 2997d72c4..000000000 --- a/.github/workflows/regression_trusted.yml +++ /dev/null @@ -1,494 +0,0 @@ -name: Regression Detector (trusted) - -on: - workflow_run: - workflows: ["Regression Detector (untrusted)"] - types: - - completed - -jobs: - compute-metadata: - name: Compute metadata for regression experiments - runs-on: ubuntu-22.04 - if: > - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' - outputs: - cpus: ${{ steps.system.outputs.CPUS }} - memory: ${{ steps.system.outputs.MEMORY }} - - comparison-sha: ${{ steps.metadata.outputs.COMPARISON_SHA }} - comparison-tag: ${{ steps.metadata.outputs.COMPARISON_TAG }} - baseline-sha: ${{ steps.metadata.outputs.BASELINE_SHA }} - baseline-tag: ${{ steps.metadata.outputs.BASELINE_TAG }} - head-sha: ${{ steps.metadata.outputs.HEAD_SHA }} - checkout-sha: ${{ steps.metadata.outputs.CHECKOUT_SHA }} - github-event-number: ${{ steps.metadata.outputs.GITHUB_EVENT_NUMBER }} - - replicas: ${{ steps.experimental-meta.outputs.REPLICAS }} - warmup-seconds: ${{ steps.experimental-meta.outputs.WARMUP_SECONDS }} - total-samples: ${{ steps.experimental-meta.outputs.TOTAL_SAMPLES }} - p-value: ${{ steps.experimental-meta.outputs.P_VALUE }} - smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }} - lading-version: ${{ steps.experimental-meta.outputs.LADING_VERSION }} - smp-wait-timeout-minutes: ${{ steps.experimental-meta.outputs.SMP_WAIT_TIMEOUT }} - - steps: - - name: Setup experimental metadata - id: experimental-meta - run: | - export WARMUP_SECONDS="45" - export REPLICAS="10" - export TOTAL_SAMPLES="600" - export P_VALUE="0.1" - export SMP_CRATE_VERSION="0.8.0" - export SMP_WAIT_TIMEOUT="55" - export LADING_VERSION="0.16.0" - - echo "warmup seconds: ${WARMUP_SECONDS}" - echo "replicas: ${REPLICAS}" - echo "total samples: ${TOTAL_SAMPLES}" - echo "regression p-value: ${P_VALUE}" - echo "smp crate version: ${SMP_CRATE_VERSION}" - echo "lading version: ${LADING_VERSION}" - echo "smp wait timeout: ${SMP_WAIT_TIMEOUT}" - - echo "WARMUP_SECONDS=${WARMUP_SECONDS}" >> $GITHUB_OUTPUT - echo "REPLICAS=${REPLICAS}" >> $GITHUB_OUTPUT - echo "TOTAL_SAMPLES=${TOTAL_SAMPLES}" >> $GITHUB_OUTPUT - echo "P_VALUE=${P_VALUE}" >> $GITHUB_OUTPUT - echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT - echo "LADING_VERSION=${LADING_VERSION}" >> $GITHUB_OUTPUT - echo "SMP_WAIT_TIMEOUT=${SMP_WAIT_TIMEOUT}" >> $GITHUB_OUTPUT - - - name: Setup system details - id: system - run: | - export CPUS="8" - export MEMORY="16g" - - echo "cpus total: ${CPUS}" - echo "memory total: ${MEMORY}" - - echo "CPUS=${CPUS}" >> $GITHUB_OUTPUT - echo "MEMORY=${MEMORY}" >> $GITHUB_OUTPUT - - # github.rest.actions.listWorkflowRunArtifacts only returns first 30 - # artifacts, and returns a { data, headers, status, url } object. The - # "data" part of this object contains the artifact data we care about. - # The fields of this data object correspond to the fields in the - # "Example Response" JSON object in - # https://docs.github.com/en/rest/actions/artifacts#list-workflow-run-artifacts. - # To return more than 30 responses, use the github.paginate API in - # https://octokit.github.io/rest.js/v19#custom-requests - # `github-script` aliases `octokit` to the `github` namespace. - - name: 'Download metadata' - uses: actions/github-script@v6.3.3 - with: - script: | - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "meta" - })[0]; - - console.log("Downloading artifact %s", matchArtifact.id); - - var download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/meta.zip', Buffer.from(download.data)); - - - run: unzip meta.zip - - - name: Setup metadata - id: metadata - run: | - cat meta - cat meta >> $GITHUB_OUTPUT - - confirm-valid-credentials: - name: Confirm AWS credentials are minimally valid - runs-on: ubuntu-22.04 - needs: - - compute-metadata - steps: - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Download SMP binary - run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - - ## - ## SUBMIT - ## - - upload-baseline-image-to-ecr: - name: Upload images to ECR - runs-on: ubuntu-22.04 - needs: - - compute-metadata - - confirm-valid-credentials - steps: - - name: 'Download baseline image' - uses: actions/github-script@v6.3.3 - with: - script: | - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "baseline-image" - })[0]; - - console.log("Downloading artifact %s", matchArtifact.id); - - var download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/baseline-image.zip', Buffer.from(download.data)); - - - run: unzip baseline-image.zip - - - name: Load baseline image - run: | - docker load --input baseline-image.tar - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Docker Login to ECR - uses: docker/login-action@v2 - with: - registry: ${{ steps.login-ecr.outputs.registry }} - - - name: Tag & push baseline image - run: | - docker tag lading:${{ needs.compute-metadata.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }} - docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }} - - upload-comparison-image-to-ecr: - name: Upload images to ECR - runs-on: ubuntu-22.04 - needs: - - compute-metadata - - confirm-valid-credentials - steps: - - name: 'Download comparison image' - uses: actions/github-script@v6.3.3 - with: - script: | - var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: ${{github.event.workflow_run.id }}, - }); - - var matchArtifact = artifacts.data.artifacts.filter((artifact) => { - return artifact.name == "comparison-image" - })[0]; - - console.log("Downloading artifact %s", matchArtifact.id); - - var download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - }); - var fs = require('fs'); - fs.writeFileSync('${{github.workspace}}/comparison-image.zip', Buffer.from(download.data)); - - - run: unzip comparison-image.zip - - - name: Load comparison image - run: | - docker load --input comparison-image.tar - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Docker Login to ECR - uses: docker/login-action@v2 - with: - registry: ${{ steps.login-ecr.outputs.registry }} - - - name: Tag & push comparison image - run: | - docker tag lading:${{ needs.compute-metadata.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }} - docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }} - - submit-job: - name: Submit regression job - runs-on: ubuntu-22.04 - needs: - - compute-metadata - - upload-baseline-image-to-ecr - - upload-comparison-image-to-ecr - steps: - - name: Check status, in-progress - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='pending' \ - -f description='Experiments submitted to the Regression Detector cluster.' \ - -f context='Regression Detector / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-metadata.outputs.checkout-sha }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - - name: Download SMP binary - run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - - - name: Submit job - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \ - --lading-version ${{ needs.compute-metadata.outputs.lading-version }} \ - --total-samples ${{ needs.compute-metadata.outputs.total-samples }} \ - --warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \ - --replicas ${{ needs.compute-metadata.outputs.replicas }} \ - --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.baseline-tag }} \ - --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-lading:${{ needs.compute-metadata.outputs.comparison-tag }} \ - --baseline-sha ${{ needs.compute-metadata.outputs.baseline-sha }} \ - --comparison-sha ${{ needs.compute-metadata.outputs.comparison-sha }} \ - --target-config-dir ${{ github.workspace }}/regression/ \ - --target-name lading-target \ - --target-command "/usr/local/bin/lading --no-target --config-path /etc/lading-target/lading.yaml --capture-path /dev/null --experiment-duration-seconds 1200" \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - uses: actions/upload-artifact@v3 - with: - name: lading-submission-metadata - path: ${{ runner.temp }}/submission-metadata - - - name: Await job - timeout-minutes: 60 - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \ - --wait \ - --wait-delay-seconds 60 \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - name: Handle cancellation if necessary - if: ${{ cancelled() }} - timeout-minutes: 60 - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \ - --submission-metadata ${{ runner.temp }}/submission-metadata - - - name: Check status, cancelled - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='failure' \ - -f description='Experiments submitted to the Regression Detector cluster cancelled.' \ - -f context='Regression Detector / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, success - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Experiments submitted to the Regression Detector cluster successfully.' \ - -f context='Regression Detector / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, failure - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Experiments submitted to the Regression Detector cluster failed.' \ - -f context='Regression Detector / submission' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - ## - ## ANALYZE - ## - - download-analysis: - name: Download regression analysis & upload report - runs-on: ubuntu-22.04 - needs: - - submit-job - - compute-metadata - steps: - - name: Check status, in-progress - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='pending' \ - -f description='Analyze experimental results from Regression Detector.' \ - -f context='Regression Detector / analyze-experiment' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - uses: actions/checkout@v3 - with: - ref: ${{ needs.compute-metadata.outputs.checkout-sha }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1-node16 - with: - aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - - - name: Download SMP binary - run: | - aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp - - - name: Download submission metadata - uses: actions/download-artifact@v3 - with: - name: lading-submission-metadata - path: ${{ runner.temp }}/ - - - name: Sync regression report to local system - env: - RUST_LOG: info - run: | - chmod +x ${{ runner.temp }}/bin/smp - - ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \ - --submission-metadata ${{ runner.temp }}/submission-metadata \ - --output-path "${{ runner.temp }}/outputs" - - - name: Read regression report - id: read-analysis - uses: juliangruber/read-file-action@v1 - with: - path: ${{ runner.temp }}/outputs/report.html - - - name: Post report to PR - uses: peter-evans/create-or-update-comment@v2 - with: - issue-number: ${{ needs.compute-metadata.outputs.github-event-number }} - edit-mode: append - body: ${{ steps.read-analysis.outputs.content }} - - - name: Upload regression report to artifacts - uses: actions/upload-artifact@v3 - with: - name: capture-artifacts - path: ${{ runner.temp }}/outputs/* - - - name: Check status, cancelled - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='failure' \ - -f description='Analyze experimental results from Regression Detector cancelled.' \ - -f context='Regression Detector / analyze-experiment' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, success - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Analyze experimental results from Regression Detector succeeded.' \ - -f context='Regression Detector / analyze-experiment' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - - - name: Check status, failure - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - /repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.head-sha }} \ - -f state='success' \ - -f description='Analyze experimental results from Regression Detector failed.' \ - -f context='Regression Detector / analyze-experiment' \ - -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/regression/Dockerfile b/regression/Dockerfile deleted file mode 100644 index 298b2c37a..000000000 --- a/regression/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# -# LADING BUILDER -# -FROM docker.io/rust:1.71.0-bullseye@sha256:bef59af02f103760cd57e8d6ccadf364954b0ae5e74ea7c7203d26744aeec051 as builder -WORKDIR /work -COPY . . -RUN apt-get update && apt-get install -y protobuf-compiler && rm -rf /var/lib/apt/lists/* -RUN cargo build --bin lading --release --locked - -# -# RUNTIME -# - -FROM docker.io/debian:bullseye-slim@sha256:f7d141c1ec6af549958a7a2543365a7829c2cdc4476308ec2e182f8a7c59b519 -COPY --from=builder /work/target/release/lading /usr/local/bin/lading - -# Smoke test -RUN ["/usr/local/bin/lading", "--help"] - -ENTRYPOINT ["/usr/local/bin/lading"] diff --git a/regression/cases/apache_common_http_both_directions_this_doesnt_make_sense/lading-target/lading.yaml b/regression/cases/apache_common_http_both_directions_this_doesnt_make_sense/lading-target/lading.yaml deleted file mode 100644 index ebd334ed4..000000000 --- a/regression/cases/apache_common_http_both_directions_this_doesnt_make_sense/lading-target/lading.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Configure the lading under test - -generator: - - http: - seed: [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131] - headers: {} - target_uri: "http://127.0.0.1:8383/" - bytes_per_second: "1 Gb" - parallel_connections: 1000 - method: - post: - maximum_prebuild_cache_size_bytes: "256 Mb" - variant: "apache_common" - -blackhole: - - http: - concurrent_requests_max: 1000 - binding_addr: "127.0.0.1:8282" - body_variant: nothing diff --git a/regression/cases/apache_common_http_both_directions_this_doesnt_make_sense/lading/lading.yaml b/regression/cases/apache_common_http_both_directions_this_doesnt_make_sense/lading/lading.yaml deleted file mode 100644 index dbfdd64f4..000000000 --- a/regression/cases/apache_common_http_both_directions_this_doesnt_make_sense/lading/lading.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# This is not the lading under test! This is the stable, released version of -# lading that is used to run the test. - -generator: - - http: - # Seed is purposely different than the seed of the lading under test - seed: [131, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127] - headers: {} - target_uri: "http://127.0.0.1:8282/" - bytes_per_second: "1 Gb" - parallel_connections: 1000 - method: - post: - maximum_prebuild_cache_size_bytes: "256 Mb" - variant: "apache_common" - -blackhole: - - http: - concurrent_requests_max: 1000 - binding_addr: "127.0.0.1:8383" - body_variant: nothing diff --git a/regression/cases/blackhole_from_apache_common_http/lading-target/lading.yaml b/regression/cases/blackhole_from_apache_common_http/lading-target/lading.yaml deleted file mode 100644 index 0263ba087..000000000 --- a/regression/cases/blackhole_from_apache_common_http/lading-target/lading.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Configure the lading under test - -blackhole: - - http: - concurrent_requests_max: 1000 - binding_addr: "127.0.0.1:8282" - body_variant: nothing diff --git a/regression/cases/blackhole_from_apache_common_http/lading/lading.yaml b/regression/cases/blackhole_from_apache_common_http/lading/lading.yaml deleted file mode 100644 index bf62e3e33..000000000 --- a/regression/cases/blackhole_from_apache_common_http/lading/lading.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# This is not the lading under test! This is the stable, released version of -# lading that is used to run the test. - -generator: - - http: - # Seed is purposely different than the seed of the lading under test - seed: [131, 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, - 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127] - headers: {} - target_uri: "http://localhost:8282/" - bytes_per_second: "1 Gb" - parallel_connections: 1000 - method: - post: - maximum_prebuild_cache_size_bytes: "256 Mb" - variant: "apache_common" diff --git a/regression/metadata.source b/regression/metadata.source deleted file mode 100644 index 4b52f2e64..000000000 --- a/regression/metadata.source +++ /dev/null @@ -1,2 +0,0 @@ -REPOSITORY="git@github.com:datadog/lading.git" -TARGET_ENVARS="" \ No newline at end of file