From 69dcbe5e2e78823b69796256fb6422fc3e1bb788 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 16 Jan 2024 13:59:38 +0100 Subject: [PATCH] WIP: Change: Ci: following guidelines --- .github/install-openvas-dependencies.sh | 1 + .github/workflows/build.yml | 25 ++++ .github/workflows/build_and_test.yml | 61 ---------- .github/workflows/ci.yml | 40 +++++++ .../{codeql-analysis-c.yml => codeql.yml} | 0 .github/workflows/control.yml | 18 +++ .github/workflows/ddependabot.yml | 0 .github/workflows/nasl.yml | 2 +- .github/workflows/push-container.yml | 0 .github/workflows/push-helm-chart.yml | 17 +++ .github/workflows/rustification.yaml | 63 ---------- .github/workflows/smoketest.yaml | 110 ++++++++++++++++++ 12 files changed, 212 insertions(+), 125 deletions(-) create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/build_and_test.yml create mode 100644 .github/workflows/ci.yml rename .github/workflows/{codeql-analysis-c.yml => codeql.yml} (100%) create mode 100644 .github/workflows/control.yml create mode 100644 .github/workflows/ddependabot.yml create mode 100644 .github/workflows/push-container.yml create mode 100644 .github/workflows/push-helm-chart.yml create mode 100644 .github/workflows/smoketest.yaml diff --git a/.github/install-openvas-dependencies.sh b/.github/install-openvas-dependencies.sh index afbbbf2232..e4f1bc6fbe 100755 --- a/.github/install-openvas-dependencies.sh +++ b/.github/install-openvas-dependencies.sh @@ -10,6 +10,7 @@ apt-get update && apt-get install --no-install-recommends --no-install-suggests clang-tools \ cmake \ curl \ + git \ lcov \ libgnutls28-dev \ libgpgme-dev \ diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..5a1d056391 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,25 @@ +name: "Build" + +on: [workflow_call] + +jobs: + OpenVAS: + runs-on: ubuntu-latest + container: greenbone/gvm-libs:stable + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + sh .github/install-openvas-dependencies.sh + - name: build + run: | + cmake -Bbuild -DCMAKE_C_COMPILER=/usr/share/clang/scan-build-14/libexec/ccc-analyzer + scan-build -o ~/scan-build-report cmake --build build + - name: Upload scan-build report + uses: actions/upload-artifact@v3 + with: + name: scan-build-report + path: ~/scan-build-report/ + retention-days: 7 + OpenVAS_Daemon: + uses: ./.github/workflows/build-rust.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml deleted file mode 100644 index d605f9edde..0000000000 --- a/.github/workflows/build_and_test.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: "openvas-c" - -on: - push: - branches: [ main, stable ] - pull_request: - branches: [ main, stable ] - -jobs: - formatting: - runs-on: ubuntu-latest - steps: - - name: Check out openvas-scanner - uses: actions/checkout@v4 - - name: Check Source Format - run: | - clang-format -i -style=file {src,misc,nasl}/*.{c,h} - git diff --exit-code - compile: - runs-on: ubuntu-latest - strategy: - matrix: - # With the upcoming changes, we require both downwards and upwards compatibility between the OpenVAS C - # code and GVM-libs. This is because, even though we will be using semantic versioning, as long as - # OpenVAS and GVM-libs remain separate repositories, we want to be notified of every change. - gvm-libs-version: - - stable - - unstable - container: greenbone/gvm-libs:${{ matrix.gvm-libs-version }} - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - sh .github/install-openvas-dependencies.sh - - name: Configure and Scan Build - run: | - cmake -Bbuild -DCMAKE_C_COMPILER=/usr/share/clang/scan-build-14/libexec/ccc-analyzer - scan-build -o ~/scan-build-report cmake --build build - - name: Upload scan-build report - uses: actions/upload-artifact@v3 - with: - name: scan-build-report - path: ~/scan-build-report/ - retention-days: 7 - unit-tests: - runs-on: ubuntu-latest - strategy: - matrix: - gvm-libs-version: - - stable - - unstable - container: greenbone/gvm-libs:${{ matrix.gvm-libs-version }} - steps: - - uses: actions/checkout@v4 - - name: install dependencies - run: | - sh .github/install-openvas-dependencies.sh - - name: unit-tests - run: | - cmake -Bbuild -DCMAKE_BUILD_TYPE=Release - CTEST_OUTPUT_ON_FAILURE=1 cmake --build build -- tests test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..6ea816ab78 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: "Linting" + +on: [workflow_call] + +jobs: + OpenVAS: + runs-on: ubuntu-latest + container: greenbone/gvm-libs:stable + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + sh .github/install-openvas-dependencies.sh + - name: Formatting + run: | + clang-format --dry-run --Werror -i -style=file {src,misc,nasl}/*.{c,h} + - name: unit-tests + run: | + cmake -Bbuild -DCMAKE_BUILD_TYPE=Release + CTEST_OUTPUT_ON_FAILURE=1 cmake --build build -- tests test + OpenVAS_Daemon: + runs-on: ubuntu-latest + defaults: + run: + working-directory: rust + steps: + - uses: actions/checkout@v4 + - run: sudo apt update && sudo apt-get install -y libpcap-dev + - run: rustup update stable && rustup default stable || rustup default stable + - run: cargo install cargo-audit + - run: cargo install typos-cli + - name: unit-tests + run: cargo test --lib --tests --workspace + - name: Clippy + run: cargo clippy -- -D warnings + - name: Audit + run: cargo audit + - run: typos + - name: Formatting + run: cargo fmt --check diff --git a/.github/workflows/codeql-analysis-c.yml b/.github/workflows/codeql.yml similarity index 100% rename from .github/workflows/codeql-analysis-c.yml rename to .github/workflows/codeql.yml diff --git a/.github/workflows/control.yml b/.github/workflows/control.yml new file mode 100644 index 0000000000..750559f64e --- /dev/null +++ b/.github/workflows/control.yml @@ -0,0 +1,18 @@ +name: "Scanner CI" + +on: + push: + branches: [ main] + tags: ["v*"] + pull_request: + workflow_dispatch: + repository_dispatch: + +jobs: + build: + uses: ./.github/workflows/build.yml + linting: + uses: ./.github/workflows/ci.yml + smoketests: + needs: [build] + uses: ./.github/workflows/ci.yml diff --git a/.github/workflows/ddependabot.yml b/.github/workflows/ddependabot.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/nasl.yml b/.github/workflows/nasl.yml index c1599250f3..f6a60f86a7 100644 --- a/.github/workflows/nasl.yml +++ b/.github/workflows/nasl.yml @@ -10,7 +10,7 @@ jobs: nasl-test: name: test runs-on: ubuntu-latest - container: greenbone/gvm-libs:unstable + container: greenbone/gvm-libs:stable steps: - uses: actions/checkout@v4 - name: install dependencies diff --git a/.github/workflows/push-container.yml b/.github/workflows/push-container.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/push-helm-chart.yml b/.github/workflows/push-helm-chart.yml new file mode 100644 index 0000000000..39bc31cf4c --- /dev/null +++ b/.github/workflows/push-helm-chart.yml @@ -0,0 +1,17 @@ +name: "Helm Push" + +on: [workflow_call] + +jobs: + helm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: greenbone/actions/helm-build-push@v3 + if: github.event_name == 'workflow_dispatch' + with: + chart-name: openvasd + registry: ${{ vars.IMAGE_REGISTRY }} + registry-subpath: helm-charts/ + registry-user: ${{ secrets.GREENBONE_BOT }} + registry-token: ${{ secrets.GREENBONE_BOT_PACKAGES_WRITE_TOKEN }} diff --git a/.github/workflows/rustification.yaml b/.github/workflows/rustification.yaml index a41b102381..3804c7136b 100644 --- a/.github/workflows/rustification.yaml +++ b/.github/workflows/rustification.yaml @@ -9,69 +9,6 @@ env: CARGO_TERM_COLOR: always jobs: - unittests: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - strategy: - matrix: - toolchain: - - stable - - beta - - nightly - steps: - - uses: actions/checkout@v4 - - run: sudo apt update && sudo apt-get install -y libpcap-dev - - run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} || rustup default ${{ matrix.toolchain }} - - run: cargo test --lib --tests --workspace - clippy: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - run: rustup update stable && rustup default stable && rustup component add clippy - - run: cargo clippy -- -D warnings - audit: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - run: rustup update stable && rustup default stable - - run: cargo install cargo-audit - - run: cargo audit - typos: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - steps: - - uses: actions/checkout@v4 - - run: rustup update stable && rustup default stable - - run: cargo install typos-cli - - run: typos - formatting: - runs-on: ubuntu-latest - defaults: - run: - working-directory: rust - strategy: - matrix: - crates: - # we verify each dir separately to make it easier to verify formatting issues or even ignore - # crates we deem not important for checking (e.g. feed-verifier) - - nasl-syntax - - storage - - nasl-interpreter - - redis-storage - - json-storage - - nasl-cli - steps: - - uses: actions/checkout@v4 rs-build-binaries: uses: ./.github/workflows/build-rust.yml verify-syntax: diff --git a/.github/workflows/smoketest.yaml b/.github/workflows/smoketest.yaml new file mode 100644 index 0000000000..f9673c1960 --- /dev/null +++ b/.github/workflows/smoketest.yaml @@ -0,0 +1,110 @@ +name: Smoketests + +on: [workflow_call] +# smoke test definition. +# It depends on build.yml that is controlled via control.yml +# +jobs: + OpenVAS_Daemon: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Start a local k8s cluster + uses: jupyterhub/action-k3s-helm@v3 + with: + k3s-channel: latest + metrics-enabled: false + - name: deploy openvasd + run: | + cd rust/examples/tls/Self-Signed\ mTLS\ Method + make delete deploy + cd - + helm uninstall openvasd --namespace openvasd|| true + helm install --namespace openvasd --create-namespace openvasd charts/openvasd/ --values charts/openvasd/values.yaml --values charts/openvasd/mtls-wo-ingress.yaml + kubectl rollout status --watch --timeout 600s deployment/openvasd --namespace openvasd + echo "OPENVASD_SERVER=https://$(kubectl get svc -n openvasd | awk 'FNR == 2 {print $(3)}')" >> $GITHUB_ENV + - name: smoketest + working-directory: rust/smoketest + env: + SCAN_CONFIG: configs/simple_scan_ssh_only.json + CLIENT_KEY: ../examples/tls/Self-Signed mTLS Method/client.rsa + CLIENT_CERT: ../examples/tls/Self-Signed mTLS Method/client.pem + run: | + make build run + + - run: FEED_DIR="feed/" sh .github/prepare-feed.sh + - uses: actions/download-artifact@v3 + with: + name: rs-binaries + path: assets + - run: mv assets/nasl-cli-x86_64-unknown-linux-gnu ./nasl-cli + - run: chmod +x ./nasl-cli + - name: verify syntax parsing + run: ./nasl-cli syntax --quiet feed/ + # find a way to include it... + verify-feed-update: + runs-on: ubuntu-latest + needs: [build] + container: + # maybe better to use builder, build openvas to have + # the version of this checkout rather than a dated official one? + image: greenbone/openvas-scanner:unstable + options: --privileged + services: + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + - run: apt-get update && apt-get install -y docker.io + - run: FEED_DIR="feed/" sh .github/prepare-feed.sh + - uses: actions/download-artifact@v3 + with: + name: rs-binaries + path: assets + - run: mv assets/nasl-cli-x86_64-unknown-linux-gnu ./nasl-cli + - run: mv assets/feed-verifier-x86_64-unknown-linux-gnu ./feed-verifier + - name: prepare setup + run: | + install -m 755 feed-verifier /usr/local/bin/ + install -m 755 nasl-cli /usr/local/bin/ + echo "db_address = tcp://redis:6379" >> /etc/openvas/openvas.conf + mv ./feed/* "$(openvas -s | grep plugins_folder | sed 's/plugins_folder = //')/" + - run: openvas -s + - run: feed-verifier || (cat /var/log/gvm/openvas.log && false) + nasl-test: + name: test + runs-on: ubuntu-latest + container: greenbone/gvm-libs:stable + steps: + - uses: actions/checkout@v4 + - name: install dependencies + run: | + sh .github/install-openvas-dependencies.sh + - name: build openvas + run: | + cmake -Bbuild -DCMAKE_BUILD_TYPE=Release + cmake --build build + - name: redis + run: | + apt-get update && apt-get install --no-install-recommends --no-install-suggests -y redis + mkdir /run/redis-openvas + redis-server config/redis-openvas.conf || exit 1 + - name: scripttests + run: | + mkdir -p /etc/openvas + echo "db_address = /run/redis-openvas/redis.sock" >> /etc/openvas/openvas.conf + cd nasl/tests + OPENVAS_NASL=../../build/nasl/openvas-nasl make check + - uses: actions/setup-go@v5 + with: + go-version: '>=1.16.0' + - name: smoketest/lint + run: | + make build + ./run -e ../../build/nasl/openvas-nasl-lint + working-directory: smoketest_lint