From 6887328fdec7d893be84884cda22aedeb5341d3b Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Thu, 5 Sep 2024 09:37:08 +0200 Subject: [PATCH 1/5] Initial proposal of a Dev container As more and more people are using VSCode on a non linux machine we would like to make it easier to start developing on openvas as a whole. With this and an extension `Dev Container` a developer can work within an already setup environment. Additionally it is designed so that someone on a linux machine can also use it with Distrobox or with a direct mounted docker command. For that the GID as well as UID can be set via build parameter for convenience there is a Makefile creating the image with shared UID and GID gatherred from `id`. To build a new image for those purposes run `make build`. Inside the container there a multiple helper integrated, run: - start_redis -- to start a redis instance for openvas-scanner - greenbone-nvt-sync -- to get a community feed - build-openvas -- to rebuild openvas-scanner - build-gvm-libs -- to rebuild gvm-libs Additionally there are github specialized clone helper scripts, for an example executing: - github-clone greenbone/openvas-scanner - github-clone greenbone/gvm-libs will clone the openvas-scanner into /workspaces/greenbone/openvas-scanner and /workspaces/greenbone/gvm-libs respectively so that they can be immediately found by the build- scripts. --- .devcontainer/Dockerfile | 52 ++++++++++++++++++++++++++++ .devcontainer/Makefile | 19 ++++++++++ .devcontainer/build-cmake-project.sh | 10 ++++++ .devcontainer/build-gvm-libs | 8 +++++ .devcontainer/build-openvas | 9 +++++ .devcontainer/devcontainer.json | 3 ++ .devcontainer/github-clone.sh | 38 ++++++++++++++++++++ .devcontainer/prepare-user-dirs.sh | 44 +++++++++++++++++++++++ 8 files changed, 183 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/Makefile create mode 100644 .devcontainer/build-cmake-project.sh create mode 100644 .devcontainer/build-gvm-libs create mode 100644 .devcontainer/build-openvas create mode 100644 .devcontainer/devcontainer.json create mode 100755 .devcontainer/github-clone.sh create mode 100644 .devcontainer/prepare-user-dirs.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..d0a7a1eaf --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,52 @@ + +FROM rust:latest +ARG UID=1000 +ARG GID=1000 + +RUN apt-get update && apt-get install -y sudo git rsync pipx redis-server +# Add prepare-user-dirs.sh and execcute it +COPY prepare-user-dirs.sh /prepare-user-dirs.sh +COPY build-cmake-project.sh /usr/local/bin/build-cmake-project.sh +RUN chmod +x /usr/local/bin/build-cmake-project.sh +COPY build-openvas /usr/local/bin/build-openvas +RUN chmod +x /usr/local/bin/build-openvas +COPY build-gvm-libs /usr/local/bin/build-gvm-libs +RUN chmod +x /usr/local/bin/build-gvm-libs +COPY github-clone.sh /usr/local/bin/github-clone +RUN chmod +x /usr/local/bin/github-clone + +RUN bash /prepare-user-dirs.sh && rm /prepare-user-dirs.sh +USER user +RUN python3 -m pipx install greenbone-feed-sync +# installing gvm-libs and openvas-scanner +RUN github-clone greenbone/gvm-libs +RUN github-clone greenbone/openvas-scanner +RUN sudo sh /workspaces/greenbone/gvm-libs/.github/install-dependencies.sh +RUN sudo sh /workspaces/greenbone/openvas-scanner/.github/install-openvas-dependencies.sh + +RUN build-gvm-libs +RUN build-openvas +# Currently we don't install scannerctl and openvasd as they don't have dependencies +# that must be preloaded in order to function. +# WORKDIR /workspaces/openvas/rust/scannerctl +# RUN cargo install --path . +# WORKDIR /workspaces/openvas/rust/openvasd +# RUN cargo install --path . +USER redis +RUN sed 's/redis-openvas/redis/' /workspaces/greenbone/openvas-scanner/config/redis-openvas.conf | tee /etc/redis/redis.conf +USER user +# We clean up the cloned repositories as they are usually mounted into the container +RUN rm -r /workspaces/greenbone + + +# RUN sudo apt-get clean && sudo rm -rf /var/lib/apt/lists/* + +SHELL ["/bin/bash", "-c"] + +RUN rustup component add rust-analyzer rust-src +ENV PATH="/home/user/.cargo/bin:${PATH}" +ENV PATH="/home/user/.local/bin:${PATH}" +RUN echo "alias start_redis='redis-server /etc/redis/redis.conf'" >> /home/user/.bashrc +ENV start_redis="redis-server /etc/redis/redis.conf" +WORKDIR /workspaces +CMD ["/bin/bash"] diff --git a/.devcontainer/Makefile b/.devcontainer/Makefile new file mode 100644 index 000000000..16ae2002b --- /dev/null +++ b/.devcontainer/Makefile @@ -0,0 +1,19 @@ +# Get the UID and GID of the user those will be used within the Dockerfile to share the same id between host and container. +UID := $(shell id -u) +GID := $(shell id -g) + +.PHONY: build + +build: + docker build \ + --build-arg UID=$(UID) \ + --build-arg GID=$(GID) \ + -t openvas-dev:latest \ + . + +run-tmp: + docker run -it --rm \ + -v $(HOME)/.ssh:/home/user/.ssh\ + -v $(HOME)/.config:/home/user/.config\ + -v $(HOME)/src:/home/user/src \ + openvas-dev:latest diff --git a/.devcontainer/build-cmake-project.sh b/.devcontainer/build-cmake-project.sh new file mode 100644 index 000000000..7d03ec823 --- /dev/null +++ b/.devcontainer/build-cmake-project.sh @@ -0,0 +1,10 @@ +#/bin/sh +[ -d "$1" ] && WORKD_DIR="$1" || ( + echo "Usage: $0 " + exit 1 +) +cd $WORKD_DIR +set -ex +cmake -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON +cmake --build build --target install +sudo ldconfig \ No newline at end of file diff --git a/.devcontainer/build-gvm-libs b/.devcontainer/build-gvm-libs new file mode 100644 index 000000000..db17c1a29 --- /dev/null +++ b/.devcontainer/build-gvm-libs @@ -0,0 +1,8 @@ +#!/bin/bash +owner=${1:-greenbone} +if [ -d "/workspaces/$owner" ]; then + target_dir="/workspaces/$owner/gvm-libs" +else + target_dir="/workspaces/gvm-libs" +fi +/usr/local/bin/build-cmake-project.sh "$target_dir" \ No newline at end of file diff --git a/.devcontainer/build-openvas b/.devcontainer/build-openvas new file mode 100644 index 000000000..db1feecce --- /dev/null +++ b/.devcontainer/build-openvas @@ -0,0 +1,9 @@ +#!/bin/bash +owner=${1:-greenbone} +if [ -d "/workspaces/$owner" ]; then + target_dir="/workspaces/$owner/openvas-scanner" +else + target_dir="/workspaces/openvas-scanner" +fi + +/usr/local/bin/build-cmake-project.sh "$target_dir" \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..dae163bfb --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,3 @@ +{ + "build": { "dockerfile": "Dockerfile" }, +} diff --git a/.devcontainer/github-clone.sh b/.devcontainer/github-clone.sh new file mode 100755 index 000000000..79c5beec2 --- /dev/null +++ b/.devcontainer/github-clone.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +if [ -z "$1" ]; then + echo "Error: Repository name is not provided." + exit 1 +fi + +IFS='/' read -r owner repo <<< "$1" + +parent_dir="/workspaces" +if [ ! -d "$parent_dir" ]; then + echo "Parent directory '$parent_dir' does not exist. Creating it." + mkdir -p "$parent_dir" +fi + +owner_dir="$parent_dir/$owner" +if [ ! -d "$owner_dir" ]; then + echo "Owner directory '$owner_dir' does not exist. Creating it." + mkdir -p "$owner_dir" +fi + +target_dir="/workspaces/$1" + +if [ -d "$target_dir" ]; then + echo "Error: Target directory '$target_dir' already exists." + exit 1 +fi + +if ls id_* &>/dev/null; then + if git clone git@github.com:$1.git "$target_dir"; then + echo "Cloning with SSH URL successful." + else + echo "Warning: Cloning with SSH URL failed. Falling back to HTTPS URL." + git clone https://github.com/$1.git "$target_dir" + fi +else + git clone https://github.com/$1.git "$target_dir" +fi \ No newline at end of file diff --git a/.devcontainer/prepare-user-dirs.sh b/.devcontainer/prepare-user-dirs.sh new file mode 100644 index 000000000..a98a48501 --- /dev/null +++ b/.devcontainer/prepare-user-dirs.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# This scripts creates the dirs defined in dirs and sets the rights to the given user and id. +# This script creates a user with a $UID as well as a group with $GID +# afterwards it creates set of directories, assigns ownership to a newly created user and group, and configures sudo permissions for the user. +# This is done to allow cmake --build build --target install to work without permission issues. + +dirs=" +/workspaces +/run/gvm +/var/log/gvm +/etc/openvas +/var/lib/openvas +/usr/local/lib +/usr/local/share/man/man1/ +/usr/local/share/man/man8/ +/usr/local/include/gvm +/usr/local/share/openvas +/usr/local/bin +/usr/local/sbin +/var/lib/openvas +/var/lib/notus +/var/lib/gvm +/var/lib/openvasd +/etc/openvasd +/run/redis +" + +set -ex +groupadd --gid "$GID" "developer" || true +# for the case that the GID already existed when we tried to create developer +# this can happen when we reuse staff from a mac os host +group_name=$(getent group "$GID" | cut -d: -f1) + +useradd --uid "$UID" --gid "$group_name" --shell /bin/bash --groups redis --create-home user + +for dir in ${dirs[@]}; do + if [ ! -d $dir ]; then + mkdir -p $dir + fi + chown -R user:$group_name $dir +done +# allow user to run sudo without password since it is intented as development +# container it is assumed that the user wants to install or manipulate the container +echo "user ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/user From dfc0d2bd1e9ba965c117e04f2ec68f17d67b0723 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Fri, 4 Oct 2024 11:29:44 +0200 Subject: [PATCH 2/5] Remove tmp and add some convencience in the Makefile --- .devcontainer/Dockerfile | 3 +- .devcontainer/Makefile | 73 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d0a7a1eaf..5d51cc2ca 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,9 +1,8 @@ - FROM rust:latest ARG UID=1000 ARG GID=1000 -RUN apt-get update && apt-get install -y sudo git rsync pipx redis-server +RUN apt-get update && apt-get install -y sudo git rsync pipx redis-server clangd # Add prepare-user-dirs.sh and execcute it COPY prepare-user-dirs.sh /prepare-user-dirs.sh COPY build-cmake-project.sh /usr/local/bin/build-cmake-project.sh diff --git a/.devcontainer/Makefile b/.devcontainer/Makefile index 16ae2002b..415860ce4 100644 --- a/.devcontainer/Makefile +++ b/.devcontainer/Makefile @@ -1,19 +1,84 @@ +# TODO: +# - add update script +# - change install-nvim to adapt update script to also update neovim + + # Get the UID and GID of the user those will be used within the Dockerfile to share the same id between host and container. UID := $(shell id -u) GID := $(shell id -g) +MF_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +# if podman exists, use it instead of docker +ifneq (,$(shell which podman)) + CONTAINERR=podman +else + CONTAINERR=docker +endif +# disable docker hints, who needs that? +export DOCKER_CLI_HINTS=false .PHONY: build +command-exists = $(CONTAINERR) exec -it openvas-dev command -v $(1) >/dev/null 2>&1 && echo "exists" || echo "not exists" +# @if [ "$$(basename $$SHELL)" = "fish" ]; then \ +get-shell = $(basename $(notdir $(SHELL))) + build: - docker build \ + $(CONTAINERR) build \ --build-arg UID=$(UID) \ --build-arg GID=$(GID) \ -t openvas-dev:latest \ . -run-tmp: - docker run -it --rm \ +start: + $(CONTAINERR) start openvas-dev + +create: + $(CONTAINERR) create -it \ + --name openvas-dev \ -v $(HOME)/.ssh:/home/user/.ssh\ -v $(HOME)/.config:/home/user/.config\ - -v $(HOME)/src:/home/user/src \ + -v $(HOME)/.gitconfig:/home/user/.gitconfig \ openvas-dev:latest + +is-running: + $(CONTAINERR) ps -q --filter "name=openvas-dev" | grep -q . + +enforce-running: + $(MAKE) is-running || $(MAKE) start || $(MAKE) create && $(MAKE) start + +install-fish: enforce-running + $(CONTAINERR) exec -it openvas-dev /bin/bash -c "sudo apt update" + $(CONTAINERR) exec -it openvas-dev /bin/bash -c "sudo apt install -y fish" + # doesn't work because of attached tty on create there is no reinit of the shell + #$(CONTAINERR) exec -it openvas-dev /bin/bash -c "sudo chsh -s /usr/bin/fish user" + + +install-rg-fzf: enforce-running + $(CONTAINERR) exec -it openvas-dev /bin/bash -c "sudo apt update" + $(CONTAINERR) exec -it openvas-dev /bin/bash -c "sudo apt install -y ripgrep fzf" + +install-nvim: install-rg-fzf + $(CONTAINERR) exec -it openvas-dev /bin/bash -c "sudo apt install -y ninja-build gettext cmake unzip curl build-essential nodejs" + $(CONTAINERR) exec -it openvas-dev /bin/bash -c "github-clone neovim/neovim" + $(CONTAINERR) exec -it openvas-dev /bin/bash -c "cd /workspaces/neovim/neovim && make CMAKE_BUILD_TYPE=RelWithDebInfo && sudo make install" + + +enter: enforce-running + @if $(call command-exists,fish); then \ + $(MAKE) enter-fish; \ + else \ + $(MAKE) enter-bash; \ + fi + +enter-bash: enforce-running + $(CONTAINERR) exec -it openvas-dev /bin/bash + +# TODO: detect running shell and use that +enter-fish: enforce-running + $(CONTAINERR) exec -it openvas-dev /usr/bin/fish + +stop: + -$(CONTAINERR) stop openvas-dev + +rm: stop + $(CONTAINERR) rm openvas-dev From e732725edcd13bd5bcc27be113108ceebbbf0588 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Tue, 15 Oct 2024 09:54:10 +0200 Subject: [PATCH 3/5] Fixes wrong directory when figuring out ssh When figuring out if ssh should be used then the ls check should be in `$HOME/.ssh` not in the execution directory of `github-clone.sh`. --- .devcontainer/github-clone.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/github-clone.sh b/.devcontainer/github-clone.sh index 79c5beec2..a4ff28c09 100755 --- a/.devcontainer/github-clone.sh +++ b/.devcontainer/github-clone.sh @@ -26,7 +26,7 @@ if [ -d "$target_dir" ]; then exit 1 fi -if ls id_* &>/dev/null; then +if ls $HOME/.ssh/id_* &>/dev/null; then if git clone git@github.com:$1.git "$target_dir"; then echo "Cloning with SSH URL successful." else @@ -35,4 +35,4 @@ if ls id_* &>/dev/null; then fi else git clone https://github.com/$1.git "$target_dir" -fi \ No newline at end of file +fi From 055899401259f91369770f06e7be8cfe0f78f010 Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Fri, 25 Oct 2024 13:37:33 +0200 Subject: [PATCH 4/5] Add pyright --- .devcontainer/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.devcontainer/Makefile b/.devcontainer/Makefile index 415860ce4..115c21161 100644 --- a/.devcontainer/Makefile +++ b/.devcontainer/Makefile @@ -52,6 +52,9 @@ install-fish: enforce-running # doesn't work because of attached tty on create there is no reinit of the shell #$(CONTAINERR) exec -it openvas-dev /bin/bash -c "sudo chsh -s /usr/bin/fish user" +install-pyright: enforce-running + $(CONTAINERR) exec -it openvas-dev /bin/bash -c "pipx install pyright" + install-rg-fzf: enforce-running $(CONTAINERR) exec -it openvas-dev /bin/bash -c "sudo apt update" From 1fb058a77052dc2ea93d98937f2e4afec0cdb15d Mon Sep 17 00:00:00 2001 From: Philipp Eder Date: Mon, 11 Nov 2024 10:25:17 +0100 Subject: [PATCH 5/5] Adds runtime dependencies for openvas --- .devcontainer/Dockerfile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5d51cc2ca..75f889205 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -2,7 +2,21 @@ FROM rust:latest ARG UID=1000 ARG GID=1000 -RUN apt-get update && apt-get install -y sudo git rsync pipx redis-server clangd +RUN apt-get update && apt-get install -y \ + sudo \ + git \ + rsync \ + pipx \ + redis-server \ + clangd \ + # Runtime dependencies, required for .devcontainer + nmap \ + snmp \ + netdiag \ + pnscan \ + # net-tools is required by some nasl plugins. + # nasl_pread: Failed to execute child process “netstat” (No such file or directory) + net-tools # Add prepare-user-dirs.sh and execcute it COPY prepare-user-dirs.sh /prepare-user-dirs.sh COPY build-cmake-project.sh /usr/local/bin/build-cmake-project.sh