Remove release downloader script

Dockerfile

@@ -3,6 +3,8 @@ FROM debian:12-slim AS builder
 
 ARG DPDK_VER=22.11
 ARG DPSERVICE_FEATURES=""
+ARG CLI_VERSION=0.1.9
+ARG EXPORTER_VERSION=0.1.8
 
 WORKDIR /workspace
 
@@ -14,7 +16,6 @@ libnuma-dev \
 numactl \
 libnuma1 \
 unzip \
-wget \
 make \
 gcc \
 g++ \
@@ -44,17 +45,18 @@ libpcap0.8-dev \
 linux-headers-${OSARCH}
 
 # Download DPDK
-RUN wget http://git.dpdk.org/dpdk/snapshot/dpdk-${DPDK_VER}.zip
-RUN unzip dpdk-${DPDK_VER}.zip
+ADD http://fast.dpdk.org/rel/dpdk-${DPDK_VER}.tar.xz dpdk.tar.xz
+RUN tar -xJf dpdk.tar.xz
 
 ENV DPDK_DIR=/workspace/dpdk-${DPDK_VER}
 
 # Copy DPDK patches
 COPY hack/*.patch hack/
-RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_gcc12.patch
-RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_log.patch
-RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_telemetry_key.patch
-RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_ethdev_conversion.patch
+RUN cd $DPDK_DIR \
+&& patch -p1 < ../hack/dpdk_22_11_gcc12.patch \
+&& patch -p1 < ../hack/dpdk_22_11_log.patch \
+&& patch -p1 < ../hack/dpdk_22_11_telemetry_key.patch \
+&& patch -p1 < ../hack/dpdk_22_11_ethdev_conversion.patch
 
 # Compile DPDK
 RUN cd $DPDK_DIR && meson setup -Dmax_ethports=132 -Dplatform=generic -Ddisable_drivers=common/dpaax,\
@@ -73,6 +75,11 @@ vhost,gpudev build -Ddisable_apps="*" -Dtests=false
 RUN cd $DPDK_DIR/build && ninja
 RUN cd $DPDK_DIR/build && ninja install
 
+# Get companion binaries from other repos
+ADD https://github.com/ironcore-dev/dpservice-cli/releases/download/v${CLI_VERSION}/github.com.ironcore-dev.dpservice-cli_${CLI_VERSION}_linux_amd64.tar.gz dpservice-cli.tgz
+ADD https://github.com/ironcore-dev/prometheus-dpdk-exporter/releases/download/v${EXPORTER_VERSION}/prometheus-dpdk-exporter_${EXPORTER_VERSION}_linux_amd64.tar.gz exporter.tgz
+RUN tar -xzf dpservice-cli.tgz && tar -xzf exporter.tgz
+
 # Prepare tools and sources
 COPY meson.build meson.build
 COPY meson_options.txt meson_options.txt
@@ -85,13 +92,6 @@ COPY tools/ tools/
 # Needed for version extraction by meson
 COPY .git/ .git/
 
-# TODO this is here and not before dpservice-bin because the tool repos are not public
-# therefore downloading using 'ADD' is not possible
-RUN --mount=type=secret,id=github_token,dst=/run/secrets/github_token \
-sh -c 'GITHUB_TOKEN=$(if [ -f /run/secrets/github_token ]; then cat /run/secrets/github_token; else echo ""; fi) \
-&& ./hack/rel_download.sh -dir=exporter -owner=ironcore-dev -repo=prometheus-dpdk-exporter -pat=$GITHUB_TOKEN \
-&& ./hack/rel_download.sh -dir=client -owner=ironcore-dev -repo=dpservice-cli -strip=2 -pat=$GITHUB_TOKEN'
-
 # Compile dpservice-bin itself
 RUN meson setup build $DPSERVICE_FEATURES && ninja -C build
 
@@ -132,9 +132,9 @@ python3-scapy \
 WORKDIR /
 COPY --from=testbuilder /workspace/test ./test
 COPY --from=testbuilder /workspace/build/src/dpservice-bin ./build/src/dpservice-bin
-COPY --from=testbuilder /workspace/client/* ./build
+COPY --from=testbuilder /workspace/github.com/ironcore-dev/dpservice-cli ./build
 COPY --from=testbuilder /workspace/xtratest_build/src/dpservice-bin ./xtratest_build/src/dpservice-bin
-COPY --from=testbuilder /workspace/client/* ./xtratest_build
+COPY --from=testbuilder /workspace/github.com/ironcore-dev/dpservice-cli ./xtratest_build
 COPY --from=testbuilder /usr/local/lib /usr/local/lib
 RUN ldconfig
 
@@ -162,12 +162,13 @@ bash-completion \
 && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /
-COPY --from=builder /workspace/build/src/dpservice-bin \
-					/workspace/build/tools/dump/dpservice-dump \
-					/workspace/client/* \
-					/workspace/exporter/* \
-					/workspace/hack/prepare.sh \
-					/usr/local/bin
+COPY --from=builder \
+/workspace/build/src/dpservice-bin \
+/workspace/build/tools/dump/dpservice-dump \
+/workspace/github.com/ironcore-dev/dpservice-cli \
+/workspace/prometheus-dpdk-exporter \
+/workspace/hack/prepare.sh \
+/usr/local/bin
 COPY --from=builder /usr/local/lib /usr/local/lib
 RUN ldconfig
 

test/grpc_client.py

@@ -9,7 +9,9 @@
 import socket
 import subprocess
 import sys
+import tarfile
 import time
+import urllib.request
 from config import grpc_port
 
 
@@ -23,20 +25,31 @@ def __str__(self):
 class GrpcClient:
 
 	def __init__(self, build_path):
-		self.cmd = build_path + "/dpservice-cli"
 		self.expectedError = 0
+		self.cmd = build_path + "/dpservice-cli"
 		if not os.access(self.cmd, os.X_OK):
-			self.cmd = build_path + "/github.com/ironcore-dev/dpservice-cli"
-			if not os.access(self.cmd, os.X_OK):
-				print(f"""
-Missing executable grpc client
-To solve this, you have a few options:
- - build one locally and copy it to {build_path}/
- - download one from GitHub manually to {build_path}/
- - use provided download script with your GitHub PAT:
- ./hack/rel_download.sh -dir=build -owner=ironcore-dev -repo=dpservice-cli -pat=<PAT>
-""", file=sys.stderr)
-				raise RuntimeError("no gRPC client")
+			self._downloadCli()
+
+	def _downloadCli(self):
+		print("dpservice-cli is missing, downloading from Github")
+		cli_version = None
+		script_path = os.path.dirname(os.path.abspath(__file__))
+		# Use the same version as the resulting docker images
+		with open(f"{script_path}/../Dockerfile") as f:
+			for line in f.readlines():
+				if "CLI_VERSION=" in line:
+					cli_version = line.split('=')[1].strip()
+					break
+		if not cli_version:
+			raise RuntimeError("no gRPC client - cannot determine a version to download")
+		cli_url = f"https://github.com/ironcore-dev/dpservice-cli/releases/download/v{cli_version}/github.com.ironcore-dev.dpservice-cli_{cli_version}_linux_amd64.tar.gz"
+		urllib.request.urlretrieve(cli_url, f"{self.cmd}.tgz")
+		with tarfile.open(f"{self.cmd}.tgz", mode="r:gz") as tarobj:
+			with tarobj.extractfile("github.com/ironcore-dev/dpservice-cli") as packed:
+				with open(self.cmd, "wb") as outfile:
+					outfile.write(packed.read());
+					os.chmod(self.cmd, 0o755)
+		os.remove(f"{self.cmd}.tgz")
 
 	def getClientVersion(self):
 		return subprocess.check_output([self.cmd, '-v']).decode('utf-8').strip()