Merge pull request #671 from ralphbean/label-args

Add vendor label and arg to all containerfiles
containers · Jul 12, 2024 · 56ba565 · 56ba565
2 parents 6deab0f + 3f01029
commit 56ba565
Show file tree

Hide file tree

Showing 17 changed files with 84 additions and 17 deletions.
diff --git a/training/Makefile b/training/Makefile
@@ -78,11 +78,11 @@ nvidia: instruct-nvidia
 
 .PHONY: amd-bootc-models nvidia-bootc-models intel-bootc-models
 amd-models: amd
-	$(MAKE) VENDOR=amd -C model bootc-models
+	$(MAKE) PROVIDER=amd -C model bootc-models
 nvidia-models: nvidia
-	$(MAKE) VENDOR=nvidia -C model bootc-models
+	$(MAKE) PROVIDER=nvidia -C model bootc-models
 intel-models: intel
-	$(MAKE) VENDOR=intel -C model bootc-models
+	$(MAKE) PROVIDER=intel -C model bootc-models
 
 .PHONY: bootc-models
 bootc-models:

diff --git a/training/README.md b/training/README.md
@@ -33,6 +33,7 @@ In order to run accelerated AI workloads, we've prepared [bootc](https://github.
 | IMAGE_TAG                 | Container image tag                             | `latest`                                    |
 | CONTAINER_TOOL            | Container tool used for build                   | `podman`                                    |
 | CONTAINER_TOOL_EXTRA_ARGS | Container tool extra arguments                  | ` `                                         |
+| VENDOR                    | Container image vendor label                    | ` `                                         |
 
 
 Note: AI content is huge and requires a lot of disk space >200GB free to build.

diff --git a/training/amd-bootc/Containerfile b/training/amd-bootc/Containerfile
@@ -8,6 +8,10 @@ FROM ${INSTRUCTLAB_IMAGE} AS ilab
 # Define the base image for the second stage
 FROM ${BASEIMAGE}
 
+ARG VENDOR=''
+LABEL vendor=${VENDOR}
+LABEL org.opencontainers.image.vendor=${VENDOR}
+
 ADD rocm.repo /etc/yum.repos.d/rocm.repo
 
 ARG EXTRA_RPM_PACKAGES=''

diff --git a/training/amd-bootc/Makefile b/training/amd-bootc/Makefile
@@ -1,5 +1,5 @@
-VENDOR ?= amd
-IMAGE_NAME ?= $(VENDOR)-bootc
+PROVIDER ?= amd
+IMAGE_NAME ?= $(PROVIDER)-bootc
 
 include ../common/Makefile.common
 
@@ -14,6 +14,7 @@ bootc: prepare-files
 		$(FROM:%=--from=%) \
 		$(INSTRUCTLAB_IMAGE:%=--build-arg INSTRUCTLAB_IMAGE=%) \
 		$(SOURCE_DATE_EPOCH:%=--timestamp=%) \
+		$(VENDOR:%=--build-arg VENDOR=%) \
 		$(if $(SSH_PUBKEY),--build-arg SSHPUBKEY='$(SSH_PUBKEY)') \
 		--cap-add SYS_ADMIN \
 		--file Containerfile \

diff --git a/training/common/Makefile.common b/training/common/Makefile.common
@@ -1,15 +1,16 @@
 FROM ?=
-VENDOR ?=
+PROVIDER ?=
 
 REGISTRY ?= quay.io
 REGISTRY_ORG ?= ai-lab
-IMAGE_NAME ?= $(VENDOR)-bootc
+IMAGE_NAME ?= $(PROVIDER)-bootc
 IMAGE_TAG ?= latest
 BOOTC_IMAGE ?= ${REGISTRY}/${REGISTRY_ORG}/${IMAGE_NAME}:${IMAGE_TAG}
 BOOTC_MODELS_IMAGE ?= ${REGISTRY}/${REGISTRY_ORG}/${IMAGE_NAME}-models:${IMAGE_TAG}
 BOOTC_CLOUD_IMAGE ?= ${REGISTRY}/${REGISTRY_ORG}/${IMAGE_NAME}-cloud:${IMAGE_TAG}
 BOOTC_CLOUD_MODELS_IMAGE ?= ${REGISTRY}/${REGISTRY_ORG}/${IMAGE_NAME}-cloud-models:${IMAGE_TAG}
 FROM_BOOTC_IMAGE ?= $(BOOTC_IMAGE)
+VENDOR ?=
 
 CONTAINER_TOOL ?= podman
 CONTAINER_TOOL_EXTRA_ARGS ?=
@@ -34,7 +35,7 @@ ARCH ?=
 DRIVER_VERSION ?=
 KERNEL_VERSION ?=
 
-INSTRUCTLAB_IMAGE = $(REGISTRY)/$(REGISTRY_ORG)/instructlab-$(VENDOR):$(IMAGE_TAG)
+INSTRUCTLAB_IMAGE = $(REGISTRY)/$(REGISTRY_ORG)/instructlab-$(PROVIDER):$(IMAGE_TAG)
 WRAPPER = $(CURDIR)/../ilab-wrapper/ilab
 QLORA_WRAPPER = $(CURDIR)/../ilab-wrapper/ilab-qlora
 TRAIN_WRAPPER =  $(CURDIR)/../ilab-wrapper/ilab-training-launcher

diff --git a/training/common/driver-toolkit/Containerfile b/training/common/driver-toolkit/Containerfile
@@ -2,6 +2,10 @@ ARG BASEIMAGE="quay.io/centos/centos:stream9"
 
 FROM ${BASEIMAGE}
 
+ARG VENDOR=''
+LABEL vendor=${VENDOR}
+LABEL org.opencontainers.image.vendor=${VENDOR}
+
 ARG KERNEL_VERSION=''
 ARG ENABLE_RT=''
 

diff --git a/training/deepspeed/Containerfile b/training/deepspeed/Containerfile
@@ -1,6 +1,10 @@
 # Containerfile for running deepspeed training 
 FROM nvcr.io/nvidia/cuda:12.1.1-cudnn8-devel-ubi9
 
+ARG VENDOR=''
+LABEL vendor=${VENDOR}
+LABEL org.opencontainers.image.vendor=${VENDOR}
+
 RUN dnf install -y python python-devel git
 RUN python -m ensurepip --upgrade
 RUN pip3 install torch==2.1.2 --index-url https://download.pytorch.org/whl/cu121

diff --git a/training/deepspeed/Makefile b/training/deepspeed/Makefile
@@ -13,6 +13,7 @@ image:
 		$(ARCH:%=--platform linux/%) \
 		$(BUILD_ARG_FILE:%=--build-arg-file=%) \
 		$(SOURCE_DATE_EPOCH:%=--timestamp=%) \
+		$(VENDOR:%=--build-arg VENDOR=%) \
 		--file Containerfile \
 		--squash-all \
 		--tag oci:../build/deepspeed-trainer
diff --git a/training/instructlab/Makefile b/training/instructlab/Makefile
@@ -36,36 +36,64 @@ INSTRUCTLAB_IMAGE_NVIDIA ?= ${REGISTRY}/${REGISTRY_ORG}/instructlab-nvidia:${IMA
 
 ../build/instructlab-nvidia/oci-layout: ./instructlab
 	rm -rf ../build/instructlab-nvidia/
-	"${CONTAINER_TOOL}" build --squash-all -t oci:../build/instructlab-nvidia instructlab/containers/cuda
+	"${CONTAINER_TOOL}" build \
+		--squash-all \
+		-t oci:../build/instructlab-nvidia \
+		$(VENDOR:%=--build-arg VENDOR=%) \
+		instructlab/containers/cuda
 
 .PHONY: nvidia
 nvidia: ../build/instructlab-nvidia/oci-layout
 
 ../build/instructlab-amd/oci-layout: ./instructlab
 	rm -rf ../build/instructlab-amd/
-	"${CONTAINER_TOOL}" build --squash-all -t oci:../build/instructlab-amd -f instructlab/containers/rocm/Containerfile instructlab
+	"${CONTAINER_TOOL}" build \
+		--squash-all \
+		-t oci:../build/instructlab-amd \
+		-f instructlab/containers/rocm/Containerfile \
+		$(VENDOR:%=--build-arg VENDOR=%) \
+		instructlab
 
 .PHONY: amd
 amd: ../build/instructlab-amd/oci-layout
 
 ../build/instructlab-intel/oci-layout: ./instructlab
 	rm -rf ../build/instructlab-intel/
-	"${CONTAINER_TOOL}" build --squash-all -t oci:../build/instructlab-intel  -f instructlab/containers/hpu/Containerfile instructlab
+	"${CONTAINER_TOOL}" build \
+		--squash-all \
+		-t oci:../build/instructlab-intel \
+		-f instructlab/containers/hpu/Containerfile \
+		$(VENDOR:%=--build-arg VENDOR=%) \
+		instructlab
 
 .PHONY: intel
 intel: ../build/instructlab-intel/oci-layout
 
 .PHONY: nvidia-quay
 nvidia-quay: instructlab
-	"${CONTAINER_TOOL}" build --squash-all -t ${INSTRUCTLAB_IMAGE_NVIDIA} instructlab/containers/cuda
+	"${CONTAINER_TOOL}" build \
+		--squash-all \
+		-t ${INSTRUCTLAB_IMAGE_NVIDIA} \
+		$(VENDOR:%=--build-arg VENDOR=%) \
+		instructlab/containers/cuda
 	"${CONTAINER_TOOL}" push ${INSTRUCTLAB_IMAGE_NVIDIA}
 
 .PHONY: amd-quay
 amd-quay: instructlab
-	"${CONTAINER_TOOL}" build --squash-all -t ${INSTRUCTLAB_IMAGE_AMD} -f instructlab/containers/rocm/Containerfile instructlab
+	"${CONTAINER_TOOL}" build \
+		--squash-all \
+		-t ${INSTRUCTLAB_IMAGE_AMD} \
+		-f instructlab/containers/rocm/Containerfile \
+		$(VENDOR:%=--build-arg VENDOR=%) \
+		instructlab
 	"${CONTAINER_TOOL}" push ${INSTRUCTLAB_IMAGE_AMD}
 
 .PHONY: intel-quay
 intel-quay: instructlab
-	"${CONTAINER_TOOL}" build --squash-all -t ${INSTRUCTLAB_IMAGE_INTEL} -f instructlab/containers/hpu/Containerfile instructlab
+	"${CONTAINER_TOOL}" build \
+		--squash-all \
+		-t ${INSTRUCTLAB_IMAGE_INTEL} \
+		-f instructlab/containers/hpu/Containerfile \
+		$(VENDOR:%=--build-arg VENDOR=%) \
+		instructlab
 	"${CONTAINER_TOOL}" push ${INSTRUCTLAB_IMAGE_INTEL}
diff --git a/training/intel-bootc/Containerfile b/training/intel-bootc/Containerfile
@@ -31,6 +31,10 @@ FROM ${BASEIMAGE}
 ARG DRIVER_VERSION=1.16.1-7
 ARG EXTRA_RPM_PACKAGES=''
 
+ARG VENDOR=''
+LABEL vendor=${VENDOR}
+LABEL org.opencontainers.image.vendor=${VENDOR}
+
 COPY --from=builder /home/builder/usr/src/habanalabs-${DRIVER_VERSION}/drivers/accel/habanalabs/habanalabs.ko /tmp/extra/habanalabs.ko
 COPY --from=builder /home/builder/usr/src/habanalabs-${DRIVER_VERSION}/drivers/infiniband/hw/hbl/habanalabs_ib.ko /tmp/extra/habanalabs_ib.ko
 COPY --from=builder /home/builder/usr/src/habanalabs-${DRIVER_VERSION}/drivers/net/ethernet/intel/hbl_cn/habanalabs_cn.ko /tmp/extra/habanalabs_cn.ko

diff --git a/training/intel-bootc/Makefile b/training/intel-bootc/Makefile
@@ -14,6 +14,7 @@ bootc: prepare-files
 		$(FROM:%=--build-arg BASEIMAGE=%) \
 		$(INSTRUCTLAB_IMAGE:%=--build-arg INSTRUCTLAB_IMAGE=%) \
 		$(SOURCE_DATE_EPOCH:%=--timestamp=%) \
+		$(VENDOR:%=--build-arg VENDOR=%) \
 		$(if $(SSH_PUBKEY),--build-arg SSHPUBKEY='$(SSH_PUBKEY)') \
 		--cap-add SYS_ADMIN \
 		--file Containerfile \

diff --git a/training/model/Containerfile b/training/model/Containerfile
@@ -1,4 +1,9 @@
 FROM registry.access.redhat.com/ubi9/ubi
+
+ARG VENDOR=''
+LABEL vendor=${VENDOR}
+LABEL org.opencontainers.image.vendor=${VENDOR}
+
 RUN dnf install -y python3-pip && python3 -m pip install huggingface_hub[cli]
 COPY entrypoint.sh /entrypoint.sh
 WORKDIR /download

diff --git a/training/nvidia-bootc/Containerfile b/training/nvidia-bootc/Containerfile
@@ -75,6 +75,9 @@ ARG BASE_URL='https://us.download.nvidia.com/tesla'
 ARG OS_VERSION_MAJOR=''
 ARG KERNEL_VERSION=''
 
+ARG VENDOR=''
+LABEL vendor=${VENDOR}
+LABEL org.opencontainers.image.vendor=${VENDOR}
 
 ARG DRIVER_TYPE=passthrough
 ENV NVIDIA_DRIVER_TYPE=${DRIVER_TYPE}

diff --git a/training/nvidia-bootc/Makefile b/training/nvidia-bootc/Makefile
@@ -1,5 +1,5 @@
-VENDOR ?= nvidia
-IMAGE_NAME ?= $(VENDOR)-bootc
+PROVIDER ?= nvidia
+IMAGE_NAME ?= $(PROVIDER)-bootc
 
 CUDA_VERSION ?=
 OS_VERSION_MAJOR ?=
@@ -22,6 +22,7 @@ bootc: driver-toolkit check-sshkey prepare-files
 		$(KERNEL_VERSION:%=--build-arg KERNEL_VERSION=%) \
 		$(OS_VERSION_MAJOR:%=--build-arg OS_VERSION_MAJOR=%) \
 		$(SOURCE_DATE_EPOCH:%=--timestamp=%) \
+		$(VENDOR:%=--build-arg VENDOR=%) \
 		$(if $(SSH_PUBKEY),--build-arg SSHPUBKEY='$(SSH_PUBKEY)') \
 		--cap-add SYS_ADMIN \
 		--file Containerfile \

diff --git a/training/tests/provision/templates/Containerfile.j2 b/training/tests/provision/templates/Containerfile.j2
@@ -1,6 +1,10 @@
 FROM quay.io/ai-lab/{{ image_name }}:latest
 ARG sshpubkey
 
+ARG VENDOR=''
+LABEL vendor=${VENDOR}
+LABEL org.opencontainers.image.vendor=${VENDOR}
+
 RUN set -eu && mkdir /usr/etc-system && \
     echo 'AuthorizedKeysFile /usr/etc-system/%u.keys' > /etc/ssh/sshd_config.d/30-auth-system.conf && \
     echo $sshpubkey > /usr/etc-system/root.keys && \

diff --git a/training/vllm/Containerfile b/training/vllm/Containerfile
@@ -1,5 +1,9 @@
 FROM quay.io/wxpe/tgis-vllm:release.4e3ff78
 
+ARG VENDOR=''
+LABEL vendor=${VENDOR}
+LABEL org.opencontainers.image.vendor=${VENDOR}
+
 USER root
 RUN ln -s /usr/lib64/libcuda.so.1 /usr/lib64/libcuda.so
-COPY mixtral.jinja .
+COPY mixtral.jinja .
diff --git a/training/vllm/Makefile b/training/vllm/Makefile
@@ -12,6 +12,7 @@ image:
 		$(ARCH:%=--platform linux/%) \
 		$(BUILD_ARG_FILE:%=--build-arg-file=%) \
 		$(SOURCE_DATE_EPOCH:%=--timestamp=%) \
+		$(VENDOR:%=--build-arg VENDOR=%) \
 		--file Containerfile \
 		--squash-all \
 		--tag oci:../build/vllm \