add support to push to remote registry (#18)

* add support to push to remote registry * ensure default to deploy is true Signed-off-by: vsoch <[email protected]>
converged-computing · Nov 30, 2023 · 320b000 · 320b000
1 parent 2627284
commit 320b000
Show file tree

Hide file tree

Showing 18 changed files with 344 additions and 31 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@
 # Binaries for programs and plugins
 ./vendor
 *.exe
+.env
 *.exe~
 *.dll
 *.so

diff --git a/api/v1alpha1/orascache_types.go b/api/v1alpha1/orascache_types.go
@@ -23,9 +23,29 @@ type OrasCacheSpec struct {
 	// +optional
 	Image string `json:"image"`
 
+	// Names of secrets for the operator
+	// +optional
+	Secrets Secrets `json:"secrets"`
+
+	// Skip deploying the registry (stateful set) implying all references
+	// are for a remote (existing) registry
+	// +kubebuilder:default=true
+	// +default=true
+	// +optional
+	Deploy bool `json:"deploy"`
+}
+
+type Secrets struct {
+
+	// Secrets for the environment for the ORAS operator sidecar pod to push
+	// e.g., oras pull -u username -p password myregistry.io/myimage:latest
+	// This should have ORAS_USER and ORAS_PASS
+	// +optional
+	OrasEnv string `json:"orasEnv"`
+
 	// Secret for the registry REGISTRY_HTTP_SECRET
 	// +optional
-	Secret string `json:"secret"`
+	RegistryHttp string `json:"registryHttp"`
 }
 
 // OrasCacheStatus defines the observed state of OrasCache

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/cache.converged-computing.github.io_orascaches.yaml b/config/crd/bases/cache.converged-computing.github.io_orascaches.yaml
@@ -35,13 +35,28 @@ spec:
           spec:
             description: OrasCacheSpec defines the desired state of OrasCache
             properties:
+              deploy:
+                default: true
+                description: Skip deploying the registry (stateful set) implying all
+                  references are for a remote (existing) registry
+                type: boolean
               image:
                 default: ghcr.io/oras-project/registry:latest
                 description: Image is the oras registry to deploy
                 type: string
-              secret:
-                description: Secret for the registry REGISTRY_HTTP_SECRET
-                type: string
+              secrets:
+                description: Names of secrets for the operator
+                properties:
+                  orasEnv:
+                    description: Secrets for the environment for the ORAS operator
+                      sidecar pod to push e.g., oras pull -u username -p password
+                      myregistry.io/myimage:latest This should have ORAS_USER and
+                      ORAS_PASS
+                    type: string
+                  registryHttp:
+                    description: Secret for the registry REGISTRY_HTTP_SECRET
+                    type: string
+                type: object
             type: object
           status:
             description: OrasCacheStatus defines the observed state of OrasCache

diff --git a/controllers/oras/orascache_controller.go b/controllers/oras/orascache_controller.go
@@ -107,9 +107,12 @@ func (r *OrasCacheReconciler) ensureOrasCache(
 	}
 
 	// The service running the oras registry is a stateful set
-	_, result, _, err = r.getStatefulSet(ctx, spec)
-	if err != nil {
-		return result, err
+	// But only deploy if we are requested to!
+	if spec.Spec.Deploy {
+		_, result, _, err = r.getStatefulSet(ctx, spec)
+		if err != nil {
+			return result, err
+		}
 	}
 	return ctrl.Result{}, nil
 }

diff --git a/controllers/oras/statefulset.go b/controllers/oras/statefulset.go
@@ -120,15 +120,30 @@ func (r *OrasCacheReconciler) createStatefulSet(
 		},
 	}
 
+	// Prepare environment variables
 	// Do we have a secret?
-	if spec.Spec.Secret != "" {
-		env := []corev1.EnvVar{{
+	if spec.Spec.Secrets.RegistryHttp != "" {
+		env := []corev1.EnvVar{}
+		env = append(env, corev1.EnvVar{
 			Name:  "REGISTRY_HTTP_SECRET",
-			Value: spec.Spec.Secret,
-		}}
+			Value: spec.Spec.Secrets.RegistryHttp,
+		})
 		set.Spec.Template.Spec.Containers[0].Env = env
 	}
 
+	// Are we also adding environment variables from a secret?
+	if spec.Spec.Secrets.OrasEnv != "" {
+		set.Spec.Template.Spec.Containers[0].EnvFrom = []v1.EnvFromSource{
+			{
+				SecretRef: &corev1.SecretEnvSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: spec.Spec.Secrets.OrasEnv,
+					},
+				},
+			},
+		}
+	}
+
 	// Controller reference always needs to be set before creation
 	ctrl.SetControllerReference(spec, set, r.Scheme)
 	err := r.Client.Create(ctx, set)

diff --git a/docs/getting_started/custom-resource-definition.md b/docs/getting_started/custom-resource-definition.md
@@ -43,14 +43,40 @@ spec:
   image: ghcr.io/oras-project/registry:latest
 ```
 
-#### secret
+#### deploy
 
-In the case of deploying more than one registry, the push secret can be customized.
+If deploy is set to false, this indicates that you don't want the operator to deploy a stateful set local registry.
+This means that you'll need to provide all jobs / pods with a remote `registry` field to replace it, and (likely) a secret to pull or push.
+See [orasEnv](#orasEnv) below for details.
 
 ```yaml
 spec:
-  # We can use all the defaults here (this is a default)
-  secret: mysecret
+  deploy: false 
+```
+
+#### secrets
+
+There are several secrets that can be added, if needed.
+
+##### orasEnv
+
+If you deploy a secret to the namespace that you want the ORAS operator to use (providing in the environment to the pod)
+you can define the name here.
+
+```yaml
+spec:
+  secrets: 
+    orasEnv: mysecret-name
+```
+
+##### registryHttp
+
+In the case of deploying more than one registry pod (not supported yet), the push secret can be customized.
+
+```yaml
+spec:
+  secrets: 
+    registryHttp: mysecret
 ```
 
 Note that this is not supported yet - likely we would want to add custom volumes (for shared storage) between more than one pod in the stateful set. 

diff --git a/docs/getting_started/user-guide.md b/docs/getting_started/user-guide.md
@@ -149,15 +149,38 @@ once, first for the job, and then for the underlying pod(s) it creates. Annotati
 | output-uri | The output unique resource identifier for the registry step, including repository, name, and tag | false | false |NA will be used if not defined, meaning the step has no outputs |
 | oras-cache | The name of the sidecar orchestrator | false | false | oras |
 | oras-container | The container with oras to run for the service | false | false | ghcr.io/oras-project/oras:v1.1.0 |
+| registry | A custom remote registry URI for all containers | false | false | unset |
+| oras-env | A secret in the same namespace to add to the sidecar | false | false | unset |
 | container | The name of the launcher container | false | false | assumes the first container found requires the launcher |
 | entrypoint | The https address of the application entrypoint to wget | false | false | [entrypoint.sh](https://raw.githubusercontent.com/converged-computing/oras-operator/main/hack/entrypoint.sh) |
 | oras-entrypoint | The https address of the oras cache sidecar entrypoint to wget | false | false | [oras-entrypoint.sh](https://raw.githubusercontent.com/converged-computing/oras-operator/main/hack/oras-entrypoint.sh) |
 | debug | Print all discovered settings in the operator log | false | false | "false" |
 | unpack | Unpack a directory compressed to .tar.gz | false | false | "true" |
 
 There should not be a need to change the oras-cache (sidecar container) unless for some reason you have another container in the pod also called oras. It is exposed for this rare case.
+More details about specific annotations are provided below.
 
-Note that when List is true, this means the annotation can be provided as a list, and more than one value can be added with the pattern `<prefix>/<field>_<count>`. Currently the only supported list field is `input-uri`, anticipating that multiple parent steps might feed into one child step.
+#### Registry
+
+By default, the URIs that are provided are prefixed with the registry that the ORAS Operator has deployed on the same cluster. However, if registry is supplied, 
+it is assumed to be used for the entire job or pod definition, and this registry is used instead. If you absolutely don't need the local registry you can
+set "deploy" to false in the operator CRD (and it will be skipped). Also note that if you are pushing to a remote registry, you'll need to provide
+the name of a secret with your login and password via `oras-env`, discussed next.
+
+#### oras-env
+
+This should be the name of a secret in the same namespace as your pod or job that has one or more secrets to add to the environment. As an example to create such a secret:
+
+```bash
+kubectl create secret generic oras-env --from-literal="ORAS_USER=${ORAS_USER}" --from-literal="ORAS_PUSH_PASS=${ORAS_PASS}" --from-literal="ORAS_PULL_PASS=${ORAS_PASS}"
+```
+The most common use case here is to provide credentials for pushing or pulling to a remote registry (when deploy is false). We provide the same credential twice for pull and push because it could
+be the case that you only want to add for one or the other, and in fact adding to a command that doesn't require it could result in an error. The secret must be in the same namespace
+as your job or pod.
+
+#### Multiple input-uri
+
+Note that when List is true, this means the annotation can be provided as a list, and more than one value can be added with the pattern `<prefix>/<field>_<count>`. Currently the only supported list field is `input-uri`, anticipating that multiple parent steps might feed into one child step. Here is an example with multiple input uris.
 
 ```yaml
 kind: Pod
@@ -215,6 +238,7 @@ kubectl logs -n oras-operator-system oras-operator-controller-manager-ff66845dd-
 You can then try one of the [examples](https://github.com/converged-computing/oras-operator/tree/main/examples) in the repository. A brief description of each is provided here,
 and likely we will add more detail as we develop them.
 
- - [tests/registry](https://github.com/converged-computing/oras-operator/tree/main/examples/tests/registry/roas.yaml): a basic example of creating an ORAS registry cache
+ - [registry](https://github.com/converged-computing/oras-operator/tree/main/examples/registry): demonstrates interaction with a remote registry and disable of a local (namespaced) deployment
+ - [tests/registry](https://github.com/converged-computing/oras-operator/tree/main/examples/tests/registry/oras.yaml): a basic example of creating an ORAS registry cache
  - [tests/hello-world](https://github.com/converged-computing/oras-operator/tree/main/examples/tests/hello-world): Writing one "hello-world" output and saving to an ORAS registry cache
  - [workflows/metrics](https://github.com/converged-computing/oras-operator/tree/main/examples/woirkflow/metrics): Running two Metrics Operator apps/metrics (LAMMPS and HWLOC) and getting results for each!
diff --git a/examples/dist/oras-operator-arm.yaml b/examples/dist/oras-operator-arm.yaml
@@ -53,13 +53,24 @@ spec:
           spec:
             description: OrasCacheSpec defines the desired state of OrasCache
             properties:
+              deploy:
+                default: true
+                description: Skip deploying the registry (stateful set) implying all references are for a remote (existing) registry
+                type: boolean
               image:
                 default: ghcr.io/oras-project/registry:latest
                 description: Image is the oras registry to deploy
                 type: string
-              secret:
-                description: Secret for the registry REGISTRY_HTTP_SECRET
-                type: string
+              secrets:
+                description: Names of secrets for the operator
+                properties:
+                  orasEnv:
+                    description: Secrets for the environment for the ORAS operator sidecar pod to push e.g., oras pull -u username -p password myregistry.io/myimage:latest This should have ORAS_USER and ORAS_PASS
+                    type: string
+                  registryHttp:
+                    description: Secret for the registry REGISTRY_HTTP_SECRET
+                    type: string
+                type: object
             type: object
           status:
             description: OrasCacheStatus defines the observed state of OrasCache

diff --git a/examples/dist/oras-operator-dev.yaml b/examples/dist/oras-operator-dev.yaml
@@ -53,13 +53,24 @@ spec:
           spec:
             description: OrasCacheSpec defines the desired state of OrasCache
             properties:
+              deploy:
+                default: true
+                description: Skip deploying the registry (stateful set) implying all references are for a remote (existing) registry
+                type: boolean
               image:
                 default: ghcr.io/oras-project/registry:latest
                 description: Image is the oras registry to deploy
                 type: string
-              secret:
-                description: Secret for the registry REGISTRY_HTTP_SECRET
-                type: string
+              secrets:
+                description: Names of secrets for the operator
+                properties:
+                  orasEnv:
+                    description: Secrets for the environment for the ORAS operator sidecar pod to push e.g., oras pull -u username -p password myregistry.io/myimage:latest This should have ORAS_USER and ORAS_PASS
+                    type: string
+                  registryHttp:
+                    description: Secret for the registry REGISTRY_HTTP_SECRET
+                    type: string
+                type: object
             type: object
           status:
             description: OrasCacheStatus defines the observed state of OrasCache

diff --git a/examples/dist/oras-operator.yaml b/examples/dist/oras-operator.yaml
@@ -53,13 +53,24 @@ spec:
           spec:
             description: OrasCacheSpec defines the desired state of OrasCache
             properties:
+              deploy:
+                default: true
+                description: Skip deploying the registry (stateful set) implying all references are for a remote (existing) registry
+                type: boolean
               image:
                 default: ghcr.io/oras-project/registry:latest
                 description: Image is the oras registry to deploy
                 type: string
-              secret:
-                description: Secret for the registry REGISTRY_HTTP_SECRET
-                type: string
+              secrets:
+                description: Names of secrets for the operator
+                properties:
+                  orasEnv:
+                    description: Secrets for the environment for the ORAS operator sidecar pod to push e.g., oras pull -u username -p password myregistry.io/myimage:latest This should have ORAS_USER and ORAS_PASS
+                    type: string
+                  registryHttp:
+                    description: Secret for the registry REGISTRY_HTTP_SECRET
+                    type: string
+                type: object
             type: object
           status:
             description: OrasCacheStatus defines the observed state of OrasCache