Skip to content

Commit

Permalink
Added creation of job template which is responsible for creating the …
Browse files Browse the repository at this point in the history
…private and public key in provider/consumer mode.
  • Loading branch information
rchikatw committed Dec 11, 2023
1 parent 19293d0 commit 8eedf02
Show file tree
Hide file tree
Showing 11 changed files with 296 additions and 10 deletions.
2 changes: 2 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,15 @@ ARG LDFLAGS

RUN GOOS="$GOOS" GOARCH="$GOARCH" go build -ldflags "$LDFLAGS" -tags netgo,osusergo -o ocs-operator main.go
RUN GOOS="$GOOS" GOARCH="$GOARCH" go build -tags netgo,osusergo -o provider-api services/provider/main.go
RUN GOOS="$GOOS" GOARCH="$GOARCH" go build -tags netgo,osusergo -o onboarding-secret-generator onboarding/main.go

# Build stage 2

FROM registry.access.redhat.com/ubi9/ubi-minimal

COPY --from=builder workspace/ocs-operator /usr/local/bin/ocs-operator
COPY --from=builder workspace/provider-api /usr/local/bin/provider-api
COPY --from=builder workspace/onboarding-secret-generator /usr/local/bin/onboarding-secret-generator
COPY --from=builder workspace/metrics/deploy/*rules*.yaml /ocs-prometheus-rules/

RUN chmod +x /usr/local/bin/ocs-operator /usr/local/bin/provider-api
Expand Down
73 changes: 71 additions & 2 deletions controllers/storagecluster/provider_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,28 @@ import (

"go.uber.org/multierr"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/reconcile"

openshiftv1 "github.com/openshift/api/template/v1"
ocsv1 "github.com/red-hat-storage/ocs-operator/v4/api/v1"
"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
"github.com/red-hat-storage/ocs-operator/v4/services/provider/server"
)

const (
ocsProviderServerName = "ocs-provider-server"
providerAPIServerImage = "PROVIDER_API_SERVER_IMAGE"
ocsProviderServerName = "ocs-provider-server"
providerAPIServerImage = "PROVIDER_API_SERVER_IMAGE"
onboardingSecretGeneratorImage = "ONBOARDING_SECRET_GENERATOR_IMAGE"
jobTemplateName = "onboarding-secret-generator"

ocsProviderServicePort = int32(50051)
ocsProviderServiceNodePort = int32(31659)
Expand Down Expand Up @@ -63,6 +69,17 @@ func (o *ocsProviderServer) ensureCreated(r *StorageClusterReconciler, instance
return res, nil
}

actualJob := &batchv1.Job{}

err := r.Client.Get(context.TODO(), types.NamespacedName{Namespace: instance.Namespace, Name: jobTemplateName}, actualJob)

if err != nil && errors.IsNotFound(err) {
r.Log.Info("onboarding job not fund")
createOnboardingSecretTemplate(instance)
} else if actualJob.Status.Conditions[len(actualJob.Status.Conditions)-1].Type != batchv1.JobComplete {
createOnboardingSecretTemplate(instance)
}

return reconcile.Result{}, nil
}

Expand Down Expand Up @@ -434,3 +451,55 @@ func RandomString(l int) string {

return string(bytes)
}

func createOnboardingSecretTemplate(sc *ocsv1.StorageCluster) *openshiftv1.Template {

return &openshiftv1.Template{
ObjectMeta: metav1.ObjectMeta{
Name: jobTemplateName,
Namespace: sc.Namespace,
},
Objects: []runtime.RawExtension{
{
Object: onboardingSecretGenratorJob(sc, jobTemplateName),
},
},
}
}

func onboardingSecretGenratorJob(sc *ocsv1.StorageCluster, jobTemplateName string) *batchv1.Job {

// Annotation template.alpha.openshift.io/wait-for-ready ensures template readiness
annotations := map[string]string{
"template.alpha.openshift.io/wait-for-ready": "true",
}

job := &batchv1.Job{
TypeMeta: metav1.TypeMeta{
Kind: "Job",
APIVersion: "template.openshift.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: jobTemplateName,
Namespace: sc.Namespace,
Annotations: annotations,
},
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
ServiceAccountName: "onboarding-secret-generator",
Containers: []corev1.Container{
{
Name: "onboarding-secret-genrerator",
Image: os.Getenv(onboardingSecretGeneratorImage),
Command: []string{"/usr/local/bin/onboarding-secret-generator"},
},
},
},
},
},
}

return job
}
Original file line number Diff line number Diff line change
Expand Up @@ -3086,6 +3086,8 @@ spec:
value: docker.io/centos/postgresql-12-centos8
- name: PROVIDER_API_SERVER_IMAGE
value: quay.io/ocs-dev/ocs-operator:latest
- name: ONBOARDING_SECRET_GENERATOR_IMAGE
value: quay.io/ocs-dev/ocs-operator:latest
- name: OPERATOR_NAMESPACE
valueFrom:
fieldRef:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: onboarding-secret-generator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: onboarding-secret-generator
subjects:
- kind: ServiceAccount
name: onboarding-secret-generator
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: onboarding-secret-generator
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- create
14 changes: 7 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,11 @@ require (
google.golang.org/protobuf v1.31.0
gopkg.in/ini.v1 v1.67.0
gotest.tools/v3 v3.5.0
k8s.io/api v0.28.4
k8s.io/apiextensions-apiserver v0.28.4
k8s.io/apimachinery v0.28.4
k8s.io/client-go v0.28.4
k8s.io/api v0.28.3
k8s.io/apiextensions-apiserver v0.28.3
k8s.io/apimachinery v0.28.3
k8s.io/client-go v0.28.3
k8s.io/klog v1.0.0
k8s.io/klog/v2 v2.100.1
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
open-cluster-management.io/api v0.11.0
Expand Down Expand Up @@ -130,9 +131,8 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiserver v0.28.4 // indirect
k8s.io/component-base v0.28.4 // indirect
k8s.io/klog v1.0.0 // indirect
k8s.io/apiserver v0.28.3 // indirect
k8s.io/component-base v0.28.3 // indirect
k8s.io/kube-aggregator v0.26.1 // indirect
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
sigs.k8s.io/container-object-storage-interface-api v0.1.0 // indirect
Expand Down
2 changes: 1 addition & 1 deletion hack/source-manifests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ function gen_ocs_csv() {
pushd config/manager
$KUSTOMIZE edit set image ocs-dev/ocs-operator="$OCS_IMAGE"
popd
$KUSTOMIZE build config/manifests/ocs-operator | $OPERATOR_SDK generate bundle -q --overwrite=false --output-dir deploy/ocs-operator --kustomize-dir config/manifests/ocs-operator --package ocs-operator --version "$CSV_VERSION" --extra-service-accounts=ocs-metrics-exporter
$KUSTOMIZE build config/manifests/ocs-operator | $OPERATOR_SDK generate bundle -q --overwrite=false --output-dir deploy/ocs-operator --kustomize-dir config/manifests/ocs-operator --package ocs-operator --version "$CSV_VERSION" --extra-service-accounts=ocs-metrics-exporter,onboarding-secret-generator
mv deploy/ocs-operator/manifests/*clusterserviceversion.yaml $OCS_CSV
cp config/crd/bases/* $ocs_crds_outdir
}
Expand Down
161 changes: 161 additions & 0 deletions onboarding/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
package main

import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"os"

openshiftv1 "github.com/openshift/api/template/v1"
ocsv1 "github.com/red-hat-storage/ocs-operator/v4/api/v1"
ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/v4/api/v1alpha1"
"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
errruntime "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apiruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
)

const (
onboardingTicketPublicKeySecret = "onboarding-ticket-public-key"
onboardingTicketPrivateKeySecret = "onboarding-ticket-private-key"
)

var (
ctx = context.Background()
serverNamespace = os.Getenv(util.WatchNamespaceEnvVar)
privatePem, publicPem string
)

func createSecret(cl client.Client, keyName, secretName string) {

desiredSecret := GetProviderAPIServerSecret(keyName, secretName)
actualSecret := &corev1.Secret{}

err := cl.Get(ctx, client.ObjectKeyFromObject(desiredSecret), actualSecret)

if err != nil && errruntime.IsNotFound(err) {
err = cl.Create(ctx, desiredSecret)
if err != nil {
klog.Error(err, "Failed to create secret.")
}
} else if err != nil {
klog.Error(err, "Failed to get secret.")

}
}

func GetProviderAPIServerSecret(key string, secretName string) *corev1.Secret {

return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
Namespace: serverNamespace,
},
Immutable: func(flag bool) *bool { return &flag }(true),
StringData: map[string]string{
"data": key,
},
// Type: v1.SecretTypeServiceAccountToken,
}
}

func newClient() (client.Client, error) {
scheme := apiruntime.NewScheme()
utilruntime.Must(ocsv1.AddToScheme(scheme))
utilruntime.Must(corev1.AddToScheme(scheme))
utilruntime.Must(ocsv1alpha1.AddToScheme(scheme))
utilruntime.Must(openshiftv1.AddToScheme(scheme))

config, err := config.GetConfig()
if err != nil {
klog.Error(err, "failed to get config check if KUBECOFIG is set.")
return nil, err
}
client, err := client.New(config, client.Options{Scheme: scheme})
if err != nil {
klog.Error(err, "failed to create controller-runtime client")
return nil, err
}

return client, nil
}

func ConvertRsaPrivateKeyAsPemStr(privateKey *rsa.PrivateKey) string {
privteKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey)
privateKeyPem := pem.EncodeToMemory(
&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: privteKeyBytes,
},
)
return string(privateKeyPem)
}

func ConvertRsaPublicKeyAsPemStr(publicKey *rsa.PublicKey) (string, error) {
publicKeyBytes, err := x509.MarshalPKIXPublicKey(publicKey)
if err != nil {
return "", err
}
publicKeyPem := pem.EncodeToMemory(
&pem.Block{
Type: "RSA PUBLIC KEY",
Bytes: publicKeyBytes,
},
)

return string(publicKeyPem), nil
}

func genratePrivateAndPublicKeys() {

// Generate RSA key.
var err error
privateKey, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
panic(err)
}

publicKey := &privateKey.PublicKey

// Export the keys to pem string
//similar to key.pem & pubkey.pem
privatePem = ConvertRsaPrivateKeyAsPemStr(privateKey)
publicPem, _ = ConvertRsaPublicKeyAsPemStr(publicKey)

}

func main() {

client, err := newClient()

if err != nil {
klog.Error(err, "failed to create controller-runtime client")
return
}

// 1. Check public key secret exist or not
secret := &v1.Secret{}
err = client.Get(ctx, types.NamespacedName{Name: onboardingTicketPublicKeySecret,
Namespace: serverNamespace}, secret)

if err != nil {
if errruntime.IsNotFound(err) {
genratePrivateAndPublicKeys()
createSecret(client, onboardingTicketPublicKeySecret, publicPem)
createSecret(client, onboardingTicketPrivateKeySecret, privatePem)
} else {
klog.Error(err, "failed to get secret. ")
}

}

}
11 changes: 11 additions & 0 deletions rbac/onboarding-secret-generator-binding.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: onboarding-secret-generator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: onboarding-secret-generator
subjects:
- kind: ServiceAccount
name: onboarding-secret-generator
13 changes: 13 additions & 0 deletions rbac/onboarding-secret-generator-role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: onboarding-secret-generator
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- create
4 changes: 4 additions & 0 deletions tools/csv-merger/csv-merger.go
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,10 @@ func unmarshalCSV(filePath string) *csvv1.ClusterServiceVersion {
Name: "PROVIDER_API_SERVER_IMAGE",
Value: *ocsContainerImage,
},
{
Name: "ONBOARDING_SECRET_GENERATOR_IMAGE",
Value: *ocsContainerImage,
},
{
Name: util.OperatorNamespaceEnvVar,
ValueFrom: &corev1.EnvVarSource{
Expand Down

0 comments on commit 8eedf02

Please sign in to comment.