diff --git a/.gitignore b/.gitignore index f67ce2eb0..bfdd9323b 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,7 @@ temp # Tilt files. .tiltbuild /tilt.d -tilt-settings.json +tilt-settings.yaml tilt_config.json .clusterstack.yaml .cluster.yaml @@ -81,3 +81,4 @@ main # helm generated yamls *.tgz.yaml *.build.yaml +.release diff --git a/Tiltfile b/Tiltfile index 8ed0751f3..4075ed217 100644 --- a/Tiltfile +++ b/Tiltfile @@ -17,18 +17,19 @@ settings = { "allowed_contexts": [ "kind-cso", ], + "local_mode": False, "deploy_cert_manager": True, "preload_images_for_kind": True, "kind_cluster_name": "cso", - "capi_version": "v1.5.2", + "capi_version": "v1.6.0", "cert_manager_version": "v1.11.0", "kustomize_substitutions": { }, } # global settings -settings.update(read_json( - "tilt-settings.json", +settings.update(read_yaml( + "tilt-settings.yaml", default = {}, )) @@ -111,19 +112,32 @@ def fixup_yaml_empty_arrays(yaml_str): return yaml_str.replace("storedVersions: null", "storedVersions: []") ## This should have the same versions as the Dockerfile -tilt_dockerfile_header_cso = """ -FROM docker.io/alpine/helm:3.12.2 as helm - -FROM docker.io/library/alpine:3.18.0 as tilt -WORKDIR / -COPY --from=helm --chown=root:root --chmod=755 /usr/bin/helm /usr/local/bin/helm -COPY manager . -""" +if settings.get("local_mode"): + tilt_dockerfile_header_cso = """ + FROM docker.io/alpine/helm:3.12.2 as helm + + FROM docker.io/library/alpine:3.18.0 as tilt + WORKDIR / + COPY --from=helm --chown=root:root --chmod=755 /usr/bin/helm /usr/local/bin/helm + COPY .tiltbuild/manager . + COPY .release/ /tmp/cluster-stacks/ + """ +else: + tilt_dockerfile_header_cso = """ + FROM docker.io/alpine/helm:3.12.2 as helm + + FROM docker.io/library/alpine:3.18.0 as tilt + WORKDIR / + COPY --from=helm --chown=root:root --chmod=755 /usr/bin/helm /usr/local/bin/helm + COPY manager . + """ # Build CSO and add feature gates def deploy_cso(): - # yaml = str(kustomizesub("./hack/observability")) # build an observable kind deployment by default - yaml = str(kustomizesub("./config/default")) + if settings.get("local_mode"): + yaml = str(kustomizesub("./config/localmode")) + else: + yaml = str(kustomizesub("./config/default")) local_resource( name = "cso-components", cmd = ["sh", "-ec", sed_cmd, yaml, "|", envsubst_cmd], @@ -152,18 +166,32 @@ def deploy_cso(): # Set up an image build for the provider. The live update configuration syncs the output from the local_resource # build into the container. - docker_build_with_restart( - ref = "ghcr.io/sovereigncloudstack/cso-staging", - context = "./.tiltbuild/", - dockerfile_contents = tilt_dockerfile_header_cso, - target = "tilt", - entrypoint = entrypoint, - only = "manager", - live_update = [ - sync(".tiltbuild/manager", "/manager"), - ], - ignore = ["templates"], - ) + if settings.get("local_mode"): + docker_build_with_restart( + ref = "ghcr.io/sovereigncloudstack/cso-staging", + context = ".", + dockerfile_contents = tilt_dockerfile_header_cso, + target = "tilt", + entrypoint = entrypoint, + live_update = [ + sync(".tiltbuild/manager", "/manager"), + sync(".release", "/tmp/cluster-stacks"), + ], + ignore = ["templates"], + ) + else: + docker_build_with_restart( + ref = "ghcr.io/sovereigncloudstack/cso-staging", + context = "./.tiltbuild/", + dockerfile_contents = tilt_dockerfile_header_cso, + target = "tilt", + entrypoint = entrypoint, + live_update = [ + sync(".tiltbuild/manager", "/manager"), + ], + ignore = ["templates"], + ) + k8s_yaml(blob(yaml)) k8s_resource(workload = "cso-controller-manager", labels = ["CSO"]) k8s_resource( diff --git a/cmd/main.go b/cmd/main.go index 4bc3da9bc..8c16842bd 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -29,6 +29,7 @@ import ( "github.com/SovereignCloudStack/cluster-stack-operator/internal/controller" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/csoversion" githubclient "github.com/SovereignCloudStack/cluster-stack-operator/pkg/github/client" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/github/client/fake" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/kube" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/utillog" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/workloadcluster" @@ -67,6 +68,7 @@ var ( clusterAddonConcurrency int logLevel string releaseDir string + localMode bool qps float64 burst int ) @@ -83,6 +85,7 @@ func main() { flag.IntVar(&clusterAddonConcurrency, "clusteraddon-concurrency", 1, "Number of ClusterAddons to process simultaneously") flag.StringVar(&logLevel, "log-level", "info", "Specifies log level. Options are 'debug', 'info' and 'error'") flag.StringVar(&releaseDir, "release-dir", "/tmp/downloads/", "Specify release directory for cluster-stack releases") + flag.BoolVar(&localMode, "local", false, "Enable local mode where no release assets will be downloaded from a remote Git repository. Useful for implementing cluster stacks.") flag.Float64Var(&qps, "qps", 50, "Enable custom query per second for kubernetes API server") flag.IntVar(&burst, "burst", 100, "Enable custom burst defines how many queries the API server will accept before enforcing the limit established by qps") @@ -116,7 +119,12 @@ func main() { // Setup the context that's going to be used in controllers and for the manager. ctx := ctrl.SetupSignalHandler() - gitFactory := githubclient.NewFactory() + var gitFactory githubclient.Factory + if localMode { + gitFactory = fake.NewFactory() + } else { + gitFactory = githubclient.NewFactory() + } restConfig := mgr.GetConfig() restConfig.QPS = float32(qps) diff --git a/config/cso/clusterstack.yaml b/config/cso/clusterstack.yaml index 035e059d2..36e5de4b3 100644 --- a/config/cso/clusterstack.yaml +++ b/config/cso/clusterstack.yaml @@ -11,4 +11,4 @@ spec: autoSubscribe: false noProvider: true versions: - - v2 \ No newline at end of file + - v2 diff --git a/config/localmode/kustomization.yaml b/config/localmode/kustomization.yaml new file mode 100644 index 000000000..835ccdd63 --- /dev/null +++ b/config/localmode/kustomization.yaml @@ -0,0 +1,43 @@ +namespace: cso-system + +namePrefix: cso- + +commonLabels: + cluster.x-k8s.io/provider: "infrastructure-cluster-stack-operator" + +resources: + - ../crd + - ../rbac + - ../manager + - ../certmanager + +patchesStrategicMerge: + - manager_config_patch.yaml + - manager_pull_policy.yaml +# vars: +# - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +# - name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# - name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +# - name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/config/localmode/manager_config_patch.yaml b/config/localmode/manager_config_patch.yaml new file mode 100644 index 000000000..576e6a6cf --- /dev/null +++ b/config/localmode/manager_config_patch.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - image: ghcr.io/sovereigncloudstack/cso-staging:dev + name: manager + args: + - --leader-elect=true + - --release-dir=/tmp + - --local=true diff --git a/config/localmode/manager_pull_policy.yaml b/config/localmode/manager_pull_policy.yaml new file mode 100644 index 000000000..74a0879c6 --- /dev/null +++ b/config/localmode/manager_pull_policy.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Always diff --git a/config/localmode/manager_webhook_patch.yaml b/config/localmode/manager_webhook_patch.yaml new file mode 100644 index 000000000..a81169da6 --- /dev/null +++ b/config/localmode/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: cso-webhook-server-cert diff --git a/config/localmode/webhookcainjection_patch.yaml b/config/localmode/webhookcainjection_patch.yaml new file mode 100644 index 000000000..f5e23673e --- /dev/null +++ b/config/localmode/webhookcainjection_patch.yaml @@ -0,0 +1,9 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/docs/topics/using-local-clusterstacks.md b/docs/topics/using-local-clusterstacks.md new file mode 100644 index 000000000..2d953121f --- /dev/null +++ b/docs/topics/using-local-clusterstacks.md @@ -0,0 +1,133 @@ +This is a quickstart guide on using the local clusterstacks. + +Requirements: +- docker +- kind +- make + +### Introduction +Our controller interacts with cluster-stacks and they are stored inside GitHub releases when used in production. While this is the ideal combination to run our controller it's not a great experience when we want to develop or test new cluster-stacks. +To avoid this problem, we introduced `local_mode` and in this mode we start our controller by passing some flags and instead of using remote cluster-stacks, we put them inside a directory locally. +This way developers working on developing new cluster-stacks get a more better experience developing cluster-stacks and iterating fast on the same. + +### using right flags +Cluster stacks in local_mode do not fetch release assets directly from GitHub and to work with that we first need to download that and use the correct flags when we start the container. +The flags we are using in this repo is following: +```bash + args: + - --leader-elect=true + - --release-dir=/tmp + - --local=true +``` +What this means is that, the controller will look for release assets under /tmp directory inside the contanier. If the controller doesn't find release asset under `/tmp` directory then you can get an error in the status of `clusterstackrelease` object. + +### getting release assets + +There are several ways to get release assets. Two common ways: + +* From a Github release +* From a directory created by [csmctl](https://github.com/SovereignCloudStack/csmctl) + +Overall it does not matter who you received the directory. + +Tilt expects the release assets under `.release` directory of the repository. + +#### getting release assets: From Github + +To download a release asset from Github use the following commands. + +```bash +mkdir -p .release/docker-ferrol-1-27-v2 +cd .release/docker-ferrol-1-27-v2 +gh release download -R sovereigncloudstack/cluster-stacks docker-ferrol-1-27-v2 +``` +You can also fetch the release asset manually. Using `gh` makes it a little bit easier to download all the release asset just by invoking one command. + +Your local `.release` directory should have the following structure. +```bash +$ tree .release/ +.release/ +└── docker-ferrol-1-27-v2 + ├── cluster-addon-values.yaml + ├── docker-ferrol-1-27-cluster-addon-v2.tgz + ├── docker-ferrol-1-27-cluster-class-v2.tgz + ├── metadata.yaml + ├── node-images.yaml + └── topology-docker.yaml + +2 directories, 6 files +``` + +*NOTE: This directory structure is very important and you should have the same in order for the controller to work and sync.* + +#### getting release assets: From csmctl + +If you use [csmctl](https://github.com/SovereignCloudStack/csmctl), then it creates the required directory under `release`. + +Example: + +``` +❯ csmctl create tests/cluster-stacks/docker/ferrol -m hash +Created releases/docker-ferrol-1-27-v0-sha-7ff9188 +``` + +Copy this directory to .release: + +``` +cp -a releases/docker-ferrol-1-27-v0-sha-7ff9188 ../cluster-stack-operator/.release +``` + + +### using local mode (toggling local mode) +From a user perspective who is using tilt to develop and test local cluster-stacks, we just have to make one change in Tiltfile and restart the complete setup all over again. + +For making changes in tilt environment, we use `tilt-settings.yaml` file. If you don't have it then please copy it from `tilt-settings.yaml.example` which is at the root of the repo using the following command. +```bash +cp tilt-settings.yaml.example tilt-settings.yaml +``` +Now, since you have your `tilt-settings.yaml` ready, you have to edit `local_mode` and make it to `true` +Your `tilt-settings.yaml` file should look following: +```yaml +allowed_contexts: + - kind-cso +local_mode: false +# truncated +``` + +Once you're toggled `local_mode` to `true`, you're ready to start your tilt setup. Use the following command to start it. +```bash +make tilt-up +``` +Once the setup is ready and you've the cluster-stack-operator pod running, you can exec into the container and check the `/tmp` directory for the presence of cluster-stacks. + +### Troubleshooting + +```bash +$ kubectl -n cso-system exec -it cso-controller-manager-5bdc445647-j4p9p -- sh +/ # tree /tmp/ +/tmp/ +└── cluster-stacks + └── docker-ferrol-1-27-v2 + ├── cluster-addon-values.yaml + ├── docker-ferrol-1-27-cluster-addon-v2.tgz + ├── docker-ferrol-1-27-cluster-class-v2.tgz + ├── metadata.yaml + ├── node-images.yaml + └── topology-docker.yaml + +2 directories, 6 files +``` +You can now click on `+` button on the right hand side of tilt UI to create a workload cluster. Once the cluster is created you can check the status of the components via the following commands. +```bash + $ kubectl get clusterstack -A +NAMESPACE NAME PROVIDER CLUSTERSTACK K8S CHANNEL AUTOSUBSCRIBE USABLE LATEST AGE REASON MESSAGE +cluster clusterstack docker ferrol 1.27 stable false v2 docker-ferrol-1-27-v2 | v1.27.3 5m16s + + $ kubectl get clusterstackreleases.clusterstack.x-k8s.io -A +NAMESPACE NAME K8S VERSION READY AGE REASON MESSAGE +cluster docker-ferrol-1-27-v2 v1.27.3 true 5m16s + + $ kubectl get clusters -A +NAMESPACE NAME CLUSTERCLASS PHASE AGE VERSION +cluster test-dfkhje docker-ferrol-1-27-v2 Provisioned 2m46s v1.27.3 +``` diff --git a/hack/version.sh b/hack/version.sh index 1b7d0b6fb..b2c5b4384 100755 --- a/hack/version.sh +++ b/hack/version.sh @@ -64,7 +64,7 @@ version::get_version_vars() { fi fi - GIT_RELEASE_TAG=$(git describe --abbrev=0 --tags) + GIT_RELEASE_TAG=$(git describe --abbrev=0 --tags 2>/dev/null) } # borrowed from k8s.io/hack/lib/version.sh and modified diff --git a/pkg/github/client/fake/github_client.go b/pkg/github/client/fake/github_client.go new file mode 100644 index 000000000..90e0bd098 --- /dev/null +++ b/pkg/github/client/fake/github_client.go @@ -0,0 +1,69 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package fake defines a fake Gitub client. +package fake + +import ( + "context" + "net/http" + + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/github/client" + "github.com/go-logr/logr" + "github.com/google/go-github/v52/github" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type fakeClient struct { + log logr.Logger +} + +type factory struct{} + +var _ = client.Client(&fakeClient{}) + +var _ = client.Factory(&factory{}) + +// NewFactory returns a new factory for Github clients. +func NewFactory() client.Factory { + return &factory{} +} + +func (*factory) NewClient(ctx context.Context) (client.Client, error) { + logger := log.FromContext(ctx) + + return &fakeClient{ + log: logger, + }, nil +} + +func (c *fakeClient) ListRelease(_ context.Context) ([]*github.RepositoryRelease, *github.Response, error) { + c.log.Info("WARNING: called ListRelease of fake Github client") + resp := &github.Response{Response: &http.Response{StatusCode: http.StatusOK}} + return nil, resp, nil +} + +func (c *fakeClient) GetReleaseByTag(_ context.Context, _ string) (*github.RepositoryRelease, *github.Response, error) { + c.log.Info("WARNING: called GetReleaseByTag of fake Github client") + resp := &github.Response{Response: &http.Response{StatusCode: http.StatusOK}} + return nil, resp, nil +} + +// DownloadReleaseAssets downloads a list of release assets. +func (c *fakeClient) DownloadReleaseAssets(_ context.Context, _ *github.RepositoryRelease, _ string, _ []string) error { + c.log.Info("WARNING: called DownloadReleaseAssets of fake Github client") + return nil +} diff --git a/tilt-settings.yaml.example b/tilt-settings.yaml.example new file mode 100644 index 000000000..2165ea68e --- /dev/null +++ b/tilt-settings.yaml.example @@ -0,0 +1,10 @@ +allowed_contexts: + - kind-cso +# change `local_mode` to true if you want to use local cluster-stacks. +local_mode: false +deploy_cert_manager: true +deploy_observability: false +preload_images_for_kind: true +kind_cluster_name: cso +capi_version: v1.6.0 +cert_manager_version: v1.13.0