From 9213669b88e5c9cd46670c6f8d6a580b66c316d0 Mon Sep 17 00:00:00 2001 From: Roke Jung Date: Fri, 3 Jun 2022 15:34:30 -0400 Subject: [PATCH 1/3] Update baremetal worker doc Signed-off-by: Roke Jung --- docs/provision_hypershift_clusters_by_mce.md | 196 +++++++++++++++++-- 1 file changed, 182 insertions(+), 14 deletions(-) diff --git a/docs/provision_hypershift_clusters_by_mce.md b/docs/provision_hypershift_clusters_by_mce.md index adea8e6..c6a8d7d 100644 --- a/docs/provision_hypershift_clusters_by_mce.md +++ b/docs/provision_hypershift_clusters_by_mce.md @@ -129,31 +129,199 @@ Upon scaling up a NodePool, a Machine will be created, and the CAPI provider wil Upon scaling down a NodePool, Agents will be unbound from the corresponding cluster. However, you must boot them with the Discovery Image once again before reusing them. -To use the Agent platform, the Infrastructure Operator must first be installed. Please see [here](https://hypershift-docs.netlify.app/how-to/agent/create-agent-cluster/) for details. +To use the Agent platform, the Infrastructure Operator must first be installed. Please see [here](https://hypershift-docs.netlify.app/how-to/agent/create-agent-cluster/) for details or you can enable it through the multiclusterengine resource. -When creating the HostedCluster resource, set spec.platform.type to "Agent" and spec.platform.agent.agentNamespace to the namespace containing the Agent CRs you would like to use. For NodePools, set spec.platform.type to "Agent", and optionally specify a label selector for selecting the Agent CRs to in spec.platform.agent.agentLabelSelector. +###### Enable assisted service on hosting cluster -The HypershiftDeployment would look like: +1. Create two persistent volumes for assisted service. +- `Capacity`: 10Gi +- `Access modes`: ReadWriteOnce +- `Volume mode`: Filesystem +- `StorageClass`: None + +2. Enable the Infrastructure Operator. ```bash -$ oc apply -f - < --type=merge -p '{"spec":{"overrides":{"components":[{"name":"assisted-service","enabled": true}]}}}' +``` + +3. Create the agentserviceconfig object. Double check the `ISO_URL` at https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${OCP_VERSION}/latest. +```bash +export DB_VOLUME_SIZE="10Gi" +export FS_VOLUME_SIZE="10Gi" +export OCP_VERSION="4.10" +export ARCH="x86_64" +export OCP_RELEASE_VERSION=$(curl -s https://mirror.openshift.com/pub/openshift-v4/${ARCH}/clients/ocp/latest-${OCP_VERSION}/release.txt | awk '/machine-os / { print $2 }') +export ISO_URL="https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${OCP_VERSION}/latest/rhcos-${OCP_VERSION}.3-${ARCH}-live.${ARCH}.iso" +export ROOT_FS_URL="https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${OCP_VERSION}/latest/rhcos-live-rootfs.${ARCH}.img" + +envsubst <<"EOF" | oc apply -f - +apiVersion: agent-install.openshift.io/v1beta1 +kind: AgentServiceConfig metadata: - name: hypershift-demo + name: agent +spec: + databaseStorage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${DB_VOLUME_SIZE} + filesystemStorage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${FS_VOLUME_SIZE} + osImages: + - openshiftVersion: "${OCP_VERSION}" + version: "${OCP_RELEASE_VERSION}" + url: "${ISO_URL}" + rootFSUrl: "${ROOT_FS_URL}" + cpuArchitecture: "${ARCH}" +EOF +``` + +4. Wait for the assisted-service pod to be ready. +```bash +until oc wait -n multicluster-engine $(oc get pods -n multicluster-engine -l app=assisted-service -o name) --for condition=Ready --timeout 10s >/dev/null 2>&1 ; do sleep 1 ; done +``` + +###### Create bare metal host and agent to be used as a worker node on hosting cluster + +The number of `BareMetalHost` resources should match the `agent` namespace should match the number of replica in `NodePool`. Follow https://github.com/openshift/hypershift/blob/main/docs/content/how-to/agent/create-agent-cluster.md#adding-a-bare-metal-worker for creating `BareMetalHost` and `agent` resources. Stop when `agent` resources are created. Skip updating the nodepool part of the documentation. Note the namespce for the `agent` resources. This namespace will be used as `agentNamespace` in `HostedCluster` resource in the next section. + + +###### Provision a hosted cluster + +Create `HostedCluster` and `NodePool` on the MCE cluster. These will be referenced by `HypershiftDeployment` to provision the hosted cluster on the target hosting cluster. We are going to create the `HostedCluster`, `NodePool` and `HypershiftDeployment` all in `default` namespace on the MCE cluster. On the hosting cluster, hypershift deployment will create `HostedCluster` and `NodePool` in `clusters` namespace. + +1. Create SSH key secret for `HostedCluster`. +```bash +envsubst <<"EOF" | oc apply -f - +apiVersion: v1 +kind: Secret +metadata: + name: agent-demo-ssh-key + namespace: default +stringData: + id_rsa.pub: +EOF +``` + +2. Create pull secret for `HostedCluster`. +```bash +export PS64=$(echo -n | base64 -w0) +envsubst <<"EOF" | oc apply -f - +apiVersion: v1 +data: + .dockerconfigjson: ${PS64} +kind: Secret +metadata: + name: agent-demo-pull-secret + namespace: default +type: kubernetes.io/dockerconfigjson +EOF +``` + +3. Create `HostedCluster`. +```bash +apiVersion: hypershift.openshift.io/v1alpha1 +kind: HostedCluster +metadata: + name: agent-demo namespace: default spec: - hostingCluster: hypershift-management-cluster # the hypershift management cluster name. - hostingNamespace: clusters # specify the namespace to which hostedcluster and noodpools belong on the hypershift management cluster. - infrastructure: - configure: True - platform: + dns: + baseDomain: + infraID: agent-demo + networking: + machineCIDR: "" + networkType: OpenShiftSDN + podCIDR: 10.132.0.0/14 + serviceCIDR: 172.32.0.0/16 platform: agent: - agentNamespace: ${AGENT_NS} + agentNamespace: type: Agent -EOF + pullSecret: + name: agent-demo-pull-secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.10.16-x86_64 + services: + - service: APIServer + servicePublishingStrategy: + nodePort: + address: + type: NodePort + - service: OAuthServer + servicePublishingStrategy: + nodePort: + address: + type: NodePort + - service: OIDC + servicePublishingStrategy: + nodePort: + address: + type: None + - service: Konnectivity + servicePublishingStrategy: + nodePort: + address: + type: NodePort + - service: Ignition + servicePublishingStrategy: + nodePort: + address: + type: NodePort + sshKey: + name: agent-demo-ssh-key ``` +4. Create `NodePool`. +```bash +apiVersion: hypershift.openshift.io/v1alpha1 +kind: NodePool +metadata: + name: agent-demo + namespace: default +spec: + clusterName: agent-demo + management: + autoRepair: false + replace: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + strategy: RollingUpdate + upgradeType: Replace + platform: + type: Agent + release: + image: quay.io/openshift-release-dev/ocp-release:4.10.16-x86_64 + replicas: 1 +``` + +5. Create `HypershiftDeployment` which references these `HostedCluster` and `NodePool`. +```bash +apiVersion: cluster.open-cluster-management.io/v1alpha1 +kind: HypershiftDeployment +metadata: + name: agent-demo + namespace: default +spec: + hostingCluster: + hostingNamespace: clusters + infrastructure: + configure: false + hostedClusterReference: + name: agent-demo + nodePoolReferences: + - name: agent-demo +``` + +6. Apply the `HypershiftDeployment` to provision the hosted cluster on the hosting cluster. + + ## Access the hosted cluster The access secrets are stored in the {hypershift-management-cluster} namespace. From 835e0fb1b5bc27ae52611797699ddb370db67f96 Mon Sep 17 00:00:00 2001 From: Roke Jung Date: Fri, 3 Jun 2022 15:50:41 -0400 Subject: [PATCH 2/3] Update baremetal worker doc Signed-off-by: Roke Jung --- docs/provision_hypershift_clusters_by_mce.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/provision_hypershift_clusters_by_mce.md b/docs/provision_hypershift_clusters_by_mce.md index c6a8d7d..06dd004 100644 --- a/docs/provision_hypershift_clusters_by_mce.md +++ b/docs/provision_hypershift_clusters_by_mce.md @@ -195,6 +195,9 @@ The number of `BareMetalHost` resources should match the `agent` namespace shoul Create `HostedCluster` and `NodePool` on the MCE cluster. These will be referenced by `HypershiftDeployment` to provision the hosted cluster on the target hosting cluster. We are going to create the `HostedCluster`, `NodePool` and `HypershiftDeployment` all in `default` namespace on the MCE cluster. On the hosting cluster, hypershift deployment will create `HostedCluster` and `NodePool` in `clusters` namespace. +**Note: If you are provisioning this hosted cluster on `local-cluster` hosting cluster, do not create `HostedCluster` and `NodePool` resources and reference them because the hypershift operator on MCE cluster for `local-cluster` hosting cluster will reconcile them to create a hosted cluster. Instead, use HypershiftDeployment `spec.hostedClusterSpec` and `spec.nodePools`. + + 1. Create SSH key secret for `HostedCluster`. ```bash envsubst <<"EOF" | oc apply -f - From 25fc0afd79300f51c1ded009e79ca59569edb49f Mon Sep 17 00:00:00 2001 From: Roke Jung Date: Mon, 6 Jun 2022 15:36:12 -0400 Subject: [PATCH 3/3] update more doc Signed-off-by: Roke Jung --- docs/provision_hypershift_clusters_by_mce.md | 92 +++++++++++++++----- 1 file changed, 71 insertions(+), 21 deletions(-) diff --git a/docs/provision_hypershift_clusters_by_mce.md b/docs/provision_hypershift_clusters_by_mce.md index 06dd004..5a354bc 100644 --- a/docs/provision_hypershift_clusters_by_mce.md +++ b/docs/provision_hypershift_clusters_by_mce.md @@ -129,9 +129,11 @@ Upon scaling up a NodePool, a Machine will be created, and the CAPI provider wil Upon scaling down a NodePool, Agents will be unbound from the corresponding cluster. However, you must boot them with the Discovery Image once again before reusing them. -To use the Agent platform, the Infrastructure Operator must first be installed. Please see [here](https://hypershift-docs.netlify.app/how-to/agent/create-agent-cluster/) for details or you can enable it through the multiclusterengine resource. +To use the Agent platform, the Assisted Service component must be enabled in the multiclusterengine resource on MCE or ACM hub cluster to install the infrastructure operator. Then infrastructure environment and bare metal host agents need to be configured prior to provisioning a hosted cluster. It is recommended to use the `local-cluster` managed cluster on MCE/ACM hub cluster as the hosting cluster so that all agent platform information is available to MCE/ACM hub cluster. -###### Enable assisted service on hosting cluster +If you want to use other MCE/ACM managed cluster as the hosting cluster, Infrastructure Operator must first be installed on the managed cluster. Please see [here](https://hypershift-docs.netlify.app/how-to/agent/create-agent-cluster/) for details. Then infrastructure environment and bare metal host agents need to be configured on the cluster prior to provisioning a hosted cluster. + +###### Enable assisted service on hosting cluster on MCE/ACM hub cluster 1. Create two persistent volumes for assisted service. - `Capacity`: 10Gi @@ -191,7 +193,7 @@ until oc wait -n multicluster-engine $(oc get pods -n multicluster-engine -l app The number of `BareMetalHost` resources should match the `agent` namespace should match the number of replica in `NodePool`. Follow https://github.com/openshift/hypershift/blob/main/docs/content/how-to/agent/create-agent-cluster.md#adding-a-bare-metal-worker for creating `BareMetalHost` and `agent` resources. Stop when `agent` resources are created. Skip updating the nodepool part of the documentation. Note the namespce for the `agent` resources. This namespace will be used as `agentNamespace` in `HostedCluster` resource in the next section. -###### Provision a hosted cluster +###### Provision a hosted cluster on local-cluster hosting cluster (MCE/ACM hub cluster) Create `HostedCluster` and `NodePool` on the MCE cluster. These will be referenced by `HypershiftDeployment` to provision the hosted cluster on the target hosting cluster. We are going to create the `HostedCluster`, `NodePool` and `HypershiftDeployment` all in `default` namespace on the MCE cluster. On the hosting cluster, hypershift deployment will create `HostedCluster` and `NodePool` in `clusters` namespace. @@ -226,14 +228,8 @@ type: kubernetes.io/dockerconfigjson EOF ``` -3. Create `HostedCluster`. +3. Prepare `HostedCluster` spec. ```bash -apiVersion: hypershift.openshift.io/v1alpha1 -kind: HostedCluster -metadata: - name: agent-demo - namespace: default -spec: dns: baseDomain: infraID: agent-demo @@ -280,13 +276,9 @@ spec: name: agent-demo-ssh-key ``` -4. Create `NodePool`. +4. Prepare one or more `NodePool` specs. ```bash -apiVersion: hypershift.openshift.io/v1alpha1 -kind: NodePool -metadata: - name: agent-demo - namespace: default +name: nodepool1 spec: clusterName: agent-demo management: @@ -304,7 +296,7 @@ spec: replicas: 1 ``` -5. Create `HypershiftDeployment` which references these `HostedCluster` and `NodePool`. +5. Create `HypershiftDeployment`. Use the `HostedCluster` spec from step 3 and the `NodePool` specs from step 4 and insert them into `spec.hostedClusterSpec` and `spec.NodePools`. ```bash apiVersion: cluster.open-cluster-management.io/v1alpha1 kind: HypershiftDeployment @@ -316,10 +308,68 @@ spec: hostingNamespace: clusters infrastructure: configure: false - hostedClusterReference: - name: agent-demo - nodePoolReferences: - - name: agent-demo + hostedClusterSpec: + dns: + baseDomain: + infraID: agent-demo + networking: + machineCIDR: "" + networkType: OpenShiftSDN + podCIDR: 10.132.0.0/14 + serviceCIDR: 172.32.0.0/16 + platform: + agent: + agentNamespace: + type: Agent + pullSecret: + name: agent-demo-pull-secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.10.16-x86_64 + services: + - service: APIServer + servicePublishingStrategy: + nodePort: + address: + type: NodePort + - service: OAuthServer + servicePublishingStrategy: + nodePort: + address: + type: NodePort + - service: OIDC + servicePublishingStrategy: + nodePort: + address: + type: None + - service: Konnectivity + servicePublishingStrategy: + nodePort: + address: + type: NodePort + - service: Ignition + servicePublishingStrategy: + nodePort: + address: + type: NodePort + sshKey: + name: agent-demo-ssh-key + nodePools: + - name: nodepool1 + spec: + clusterName: agent-demo + management: + autoRepair: false + replace: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + strategy: RollingUpdate + upgradeType: Replace + platform: + type: Agent + release: + image: quay.io/openshift-release-dev/ocp-release:4.10.16-x86_64 + replicas: 1 ``` 6. Apply the `HypershiftDeployment` to provision the hosted cluster on the hosting cluster.