feat: add GCP ccm

Update docs to reflect deploying GCP Cloud Control Manager (CCM) Signed-off-by: Noel Georgi <[email protected]>
siderolabs · Nov 10, 2021 · fa4c673 · fa4c673
1 parent e6f90bb
commit fa4c673
Show file tree

Hide file tree

Showing 3 changed files with 183 additions and 14 deletions.
diff --git a/website/content/docs/v0.14/Cloud Platforms/gcp.md b/website/content/docs/v0.14/Cloud Platforms/gcp.md
@@ -273,6 +273,7 @@ resources:
     properties:
       zone: us-west2-c
       talosVersion: v0.13.2
+      externalCloudProvider: false
       controlPlaneNodeCount: 5
       controlPlaneNodeType: n1-standard-1
       workerNodeCount: 3
@@ -282,6 +283,14 @@ outputs:
     value: $(ref.talos-ha.bucketName)
 ```
 
+#### Enabling external cloud provider
+
+Note: The `externalCloudProvider` property is set to `false` by default.
+The [manifest](https://gist.github.com/frezbo/0c1fab2618b1ef5713365444d1cbf75c#file-ccm-gcp-yaml-L222) used for deploying the ccm (cloud controller manager) is currently using a personal tagged image since there are no public images for the [ccm](https://github.com/kubernetes/cloud-provider-gcp) yet.
+Use a custom built image for the ccm deployment if required and update the link in `talos-ha.jinja`.
+
+If the external cloud provider is enabled, the default service account used by deployment manager needs to have [Role Administrator](https://cloud.google.com/deployment-manager/docs/access-control#access_control_for) role assigned to it.
+
 ### Creating the deployment
 
 Now we are ready to create the deployment.
@@ -303,10 +312,25 @@ First we need to get the deployment outputs.
 OUTPUTS=$(gcloud deployment-manager deployments describe "${DEPLOYMENT_NAME}" --format json | jq '.outputs[]')
 
 BUCKET_NAME=$(jq -r '. | select(.name == "bucketName").finalValue' <<< "${OUTPUTS}")
+# used when cloud controller is enabled
+SERVICE_ACCOUNT=$(jq -r '. | select(.name == "serviceAccount").finalValue' <<< "${OUTPUTS}")
+ROLE=$(jq -r '. | select(.name == "role").finalValue' <<< "${OUTPUTS}")
+PROJECT=$(jq -r '. | select(.name == "project").finalValue' <<< "${OUTPUTS}")
+```
+
+Note: If cloud controller manager is enabled, the below command needs to be run to allow the controller custom role to access cloud resources
+
+```bash
+gcloud projects add-iam-policy-binding \
+  "${PROJECT}" \
+  --member "serviceAccount:${SERVICE_ACCOUNT}" \
+  --role "${ROLE}"
 ```
 
 ### Downloading talos and kube config
 
+In addition to the `talosconfig` and `kubeconfig` files, the storage bucket contains the `controlplane.yaml` and `worker.yaml` files used to join additional nodes to the cluster.
+
 ```bash
 gsutil cp "gs://${BUCKET_NAME}/generated/talosconfig" .
 gsutil cp "gs://${BUCKET_NAME}/generated/kubeconfig" .
@@ -329,3 +353,12 @@ Warning: This will delete the deployment and all resources associated with it.
 gsutil rm -r "gs://${BUCKET_NAME}"
 gcloud deployment-manager deployments delete "${DEPLOYMENT_NAME}"
 ```
+
+Run below if cloud controller manager is enabled
+
+```bash
+gcloud projects delete-iam-policy-binding \
+  "${PROJECT}" \
+  --member "serviceAccount:${SERVICE_ACCOUNT}" \
+  --role "${ROLE}"
+```
diff --git a/website/content/docs/v0.14/Cloud Platforms/gcp/config.yaml b/website/content/docs/v0.14/Cloud Platforms/gcp/config.yaml
@@ -7,10 +7,17 @@ resources:
     properties:
       zone: us-west2-c
       talosVersion: v0.13.2
+      externalCloudProvider: false
       controlPlaneNodeCount: 3
       controlPlaneNodeType: n1-standard-1
       workerNodeCount: 1
       workerNodeType: n1-standard-1
 outputs:
   - name: bucketName
     value: $(ref.talos-ha.bucketName)
+  - name: role
+    value: $(ref.talos-ha.role)
+  - name: serviceAccount
+    value: $(ref.talos-ha.serviceAccount)
+  - name: project
+    value: $(ref.talos-ha.project)
diff --git a/website/content/docs/v0.14/Cloud Platforms/gcp/talos-ha.jinja b/website/content/docs/v0.14/Cloud Platforms/gcp/talos-ha.jinja
@@ -101,12 +101,106 @@ resources:
     - IPProtocol: TCP
       ports:
       - 50000
+{% if properties["externalCloudProvider"] %}
+- type: gcp-types/iam-v1:projects.roles
+  name: {{ env["deployment"] }}-cloud-controller-manager-role
+  properties:
+    parent: projects/{{ env["project"] }}
+    roleId: {{ env["deployment"] | replace("-", "_") }}_ccm_role
+    role:
+      title: Cloud Controller Manager
+      description: Cloud Controller Manager role
+      includedPermissions:
+        - compute.addresses.create
+        - compute.addresses.createInternal
+        - compute.addresses.delete
+        - compute.addresses.deleteInternal
+        - compute.addresses.get
+        - compute.addresses.list
+        - compute.addresses.setLabels
+        - compute.addresses.use
+        - compute.addresses.useInternal
+        - compute.firewalls.create
+        - compute.firewalls.delete
+        - compute.firewalls.get
+        - compute.firewalls.list
+        - compute.firewalls.update
+        - compute.forwardingRules.create
+        - compute.forwardingRules.delete
+        - compute.forwardingRules.get
+        - compute.forwardingRules.list
+        - compute.forwardingRules.pscCreate
+        - compute.forwardingRules.pscDelete
+        - compute.forwardingRules.pscSetLabels
+        - compute.forwardingRules.pscSetTarget
+        - compute.forwardingRules.pscUpdate
+        - compute.forwardingRules.setLabels
+        - compute.forwardingRules.setTarget
+        - compute.forwardingRules.update
+        - compute.forwardingRules.use
+        - compute.healthChecks.create
+        - compute.healthChecks.delete
+        - compute.healthChecks.get
+        - compute.healthChecks.list
+        - compute.healthChecks.update
+        - compute.healthChecks.use
+        - compute.healthChecks.useReadOnly
+        - compute.httpHealthChecks.create
+        - compute.httpHealthChecks.create
+        - compute.httpHealthChecks.delete
+        - compute.httpHealthChecks.delete
+        - compute.httpHealthChecks.get
+        - compute.httpHealthChecks.get
+        - compute.httpHealthChecks.list
+        - compute.httpHealthChecks.list
+        - compute.httpHealthChecks.update
+        - compute.httpHealthChecks.update
+        - compute.httpHealthChecks.use
+        - compute.httpHealthChecks.use
+        - compute.httpHealthChecks.useReadOnly
+        - compute.httpsHealthChecks.create
+        - compute.httpsHealthChecks.delete
+        - compute.httpsHealthChecks.get
+        - compute.httpsHealthChecks.list
+        - compute.httpsHealthChecks.update
+        - compute.httpsHealthChecks.use
+        - compute.httpsHealthChecks.useReadOnly
+        - compute.instances.get
+        - compute.instances.list
+        - compute.instances.use
+        - compute.networks.updatePolicy
+        - compute.routes.create
+        - compute.routes.delete
+        - compute.routes.get
+        - compute.routes.list
+        - compute.targetPools.addHealthCheck
+        - compute.targetPools.addInstance
+        - compute.targetPools.create
+        - compute.targetPools.delete
+        - compute.targetPools.get
+        - compute.targetPools.list
+        - compute.targetPools.removeHealthCheck
+        - compute.targetPools.removeInstance
+        - compute.targetPools.update
+        - compute.targetPools.use
+- type: gcp-types/iam-v1:projects.serviceAccounts
+  name: {{ env["deployment"] }}-ccm-sa
+  properties:
+    displayName: Cloud Controller Manager
+    accountId: {{ env["deployment"] }}-ccm-sa
+{% endif %}
 {% for index in range(properties["controlPlaneNodeCount"]) %}
 - type: compute.v1.instance
   name: {{ env["deployment"] }}-talos-controlplane-{{ index }}
   properties:
     zone: {{ properties["zone"] }}
     machineType: zones/{{ properties["zone"] }}/machineTypes/{{ properties["controlPlaneNodeType"] }}
+{% if properties["externalCloudProvider"] %}
+    serviceAccounts:
+    - email: $(ref.{{ env["deployment"] }}-ccm-sa.email)
+      scopes:
+      - https://www.googleapis.com/auth/compute
+{% endif %}
     tags:
       items:
       - talos-controlplane
@@ -131,6 +225,12 @@ resources:
   properties:
     zone: {{ properties["zone"] }}
     machineType: zones/{{ properties["zone"] }}/machineTypes/{{ properties["workerNodeType"] }}
+{% if properties["externalCloudProvider"] %}
+    serviceAccounts:
+    - email: $(ref.{{ env["deployment"] }}-ccm-sa.email)
+      scopes:
+      - https://www.googleapis.com/auth/compute
+{% endif %}
     tags:
       items:
       - talos-workers
@@ -174,20 +274,29 @@ resources:
         - /bin/sh
         - -ec
         - |
-          chmod +x talosctl-linux-amd64
-          mv talosctl-linux-amd64 /usr/local/bin/talosctl
-          mkdir -p generated
+          chmod +x talosctl-linux-amd64 && \
+          mv talosctl-linux-amd64 /usr/local/bin/talosctl && \
+          mkdir -p generated && \
           talosctl gen config \
             {{ env["deployment"] }} \
             https://$(ref.{{ env["deployment"] }}-talos-lb-ip.address):443 \
-            --output-dir generated/
+            {% if properties["externalCloudProvider"] %} --config-patch '[{"op": "add", "path": "/cluster/externalCloudProvider", "value": {"enabled": true, "manifests": ["https://gist.githubusercontent.com/frezbo/0c1fab2618b1ef5713365444d1cbf75c/raw/5f19a0a8e2d46481bb0ca9b98384c43285dba980/ccm-gcp.yaml"]}}]' \{% endif %}
+            --output-dir generated/ && \
 {% for index in range(properties["controlPlaneNodeCount"]) %}
-          echo "applying config for {{ env["deployment"] }}-talos-controlplane-{{ index }}"
-          talosctl apply-config --insecure --nodes $(ref.{{ env["deployment"] }}-talos-controlplane-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP) --endpoints $(ref.{{ env["deployment"] }}-talos-controlplane-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP) --file generated/controlplane.yaml
+          echo "applying config for {{ env["deployment"] }}-talos-controlplane-{{ index }}" && \
+          talosctl apply-config \
+            --insecure \
+            --nodes $(ref.{{ env["deployment"] }}-talos-controlplane-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP) \
+            --endpoints $(ref.{{ env["deployment"] }}-talos-controlplane-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP) \
+            --file generated/controlplane.yaml && \
 {% endfor %}
 {%  for index in range(properties["workerNodeCount"]) %}
-          echo "applying config for {{ env["deployment"] }}-talos-worker-{{ index }}"
-          talosctl apply-config --insecure --nodes $(ref.{{ env["deployment"] }}-talos-worker-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP) --endpoints $(ref.{{ env["deployment"] }}-talos-worker-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP) --file generated/worker.yaml
+          echo "applying config for {{ env["deployment"] }}-talos-worker-{{ index }}" && \
+          talosctl apply-config \
+            --insecure \
+            --nodes $(ref.{{ env["deployment"] }}-talos-worker-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP) \
+            --endpoints $(ref.{{ env["deployment"] }}-talos-worker-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP) \
+            --file generated/worker.yaml && \
 {% endfor %}
           # wait before bootstrapping
           wait_count=100
@@ -198,19 +307,39 @@ resources:
               echo "Timeout waiting for talos-controlplane-0 to be ready for bootstrap"
               exit 1
             fi
-          done
-          talosctl --talosconfig generated/talosconfig --nodes $(ref.{{ env["deployment"] }}-talos-controlplane-0.networkInterfaces[0].accessConfigs[0].natIP) --endpoints $(ref.{{ env["deployment"] }}-talos-controlplane-0.networkInterfaces[0].accessConfigs[0].natIP) bootstrap
-          talosctl --talosconfig generated/talosconfig --nodes $(ref.{{ env["deployment"] }}-talos-controlplane-0.networkInterfaces[0].accessConfigs[0].natIP) --endpoints $(ref.{{ env["deployment"] }}-talos-controlplane-0.networkInterfaces[0].accessConfigs[0].natIP) kubeconfig generated/
-          talosctl --talosconfig generated/talosconfig config endpoint {% for index in range(properties["controlPlaneNodeCount"]) %}$(ref.{{ env["deployment"] }}-talos-controlplane-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP){% if not loop.last %} {% endif %}{% endfor %} {% for index in range(properties["workerNodeCount"]) %}$(ref.{{ env["deployment"] }}-talos-worker-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP){% if not loop.last %} {% endif %}{% endfor %}
-          talosctl --talosconfig generated/talosconfig config node {% for index in range(properties["controlPlaneNodeCount"]) %}$(ref.{{ env["deployment"] }}-talos-controlplane-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP){% if not loop.last %} {% endif %}{% endfor %} {% for index in range(properties["workerNodeCount"]) %}$(ref.{{ env["deployment"] }}-talos-worker-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP){% if not loop.last %} {% endif %}{% endfor %}
+          done && \
+          talosctl \
+            --talosconfig generated/talosconfig \
+            --nodes $(ref.{{ env["deployment"] }}-talos-controlplane-0.networkInterfaces[0].accessConfigs[0].natIP) \
+            --endpoints $(ref.{{ env["deployment"] }}-talos-controlplane-0.networkInterfaces[0].accessConfigs[0].natIP) \
+            bootstrap && \
+          talosctl \
+            --talosconfig generated/talosconfig \
+            --nodes $(ref.{{ env["deployment"] }}-talos-controlplane-0.networkInterfaces[0].accessConfigs[0].natIP) \
+            --endpoints $(ref.{{ env["deployment"] }}-talos-controlplane-0.networkInterfaces[0].accessConfigs[0].natIP) \
+            kubeconfig generated/ && \
+          talosctl \
+            --talosconfig generated/talosconfig \
+            config endpoint \
+            {% for index in range(properties["controlPlaneNodeCount"]) %}$(ref.{{ env["deployment"] }}-talos-controlplane-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP){% if not loop.last %} {% endif %}{% endfor %} {% for index in range(properties["workerNodeCount"]) %}$(ref.{{ env["deployment"] }}-talos-worker-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP){% if not loop.last %} {% endif %}{% endfor %} && \
+          talosctl \
+            --talosconfig generated/talosconfig \
+            config node \
+            {% for index in range(properties["controlPlaneNodeCount"]) %}$(ref.{{ env["deployment"] }}-talos-controlplane-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP){% if not loop.last %} {% endif %}{% endfor %} {% for index in range(properties["workerNodeCount"]) %}$(ref.{{ env["deployment"] }}-talos-worker-{{ index }}.networkInterfaces[0].accessConfigs[0].natIP){% if not loop.last %} {% endif %}{% endfor %}
     - name: gcr.io/cloud-builders/gsutil
       args:
         - -m
         - cp
         - -r
         - generated
         - gs://$(ref.{{ env["deployment"] }}-talos-assets.name)/
-    timeout: 120s
+    timeout: 360s
 outputs:
 - name: bucketName
   value: $(ref.{{ env["deployment"] }}-talos-assets.name)
+- name: role
+  value: $(ref.{{ env["deployment"] }}-cloud-controller-manager-role.name)
+- name: serviceAccount
+  value: $(ref.{{ env["deployment"] }}-ccm-sa.email)
+- name: project
+  value: {{ env["project"] }}