#25: restructure CLI to common launcher, small fix and basic Integrat…

…ionTest doc

doc/IntegrationTests.md

@@ -0,0 +1,33 @@
+# Integration Tests
+
+This file describes Integration Tests, that may get implemented later.
+After we have the priorities assigned to implement the Integration Tests,
+we can check this document to priorize and implement Integratio Tests.
+
+## IT1: Setup cluster with default Configuration
+
+This test will provision a cluster with distinct control plane and linux worker
+nodes.
+
+Ensures:
+- Provisioning runs without errors
+- API Server is listening
+- All nodes are online
+
+## IT1: Sub-Tests
+
+The Sub-Tests are based on the previous setup, runs additional checks:
+
+- Ingress is bound to 'k8s_loadbalancers_ingress_ip'
+- Services can be of type LoadBalancer
+- wd-nginx-test application can be browsed using the ingress-rule
+
+
+# Features considered "untestable"
+
+The features below can't be tested in because of resource availablilty:
+
+## Trident Configuration
+
+Because we have no ontap-nas in the pipeline runner, we can't test Dynamic 
+Volume Provisioning with the Trident configuration.

lib/ansible/roles/lnxclp-first/tasks/main.yml

@@ -3,14 +3,8 @@
 ##
 ## It is dynamically included by cluster.yml playbook
 
-- name: clustersetup - sync ./initk8s init script
-  template:
-    src: initk8s
-    dest: "{{ k8s_setup_dir }}/initk8s"
-    mode: 755
-
-- name: "clustersetup - running init script (you may cat {{ k8s_setup_logdir }}/kubeadm.log to check progress"
-  shell: "{{ k8s_setup_dir }}/initk8s >> {{ k8s_setup_logdir }}/kubeadm.log"
+- name: "clustersetup - running k8s-setup init (you may cat {{ k8s_setup_logdir }}/k8s-setup.log to check progress"
+  shell: "{{ k8s_setup_dir }}/k8s-setup init >> {{ k8s_setup_logdir }}/k8s-setup.log"
 
 - name: clustersetup - fetch admin.conf to provisioning client for later import
   fetch:

lib/ansible/roles/lnxclp-new/tasks/main.yml

@@ -9,17 +9,21 @@
   run_once: true
   register: join_cmd
 
+- name: clustersetup - writing /tmp/k8s-setup-join-cmd
+  copy:
+    dest: /tmp/k8s-setup-join-cmd
+    content: "{{ join_cmd }}"
+
 - name: clustersetup - get certificate-key from lnxclp_setup
   shell: kubeadm init phase upload-certs --upload-certs | tail -n 1
   delegate_to: "{{ groups['lnxclp_setup'] | first }}"
   run_once: true
   register: cert_cmd
 
-- name: clustersetup - sync ./joink8s join script
-  template:
-    src: joink8s
-    dest: "{{ k8s_setup_dir }}/joink8s"
-    mode: 755
+- name: clustersetup - writing /tmp/k8s-setup-join-certificatekey
+  copy:
+    dest: /tmp//tmp/k8s-setup-join-certificatekey
+    content: "{{ cert_cmd }}"
 
-- name: "clustersetup - running join script (you may cat {{ k8s_setup_logdir }}/kubeadm.log to check progress"
-  shell: "{{ k8s_setup_dir }}/joink8s >> {{ k8s_setup_logdir }}/kubeadm.log"
+- name: "clustersetup - running k8s-setup joinclp (you may cat {{ k8s_setup_logdir }}/k8s-setup.log to check progress"
+  shell: "{{ k8s_setup_dir }}/k8s-setup joinclp >> {{ k8s_setup_logdir }}/k8s-setup.log"

lib/ansible/roles/lnxkubeadm-common/tasks/main.yml

@@ -19,24 +19,24 @@
     - "{{ k8s_setup_dir }}"
     - "{{ k8s_setup_logdir }}"
 
-- name: clustersetup - sync ./k8sstate test script
+- name: clustersetup - sync ./k8s-setup script for node operations
   template:
-    src: k8sstate
-    dest: "{{ k8s_setup_dir }}/k8sstate"
+    src: k8s-setup
+    dest: "{{ k8s_setup_dir }}/k8s-setup"
     mode: 755
 
-- name: "clustersetup - run {{ k8s_setup_dir }}/k8sstate to check configuration state"
-  command: "{{ k8s_setup_dir }}/k8sstate"
+- name: clustersetup - sync ./k8s-setup.env script containing configuration values
+  template:
+    src: k8s-setup.env
+    dest: "{{ k8s_setup_dir }}/k8s-setup.env"
+    mode: 755
+
+- name: "clustersetup - running 'k8s-setup state' to check configuration state"
+  command: "{{ k8s_setup_dir }}/k8s-setup state"
   changed_when: false
   register: k8sstate_cmd
 
 - set_fact:
     k8s_configured: "{{ true if 'k8s_configured=yes' in k8sstate_cmd.stdout else false }}"
   no_log: yes
 
-- name: clustersetup - sync ./resetk8s script to teardown configuration for testing"
-  template:
-    src: resetk8s
-    dest: "{{ k8s_setup_dir }}/resetk8s"
-    mode: 755
-

lib/ansible/roles/lnxkubeadm-common/templates/k8s-setup

@@ -0,0 +1,187 @@
+#! /bin/sh
+set -eu
+
+###########################################
+# Provisioning CLI for cluster-nodes
+#
+
+usage()
+{
+    echo "./k8s-setup COMMAND"
+    echo ""
+    echo "Commands:"
+    echo "   state   Evaluates the state of the node"
+    echo "   init    Runs kubeadm init for the first control plane node"
+    echo "   joinwrk Runs kubeadm join for a new worker node"
+    echo "   joinclp Runs kubeadm join for a new control plane node"
+    echo "   reset   Runs kubeadm reset (only for testing!)"
+}
+
+state()
+{
+    CONFFILE=/etc/kubernetes/kubelet.conf
+
+    if [ ! -f "$CONFFILE" ]; then
+        echo "k8s_configured=no"
+        exit 0
+    fi
+
+    EXPECTED_URL='https://$k8s_apiserver_endpoint'
+    CONFIGURED_URL=$(cat $CONFFILE | grep "server: " | awk '{print $2}')
+
+    echo "Configured URL: $CONFIGURED_URL"
+    echo "Expected URL: $EXPECTED_URL"
+
+    if [ "$EXPECTED_URL" == "$CONFIGURED_URL" ]; then
+        echo "k8s_configured=yes"
+        exit 0
+    else
+        >&2 echo "This node is not member of the configured cluster! You may create a new node from scratch. "
+    fi
+}
+
+init()
+{
+
+    # perform the initialization
+    kubeadm init \
+        --control-plane-endpoint $k8s_apiserver_endpoint \
+        --service-dns-domain $k8s_cluster_dnsname \
+        --pod-network-cidr $k8s_pod_network_cidr \
+        --service-cidr $k8s_service_cidr \
+        --apiserver-advertise-address $k8s_apiserver_advertise_address \
+        --kubernetes-version $k8s_version \
+        --apiserver-bind-port 6443 \
+        --upload-certs
+
+    kubeconfig
+
+    exit 0
+}
+
+joinwrk()
+{
+    if [ ! -e "/tmp/k8s-setup-joincmd" ]; then
+        echo "/tmp/k8s-setup-joincmd not found"
+        exit 1
+    fi
+
+    joincmd=$(cat /tmp/k8s-setup-joincmd)
+
+    echo "Running kubeadm join"
+    $joincmd
+
+    kubeconfig
+
+    exit 0
+}
+
+joinclp()
+{
+    if [ ! -e "/tmp/k8s-setup-join-cmd" ]; then
+        echo "/tmp/k8s-setup-joincmd not found"
+        exit 1
+    fi
+
+    if [ ! -e "/tmp/k8s-setup-join-certificatekey" ]; then
+        echo "/tmp/k8s-setup-join-certificatekey"
+        exit 1
+    fi
+
+    joincmd=$(cat /tmp/k8s-setup-joincmd)
+    certificatekey=$(cat /tmp/k8s-setup-join-certificatekey)
+
+    echo "Running kubeadm join"
+    $joincmd \
+        --control-plane \
+        --certificate-key $certificatekey \
+        --apiserver-advertise-address $k8s_apiserver_advertise_address \
+        --apiserver-bind-port 6443
+
+    kubeconfig
+
+    exit 0
+}
+
+reset()
+{
+    echo "Resetting kubernetes"
+    kubeadm reset -f
+
+    # remove host config
+    rm -rf /etc/kubernetes/*
+
+    # and recreate manifests, as these existed before kubeadm init
+    mkdir -p /etc/kubernetes/manifests
+
+    # remove kubeconfig
+    rm -rf ~/.kube
+
+    exit 0
+}
+
+kubeconfig()
+{
+    # copy the admin.conf as kubeconfig to home/.kube
+    mkdir -p $HOME/.kube
+    cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
+    chown $(id -u):$(id -g) $HOME/.kube/config
+    echo 'source <(kubectl completion bash)' >>~/.bashrc
+}
+
+# get sourcedir
+sourcedir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# read variables
+cat $sourcedir/k8s-setup.env | grep -o '^[^#]*' | xargs > /tmp/workaround
+source /tmp/workaround
+rm /tmp/workaround
+
+## TODO: revisit workaround
+## original version:
+# source <(cat $sourcedir/k8s-setup.env | grep -o '^[^#]*' | xargs)
+## This failed with 
+## [root@lnxclp1 k8s-setup]# ./k8s-setup state
+## ./k8s-setup: line 136: syntax error near unexpected token `('
+## But when executed with trace, it doesn't show the syntax error... :-(
+## [root@lnxclp1 k8s-setup]# bash -x ./k8s-setup state
+
+# handle command in $1
+set +u
+command=$1
+set -u
+
+case $command in
+
+    state)
+        state
+        ;;
+
+    init)
+        init
+        ;;
+
+    joinwrk)
+        joinwrk
+        ;;
+
+    joinclp)
+        joinlp;;
+
+    reset)
+        reset
+        exit 0
+        ;;
+
+    *)
+        usage
+        exit 1
+esac
+
+
+###########################################
+# Deletes the local cluster state
+#   
+#   THIS IS FOR TESTING ONLY!
+#   NOT TESTED FOR PRODUCTION USE!
+

lib/ansible/roles/lnxkubeadm-common/templates/k8s-setup.env

@@ -0,0 +1,12 @@
+#!/bin/sh
+###########################################
+# This file is generated by ansible, based on the configured values
+# Don't alter this file directly, because it will be overwritten on provisioning
+
+k8s_version="{{k8s_version}}"
+k8s_cluster_dnsname="{{k8s_cluster_dnsname}}"
+k8s_apiserver_port={{k8s_apiserver_port}}
+k8s_apiserver_endpoint="{{k8s_apiserver_hostname}}.{{k8s_cluster_dnsname}}:{{k8s_apiserver_port}}"
+k8s_pod_network_cidr="{{k8s_pod_network_cidr}}"
+k8s_service_cidr="{{k8s_service_cidr}}"
+k8s_apiserver_advertise_address="{{ ansible_facts[host_primary_interface_name].ipv4.address }}"