Changes for k8s and glusterfs

apps/docker_net/flannel/ansible/group_vars/etcd_server

@@ -1,2 +1,2 @@
-etcd_ip: 192.168.11.244
+etcd_ip: 192.168.1.115
 Backend_Type: vxlan

apps/docker_net/flannel/ansible/hosts

@@ -1,5 +1,5 @@
 [etcd_server]
-192.168.11.244
+192.168.1.115
 
 [flannel_servers]
 192.168.11.232

apps/docker_net/flannel/ansible/site.retry

@@ -0,0 +1 @@
+192.168.1.115

apps/docker_net/flannel/ansible/site.yml.bak

@@ -0,0 +1,22 @@
+---
+- hosts: etcd_server
+
+  roles:
+  - role: etcd
+
+  tasks:
+  - name: copy flannel-config.json
+    template: src=flannel-config.json dest=/tmp/flannel-config.json 
+
+  - name: set flannel-config.json
+    shell: etcdctl --endpoints=http://{{ etcd_ip }}:2379 set /flannel/network/config < flannel-config.json
+    args:
+      chdir: /tmp/
+
+
+- hosts: flannel_servers
+
+  roles:
+  - role: docker
+  - role: flannel
+

apps/glusterfs/README.md

@@ -0,0 +1,18 @@
+* [Introduction](#1)
+* [Benchmark Test](#2)
+* [Others](#3)
+
+# Network Benchmark Test 
+## <a name="1">Introduction</a>
+Gluster is a software defined distributed storage that can scale to several petabytes. It provides interfaces for object, block and file storage.
+
+## <a name="2">Benchmark Test</a>
+### Test Topology
+- `vim ansible/hosts` to change your server and client ip address.
+- `vim ansible/group_vars/` to change variable related to test.
+
+### Setup
+- `./setup.sh` to setup kubernetes in servers.
+
+## <a name="3">Others</a>
+

apps/glusterfs/ansible/hosts

@@ -0,0 +1,4 @@
+[glusterfs]
+192.168.1.86
+192.168.1.115
+192.168.1.171

apps/glusterfs/ansible/roles/glusterfs

@@ -0,0 +1 @@
+../../../../lib/ansible/roles/glusterfs/

apps/glusterfs/ansible/site.yml

@@ -0,0 +1,6 @@
+---
+- hosts: glusterfs
+  roles:
+  - role: glusterfs
+
+

apps/glusterfs/setup.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+CURDIR=$(cd `dirname $0`; pwd)
+
+pushd ${CURDIR}/ansible > /dev/null
+
+ansible-playbook -i hosts site.yml  --user=root --extra-vars "ansible_sudo_pass=root"
+
+popd > /dev/null
+

apps/kubernetes/README.md

@@ -0,0 +1,19 @@
+* [Introduction](#1)
+* [Benchmark Test](#2)
+* [Others](#3)
+
+# Network Benchmark Test 
+## <a name="1">Introduction</a>
+Kubernetes is an open source system for managing containerized applications across multiple hosts, providing basic mechanisms for deployment, maintenance, and scaling of applications.
+
+
+## <a name="2">Benchmark Test</a>
+### Test Topology
+- `vim ansible/hosts` to change your server and client ip address.
+- `vim ansible/group_vars/` to change variable related to test.
+
+### Setup
+- `./setup.sh` to setup kubernetes in servers.
+
+## <a name="3">Others</a>
+

apps/kubernetes/ansible/group_vars/all

@@ -0,0 +1,2 @@
+# arm64 or amd64
+Arch: arm64

apps/kubernetes/ansible/group_vars/etcd_server

@@ -0,0 +1,2 @@
+etcd_ip: 192.168.1.115
+Backend_Type: vxlan

apps/kubernetes/ansible/group_vars/flannel_servers

@@ -0,0 +1,2 @@
+Etcd_Server: 192.168.1.115
+Netcard: eth2

apps/kubernetes/ansible/hosts

@@ -0,0 +1,9 @@
+[etcd_server]
+192.168.1.115
+
+[kubernetes_master]
+192.168.1.86
+
+[kubernetes_minion]
+192.168.1.115
+192.168.1.171

apps/kubernetes/ansible/roles/docker

@@ -0,0 +1 @@
+../../../../lib/ansible/roles/docker/

apps/kubernetes/ansible/roles/etcd

@@ -0,0 +1 @@
+../../../../lib/ansible/roles/docker_net/etcd/

apps/kubernetes/ansible/roles/flannel

@@ -0,0 +1 @@
+../../../../lib/ansible/roles/docker_net/flannel/

apps/kubernetes/ansible/roles/kubernetes_common

@@ -0,0 +1 @@
+../../../../lib/ansible/roles/kubernetes/kubernetes_common

apps/kubernetes/ansible/roles/kubernetes_master

@@ -0,0 +1 @@
+../../../../lib/ansible/roles/kubernetes/kubernetes_master/

apps/kubernetes/ansible/roles/kubernetes_minion

@@ -0,0 +1 @@
+../../../../lib/ansible/roles/kubernetes/kubernetes_minion/

apps/kubernetes/ansible/site.yml

@@ -0,0 +1,23 @@
+---
+#- hosts: etcd_server
+#  roles:
+#  - role: etcd
+
+- hosts:
+  - kubernetes_minion
+  - kubernetes_master
+  roles:
+  - docker
+  - kubernetes_common
+
+- hosts: kubernetes_master
+  roles:
+  - kubernetes_master
+
+- hosts: kubernetes_minion
+  roles:
+  - kubernetes_minion
+
+- hosts: kubernetes_master
+  roles:
+  - kubernetes_glusterfs

apps/kubernetes/setup.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+CURDIR=$(cd `dirname $0`; pwd)
+
+pushd ${CURDIR}/ansible > /dev/null
+
+ansible-playbook -i hosts site.yml  --user=root --extra-vars "ansible_sudo_pass=root"
+
+popd > /dev/null
+

lib/ansible/roles/docker_net/etcd/tasks/install_etcd.yml

@@ -7,13 +7,17 @@
   args:
     chdir: /tmp/
 
-- name: transfer start_etcd.sh
-  template: src=start_etcd.sh dest=/tmp/start_etcd.sh mode=0755
+- name: transfer etcd.service
+  template: src=etcd.service dest=/lib/systemd/system/etcd.service mode=0644
+
+- name: make etcd config dir
+  file: path=/etc/etcd mode=0644 state=directory
+- name: make etcd work dir
+  file: path=/var/lib/etcd mode=0644 state=directory
+
+- name: transfer etcd config
+  template: src=etcd.conf dest=/etc/etcd/etcd.conf mode=0644
 
 - name: start etcd
-  shell: ./start_etcd.sh
-  async: 10
-  poll: 0
-  args:
-    chdir: /tmp/
+  service: name=etcd enabled=yes
 

lib/ansible/roles/docker_net/etcd/templates/etcd.conf

@@ -0,0 +1,47 @@
+# [member]
+ETCD_NAME=default
+ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
+#ETCD_WAL_DIR=""
+#ETCD_SNAPSHOT_COUNT="10000"
+#ETCD_HEARTBEAT_INTERVAL="100"
+#ETCD_ELECTION_TIMEOUT="1000"
+#ETCD_LISTEN_PEER_URLS="http://localhost:2380"
+ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"
+#ETCD_MAX_SNAPSHOTS="5"
+#ETCD_MAX_WALS="5"
+#ETCD_CORS=""
+#
+#[cluster]
+#ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.1.86:2380"
+# if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."
+#ETCD_INITIAL_CLUSTER="default=http://localhost:2380"
+#ETCD_INITIAL_CLUSTER_STATE="new"
+#ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
+ETCD_ADVERTISE_CLIENT_URLS="http://0.0.0.0:2379"
+#ETCD_DISCOVERY=""
+#ETCD_DISCOVERY_SRV=""
+#ETCD_DISCOVERY_FALLBACK="proxy"
+#ETCD_DISCOVERY_PROXY=""
+#
+#[proxy]
+#ETCD_PROXY="off"
+#ETCD_PROXY_FAILURE_WAIT="5000"
+#ETCD_PROXY_REFRESH_INTERVAL="30000"
+#ETCD_PROXY_DIAL_TIMEOUT="1000"
+#ETCD_PROXY_WRITE_TIMEOUT="5000"
+#ETCD_PROXY_READ_TIMEOUT="0"
+#
+#[security]
+#ETCD_CERT_FILE=""
+#ETCD_KEY_FILE=""
+#ETCD_CLIENT_CERT_AUTH="false"
+#ETCD_TRUSTED_CA_FILE=""
+#ETCD_PEER_CERT_FILE=""
+#ETCD_PEER_KEY_FILE=""
+#ETCD_PEER_CLIENT_CERT_AUTH="false"
+#ETCD_PEER_TRUSTED_CA_FILE=""
+#
+#[logging]
+#ETCD_DEBUG="false"
+# examples for -log-package-levels etcdserver=WARNING,security=DEBUG
+#ETCD_LOG_PACKAGE_LEVELS=""

lib/ansible/roles/docker_net/etcd/templates/etcd.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Etcd Server
+After=network.target
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=notify
+WorkingDirectory=/var/lib/etcd/
+EnvironmentFile=-/etc/etcd/etcd.conf
+User=etcd
+# set GOMAXPROCS to number of processors
+ExecStart=/bin/bash -c "ETCD_UNSUPPORTED_ARCH=arm64 GOMAXPROCS=$(nproc) /usr/bin/etcd --name=\"${ETCD_NAME}\" --data-dir=\"${ETCD_DATA_DIR}\" --listen-client-urls=\"${ETCD_LISTEN_CLIENT_URLS}\""
+Restart=on-failure
+LimitNOFILE=65536
+
+[Install]
+WantedBy=multi-user.target

lib/ansible/roles/glusterfs/tasks/main.yml

@@ -0,0 +1,23 @@
+---
+- name: Transfer GlusterFS repo
+  template: src=gluster.repo dest=/etc/yum.repos.d/gluster.repo
+
+- name: Install GlusterFS Server
+  yum: name=glusterfs-server state=present
+
+- name: Install GlusterFS Client
+  yum: name=glusterfs-client state=present
+
+- name: Enable GlusterFS
+  service: name=glusterd state=started enabled=yes
+
+- name: Peer Probe
+  shell: |
+    servers="{{serverlist}}"
+    for i in ${servers}; do
+      gluster peer probe ${i};
+    done
+  args:
+    executable: /bin/bash
+  vars:
+    serverlist: "{{groups['glusterfs']|map('extract',hostvars,['ansible_default_ipv4','address'])|join(' ')}}"

lib/ansible/roles/kubernetes/kubernetes_common/tasks/main.yml

@@ -1,6 +1,6 @@
 ---
-- name: Install kubernetes by yum
-  yum: pkg=kubernetes state=latest
+#- name: Install kubernetes by yum
+#  yum: pkg=kubernetes state=latest
 
 - name: Judge whether firewalld installed
   command: "rpm -q firewalld"
@@ -10,8 +10,11 @@
 
 - name: Find ip addresses
   set_fact:
-    master_ip: "{{hostvars[groups['masters'][0]]['ansible_default_ipv4']['address']}}"
-    etcd_ip: "{{hostvars[groups['etcd'][0]]['ansible_default_ipv4']['address']}}"
+    master_ip_address: "{{hostvars[groups['kubernetes_master'][0]]['ansible_default_ipv4']['address']}}"
+    etcd_ip_address: "{{hostvars[groups['etcd_server'][0]]['ansible_default_ipv4']['address']}}"
+
+- name: Make kubernetes config dir
+  file: path=/etc/kubernetes state=directory mode=0644
 
 - name: Write common config file
   template: src=config.j2 dest=/etc/kubernetes/config

lib/ansible/roles/kubernetes/kubernetes_common/templates/config.j2

@@ -19,4 +19,4 @@ KUBE_LOG_LEVEL="--v=0"
 KUBE_ALLOW_PRIV="--allow-privileged=false"
 
 # How the controller-manager, scheduler, and proxy find the apiserver
-KUBE_MASTER="--master=http://{{master_ip}}:8080"
+KUBE_MASTER="--master=http://{{master_ip_address}}:8080"

lib/ansible/roles/kubernetes/kubernetes_master/tasks/main.yml

@@ -3,10 +3,10 @@
   template: src=apiserver.j2 dest=/etc/kubernetes/apiserver
 
 - name: Write config file for controller-manater
-  template: src=controller-manater.j2 dest=/etc/kubernetes/controller-manater
+  template: src=controller-manager.j2 dest=/etc/kubernetes/controller-manager
 
 - name: Write config file for scheduler
-  template: src=scheduler.j2 dest=/etc/kubernetes/scheduler.j2
+  template: src=scheduler.j2 dest=/etc/kubernetes/scheduler
 
 - name: Enable apiserver
   service: name=kube-apiserver enabled=yes state=started
@@ -18,5 +18,5 @@
   service: name=kube-scheduler enabled=yes state=started
 
 - name: Expose firewalld port for kube-apiserver
-  firewalld: port-8080/tcmp permanent=false stated=enabled
+  firewalld: port=8080/tcmp permanent=false state=enabled
   when: installed_firewalld.rc == 0

lib/ansible/roles/kubernetes/kubernetes_master/templates/apiserver.j2

@@ -14,7 +14,7 @@ KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
 # KUBELET_PORT="--kubelet-port=10250"
 
 # Comma separated list of nodes in the etcd cluster
-KUBE_ETCD_SERVERS="--etcd-servers={{etcd_ip_address}}"
+KUBE_ETCD_SERVERS="--etcd-servers=http://{{etcd_ip_address}}:2379"
 
 # Address range to use for services
 KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.254.0.0/16"

lib/ansible/roles/kubernetes/kubernetes_minion/templates/kubelet.j2

@@ -1,14 +1,14 @@
 ###
 # kubernetes kubelet (minion) config
-
+KUBELET_API_SERVER="--api_servers=http://{{master_ip_address}}:8080"
 # The address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)
 KUBELET_ADDRESS="--address=0.0.0.0"
 
 # The port for the info server to serve on
 # KUBELET_PORT="--port=10250"
 
 # You may leave this blank to use the actual hostname
-KUBELET_HOSTNAME="--hostname-override={{ansible_hostname}}"
+KUBELET_HOSTNAME="--hostname-override={{ansible_default_ipv4.address}}"
 
 # Add your own!
 KUBELET_ARGS="--cgroup-driver=systemd"