From fcc856b93a932410ce563e4e0e524435b64ec59e Mon Sep 17 00:00:00 2001 From: home Date: Mon, 16 Dec 2019 15:00:43 +0530 Subject: [PATCH] Initial Commit --- .DS_Store | Bin 0 -> 6148 bytes jenkins-ci/Dockerfile | 70 +++++ jenkins-ci/jenkins.yaml | 142 ++++++++++ logging/elastic-stack.yaml | 46 ++++ logging/fluentd-config.yaml | 455 ++++++++++++++++++++++++++++++++ logging/fluentd-daemonset.yaml | 60 +++++ logging/fluentd-rbac.yaml | 38 +++ logging/kibana.yaml | 42 +++ pgadmin/pgadmin-workload.yaml | 42 +++ postgres/postgres-workload.yaml | 85 ++++++ rabbit-mq/rabbit-workload.yaml | 39 +++ zipkin/zipkin-workload.yaml | 35 +++ 12 files changed, 1054 insertions(+) create mode 100644 .DS_Store create mode 100644 jenkins-ci/Dockerfile create mode 100644 jenkins-ci/jenkins.yaml create mode 100644 logging/elastic-stack.yaml create mode 100644 logging/fluentd-config.yaml create mode 100644 logging/fluentd-daemonset.yaml create mode 100644 logging/fluentd-rbac.yaml create mode 100644 logging/kibana.yaml create mode 100644 pgadmin/pgadmin-workload.yaml create mode 100644 postgres/postgres-workload.yaml create mode 100644 rabbit-mq/rabbit-workload.yaml create mode 100644 zipkin/zipkin-workload.yaml diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..45cf31dfb02ae8e9b18794dd108dac2a793c59e6 GIT binary patch literal 6148 zcmeHKJx{|h5Pb&)f><&#dSqc>W{IjYF!Kj!D5@#~2^1El{BwBcI}j;B-4H@|k^L_A zJAXVWb`F3{UlwOT37}w66m>>Sn@78LoV-VrJjMzSSY9^mP2G$P^cS0S?FVSkV#(*r z`rDd4<=cj3c1PwM3*0N;cKai{K*zDycxKEKZn56NKO}38n$LH^I!>B`n zP++LQuDzCe|6lNv8BOwsA@LRpgaZFe0h!FMW>ap;@78bI)4MjYT(hVtuUCggfAAB) i7d=OgbEowu^~tLodqW*X+jBTEegv{0QH27(pujsy^mC5@ literal 0 HcmV?d00001 diff --git a/jenkins-ci/Dockerfile b/jenkins-ci/Dockerfile new file mode 100644 index 0000000..eab2fcc --- /dev/null +++ b/jenkins-ci/Dockerfile @@ -0,0 +1,70 @@ +FROM jenkins/jenkins:lts +USER root +# Pipeline +RUN /usr/local/bin/install-plugins.sh workflow-aggregator && \ +/usr/local/bin/install-plugins.sh github && \ +/usr/local/bin/install-plugins.sh ws-cleanup && \ +/usr/local/bin/install-plugins.sh greenballs && \ +/usr/local/bin/install-plugins.sh simple-theme-plugin && \ +/usr/local/bin/install-plugins.sh kubernetes && \ +/usr/local/bin/install-plugins.sh docker-workflow && \ +/usr/local/bin/install-plugins.sh kubernetes-cli && \ +/usr/local/bin/install-plugins.sh github-branch-source + +# install Maven, Java, Docker, AWS +RUN apt-get update && \ +apt-get -y install apt-transport-https \ +ca-certificates \ +curl \ +gnupg2 \ +software-properties-common && \ +curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg > /tmp/dkey; apt-key add /tmp/dkey && \ +add-apt-repository \ +"deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \ +$(lsb_release -cs) \ +stable" && \ +apt-get update && \ +apt-get -y install docker-ce +RUN apt-get update && \ +apt-get install -yq --no-install-recommends wget pwgen ca-certificates && \ +apt-get clean && \ +rm -rf /var/lib/apt/lists/* + +RUN apt-get update && \ +apt-get install -y openjdk-8-jdk && \ +apt-get install -y ant && \ +apt-get clean +RUN apt-get update && \ +apt-get install ca-certificates-java && \ +apt-get clean && \ +update-ca-certificates -f; +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ + +RUN export JAVA_HOME + +RUN wget http://mirrors.gigenet.com/apache/maven/maven-3/3.6.2/binaries/apache-maven-3.6.2-bin.tar.gz +RUN apt-get update -y && apt-get install maven -y + + + + + +# Kubectl + +RUN wget https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x ./kubectl && mv ./kubectl /usr/local/bin/kubectl + + + +# Need to ensure the gid here matches the gid on the host node. We ASSUME (hah!) this + +# will be stable....keep an eye out for unable to connect to docker.sock in the builds + +# RUN delgroup ping && delgroup docker && addgroup -g 999 docker && addgroup jenkins docker + + + +# See https://github.com/kubernetes/minikube/issues/956. + +# THIS IS FOR MINIKUBE TESTING ONLY - it is not production standard (we're running as root!) + +RUN chown -R root "$JENKINS_HOME" /usr/share/jenkins/ref diff --git a/jenkins-ci/jenkins.yaml b/jenkins-ci/jenkins.yaml new file mode 100644 index 0000000..0106180 --- /dev/null +++ b/jenkins-ci/jenkins.yaml @@ -0,0 +1,142 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: jenkins + namespace: default +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: jenkins + namespace: default +rules: +- apiGroups: [""] + resources: ["pods","services"] + verbs: ["create","delete","get","list","patch","update","watch"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["create","delete","get","list","patch","update","watch"] +- apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create","delete","get","list","patch","update","watch"] +- apiGroups: [""] + resources: ["pods/log"] + verbs: ["get","list","watch"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create","delete","get","list","patch","update","watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: jenkins + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: jenkins +subjects: +- kind: ServiceAccount + name: jenkins +--- +# Allows jenkins to create persistent volumes +# This cluster role binding allows anyone in the "manager" group to read secrets in any namespace. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: jenkins-crb +subjects: +- kind: ServiceAccount + namespace: default + name: jenkins +roleRef: + kind: ClusterRole + name: jenkinsclusterrole + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # "namespace" omitted since ClusterRoles are not namespaced + name: jenkinsclusterrole +rules: +- apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["create","delete","get","list","patch","update","watch"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jenkins + namespace: default +spec: + selector: + matchLabels: + app: jenkins + replicas: 1 + template: + metadata: + labels: + app: jenkins + spec: + containers: + - name: jenkins + image: myjenkins:latest + env: + - name: JAVA_OPTS + value: -Djenkins.install.runSetupWizard=false + ports: + - name: http-port + containerPort: 8080 + - name: jnlp-port + containerPort: 50000 + volumeMounts: + - name: jenkins-home + mountPath: /data/pv0001/ + - name: docker-sock-volume + mountPath: "/var/run/docker.sock" + imagePullPolicy: "IfNotPresent" + volumes: + # This allows jenkins to use the docker daemon on the host, for running builds + # see https://stackoverflow.com/questions/27879713/is-it-ok-to-run-docker-from-inside-docker + - name: docker-sock-volume + hostPath: + path: /var/run/docker.sock + - name: jenkins-home + hostPath: + path: /data/pv0001/ + serviceAccountName: jenkins +--- +apiVersion: v1 +kind: Service +metadata: + name: jenkins + namespace: default +spec: + type: NodePort + ports: + - name: ui + port: 8080 + targetPort: 8080 + nodePort: 31000 + - name: jnlp + port: 50000 + targetPort: 50000 + selector: + app: jenkins +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv0001 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 5Gi + hostPath: + path: /data/pv0001/ \ No newline at end of file diff --git a/logging/elastic-stack.yaml b/logging/elastic-stack.yaml new file mode 100644 index 0000000..5845947 --- /dev/null +++ b/logging/elastic-stack.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch +spec: + selector: + matchLabels: + component: elasticsearch + template: + metadata: + labels: + component: elasticsearch + spec: + containers: + - name: elasticsearch + image: elasticsearch:7.3.2 + env: + - name: discovery.type + value: single-node + ports: + - containerPort: 9200 + name: http + protocol: TCP + resources: + limits: + cpu: 500m + memory: 4Gi + requests: + cpu: 500m + memory: 4Gi + +--- + +apiVersion: v1 +kind: Service +metadata: + name: elasticsearch + labels: + service: elasticsearch +spec: + type: NodePort + selector: + component: elasticsearch + ports: + - port: 9200 + targetPort: 9200 \ No newline at end of file diff --git a/logging/fluentd-config.yaml b/logging/fluentd-config.yaml new file mode 100644 index 0000000..9aaeb67 --- /dev/null +++ b/logging/fluentd-config.yaml @@ -0,0 +1,455 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: fluentd-es-config-v0.1.4 + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + system.conf: |- + + root_dir /tmp/fluentd-buffers/ + + + containers.input.conf: |- + # This configuration file for Fluentd / td-agent is used + # to watch changes to Docker log files. The kubelet creates symlinks that + # capture the pod name, namespace, container name & Docker container ID + # to the docker logs for pods in the /var/log/containers directory on the host. + # If running this fluentd configuration in a Docker container, the /var/log + # directory should be mounted in the container. + # + # These logs are then submitted to Elasticsearch which assumes the + # installation of the fluent-plugin-elasticsearch & the + # fluent-plugin-kubernetes_metadata_filter plugins. + # See https://github.com/uken/fluent-plugin-elasticsearch & + # https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter for + # more information about the plugins. + # + # Example + # ======= + # A line in the Docker log file might look like this JSON: + # + # {"log":"2014/09/25 21:15:03 Got request with path wombat\n", + # "stream":"stderr", + # "time":"2014-09-25T21:15:03.499185026Z"} + # + # The time_format specification below makes sure we properly + # parse the time format produced by Docker. This will be + # submitted to Elasticsearch and should appear like: + # $ curl 'http://elasticsearch-logging:9200/_search?pretty' + # ... + # { + # "_index" : "logstash-2014.09.25", + # "_type" : "fluentd", + # "_id" : "VBrbor2QTuGpsQyTCdfzqA", + # "_score" : 1.0, + # "_source":{"log":"2014/09/25 22:45:50 Got request with path wombat\n", + # "stream":"stderr","tag":"docker.container.all", + # "@timestamp":"2014-09-25T22:45:50+00:00"} + # }, + # ... + # + # The Kubernetes fluentd plugin is used to write the Kubernetes metadata to the log + # record & add labels to the log record if properly configured. This enables users + # to filter & search logs on any metadata. + # For example a Docker container's logs might be in the directory: + # + # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b + # + # and in the file: + # + # 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log + # + # where 997599971ee6... is the Docker ID of the running container. + # The Kubernetes kubelet makes a symbolic link to this file on the host machine + # in the /var/log/containers directory which includes the pod name and the Kubernetes + # container name: + # + # synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log + # -> + # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log + # + # The /var/log directory on the host is mapped to the /var/log directory in the container + # running this instance of Fluentd and we end up collecting the file: + # + # /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log + # + # This results in the tag: + # + # var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log + # + # The Kubernetes fluentd plugin is used to extract the namespace, pod name & container name + # which are added to the log message as a kubernetes field object & the Docker container ID + # is also added under the docker field object. + # The final tag is: + # + # kubernetes.var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log + # + # And the final log record look like: + # + # { + # "log":"2014/09/25 21:15:03 Got request with path wombat\n", + # "stream":"stderr", + # "time":"2014-09-25T21:15:03.499185026Z", + # "kubernetes": { + # "namespace": "default", + # "pod_name": "synthetic-logger-0.25lps-pod", + # "container_name": "synth-lgr" + # }, + # "docker": { + # "container_id": "997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b" + # } + # } + # + # This makes it easier for users to search for logs by pod name or by + # the name of the Kubernetes container regardless of how many times the + # Kubernetes pod has been restarted (resulting in a several Docker container IDs). + + # Json Log Example: + # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} + # CRI Log Example: + # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here + + @id fluentd-containers.log + @type tail + path /var/log/containers/*.log + pos_file /var/log/es-containers.log.pos + time_format %Y-%m-%dT%H:%M:%S.%NZ + tag raw.kubernetes.* + read_from_head true + + @type multi_format + + format json + time_key time + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format /^(? + + + + # Detect exceptions in the log output and forward them as one log entry. + + @id raw.kubernetes + @type detect_exceptions + remove_tag_prefix raw + message log + stream stream + multiline_flush_interval 5 + max_bytes 500000 + max_lines 1000 + + + system.input.conf: |- + # Example: + # 2015-12-21 23:17:22,066 [salt.state ][INFO ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081 + + @id minion + @type tail + format /^(?