pcp/prometheus implant: enable collection of intra-pod data for monit…

…oring (#450)

This patch adds two pcp-related background processes to the container,
costing about 25MB more memory.  They allow collection of intra-pod
system stats, plus prometheus metrics at http://localhost:8080/metrics
for relay to a central logging server.  They are passive (use zero
CPU) unless such a connection is made.

Dockerfile.deploy

@@ -18,10 +18,21 @@ RUN cd /tmp \
 COPY bin/fabric8-tenant ${INSTALL_PREFIX}/bin/fabric8-tenant
 RUN mkdir ${KUBECONFIG_DIR} && chmod +777 ${KUBECONFIG_DIR}
 
+# Install little pcp pmcd server for metrics collection
+# would prefer only pmcd, and not the /bin/pm*tools etc.
+COPY pcp.repo /etc/yum.repos.d/pcp.repo
+RUN yum install -y pcp pcp-pmda-prometheus && yum clean all && \
+    mkdir -p /etc/pcp /var/run/pcp /var/lib/pcp /var/log/pcp && \
+    chown -R ${F8_USER_NAME} /etc/pcp /var/run/pcp /var/lib/pcp /var/log/pcp && \
+    chmod -R ug+rw /etc/pcp /var/run/pcp /var/lib/pcp /var/log/pcp
+COPY ./tenant+pmcd.sh /tenant+pmcd.sh
+EXPOSE 44321
+
+
 # From here onwards, any RUN, CMD, or ENTRYPOINT will be run under the following user
 USER ${F8_USER_NAME}
 
 WORKDIR ${INSTALL_PREFIX}
-ENTRYPOINT [ "bin/fabric8-tenant" ]
+ENTRYPOINT [ "/tenant+pmcd.sh" ]
 
 EXPOSE 8080

pcp.repo

@@ -0,0 +1,10 @@
+[fche-pcp]
+name=Copr repo for pcp owned by fche
+baseurl=https://copr-be.cloud.fedoraproject.org/results/fche/pcp/epel-7-$basearch/
+type=rpm-md
+skip_if_unavailable=True
+gpgcheck=1
+gpgkey=https://copr-be.cloud.fedoraproject.org/results/fche/pcp/pubkey.gpg
+repo_gpgcheck=0
+enabled=1
+enabled_metadata=1

tenant+pmcd.sh

@@ -0,0 +1,52 @@
+#! /bin/sh
+
+set -eu
+
+# Initialize a little unprivileged pcp pmcd metrics collector
+# process within this container; run this in a background subshell.
+# No special signal handling or cleanup required.
+(
+# Setup pmcd to run in unprivileged mode of operation
+. /etc/pcp.conf
+
+PATH=$PATH:$PCP_BINADM_DIR
+export PATH
+
+# Configure pmcd with a minimal set of DSO agents
+rm -f $PCP_PMCDCONF_PATH; # start empty
+echo "# Name  ID  IPC  IPC Params  File/Cmd" >> $PCP_PMCDCONF_PATH;
+echo "pmcd     2  dso  pmcd_init   $PCP_PMDAS_DIR/pmcd/pmda_pmcd.so"   >> $PCP_PMCDCONF_PATH;
+echo "proc     3  dso  proc_init   $PCP_PMDAS_DIR/proc/pmda_proc.so"   >> $PCP_PMCDCONF_PATH;
+echo "linux   60  dso  linux_init  $PCP_PMDAS_DIR/linux/pmda_linux.so" >> $PCP_PMCDCONF_PATH;
+echo "prometheus 144 pipe binary python $PCP_PMDAS_DIR/prometheus/pmdaprometheus.python" >> $PCP_PMCDCONF_PATH;
+rm -f $PCP_VAR_DIR/pmns/root_xfs $PCP_VAR_DIR/pmns/root_jbd2 $PCP_VAR_DIR/pmns/root_root $PCP_VAR_DIR/pmns/root
+echo 'prometheus	144:*:*' > $PCP_VAR_DIR/pmns/prometheus
+touch $PCP_VAR_DIR/pmns/.NeedRebuild
+
+# allow unauthenticated access to proc.* metrics (default is false)
+export PROC_ACCESS=1
+export PMCD_ROOT_AGENT=0
+
+# NB: we can't use the rc.pmcd script.  It assumes that it's run as root.
+cd $PCP_VAR_DIR/pmns
+./Rebuild
+pmnsadd -n root prometheus
+
+# Add our lone prometheus source
+URLSD=$PCP_PMDAS_DIR/prometheus/urls.d
+mkdir -p $URLSD
+echo 'http://localhost:8080/metrics' > $URLSD/wit.url
+
+cd $PCP_LOG_DIR
+
+: "${PCP_HOSTNAME:=`hostname`}"
+# possibly: filter pod name?
+
+# We can log in plaintext to stdout also, even though WIT uses
+# JSON.  pmcd is not chatty and only speaks up during errors.
+exec /usr/libexec/pcp/bin/pmcd -l /dev/force-logging-to-stderr -f -A -H $PCP_HOSTNAME
+) &
+sleep 5 # give time for pmcd's startup messages, so it doesn't intermix with WIT's
+
+
+exec bin/fabric8-tenant ${1+"$@"}