Sync upstream june6

.gitreview

@@ -1,5 +1,5 @@
 [gerrit]
-host=review.openstack.org
+host=review.opendev.org
 port=29418
 project=openstack/kolla.git
 defaultbranch=stable/rocky

.zuul.d/centos.yaml

@@ -12,6 +12,10 @@
             required-projects:
               - openstack/kolla-ansible
               - openstack/requirements
+        - kolla-ansible-bifrost-centos-source:
+            files: ^docker\/bifrost\/.*
+        - kolla-ansible-centos-source-ceph:
+            files: ^docker\/(ceph|cinder)\/.*
     gate:
       jobs:
         - kolla-build-centos-source

.zuul.d/oraclelinux.yaml

@@ -12,6 +12,8 @@
             required-projects:
               - openstack/kolla-ansible
               - openstack/requirements
+        - kolla-ansible-oraclelinux-source-ceph:
+            files: ^docker\/(ceph|cinder)\/.*
     gate:
       jobs:
         - kolla-build-oraclelinux-source

.zuul.d/ubuntu.yaml

@@ -19,6 +19,8 @@
             required-projects:
               - openstack/kolla-ansible
               - openstack/requirements
+        - kolla-ansible-ubuntu-source-ceph:
+            files: ^docker\/(ceph|cinder)\/.*
     gate:
       jobs:
         - kolla-build-ubuntu-source

contrib/template-override/monasca-template-overrides.j2

@@ -0,0 +1,3 @@
+{% extends parent_template %}
+
+{% set influxdb_packages_override = ["influxdb-1.3.7"] %}

doc/source/admin/template-override/monasca.rst

@@ -0,0 +1,18 @@
+==============================================
+Building non-standard Monasca container images
+==============================================
+
+The `monasca-template-overrides.j2` file shows how to build
+any images required for Monasca which are non-standard. Currently this
+includes only InfluxDB. When Monasca is upgraded this file will
+no longer be required.
+
+InfluxDB
+--------
+
+Build the container by executing the following command:
+
+.. code-block:: console
+
+   kolla-build --template-override contrib/template-override/monasca-template-overrides.j2 influxdb
+

docker/base/elasticsearch.repo

@@ -1,6 +1,6 @@
-[elasticsearch-kibana-logstash-5.x]
-name=ELK repository for 5.x packages
-baseurl=https://artifacts.elastic.co/packages/5.x/yum
+[elasticsearch-kibana-logstash-6.x]
+name=ELK repository for 6.x packages
+baseurl=https://artifacts.elastic.co/packages/6.x/yum
 gpgcheck=1
 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
 enabled=1

docker/base/sources.list.debian

@@ -14,7 +14,7 @@ deb https://packages.grafana.com/oss/deb stable main
 deb [arch=arm64] https://dl.bintray.com/fg2it/deb-arm64/ stretch main
 
 # elasticsearch (arch:all), logstash (arch:all), kibana (arch:amd64)
-deb [arch=amd64] https://artifacts.elastic.co/packages/5.x/apt stable main
+deb [arch=amd64] https://artifacts.elastic.co/packages/6.x/apt stable main
 
 # main docker repo
 deb https://download.docker.com/linux/debian stretch stable

docker/base/sources.list.ubuntu

@@ -13,7 +13,7 @@ deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted universe
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu bionic-updates/rocky main
 
 # Elasticsearch, Logstash & Kibana repo
-deb https://artifacts.elastic.co/packages/5.x/apt stable main
+deb https://artifacts.elastic.co/packages/6.x/apt stable main
 
 # InfluxDB repo
 deb https://repos.influxdata.com/ubuntu bionic stable

docker/caso/Dockerfile.j2

@@ -0,0 +1,37 @@
+FROM {{ namespace }}/{{ image_prefix }}openstack-base:{{ tag }}
+LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+
+{% block caso_header %}{% endblock %}
+
+{% import "macros.j2" as macros with context %}
+
+{% if base_distro in ['centos', 'oraclelinux', 'rhel'] %}
+    {% set caso_packages = [
+        'cronie',
+        'python-pip',
+    ] %}
+{% elif base_distro in ['debian', 'ubuntu'] %}
+    {% set caso_packages = [
+        'cron',
+        'python-pip',
+    ] %}
+{% endif %}
+
+{{ macros.install_packages(caso_packages | customizable("packages")) }}
+
+{{ macros.configure_user(name='caso') }}
+
+{% set caso_pip_packages = [
+    'caso'
+] %}
+
+RUN {{ macros.install_pip(caso_pip_packages | customizable("pip_packages"),  constraints = false) }} \
+    && mkdir -p /etc/caso \
+    && chown -R caso: /etc/caso
+
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+
+RUN touch /usr/local/bin/kolla_caso_extend_start \
+    && chmod 755 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_caso_extend_start
+
+{% block caso_base_footer %}{% endblock %}

docker/caso/extend_start.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# Create log directory, with appropriate permissions
+CASO_LOG_DIR="/var/log/kolla/caso"
+if [[ ! -d "$CASO_LOG_DIR" ]]; then
+    mkdir -p $CASO_LOG_DIR
+fi
+if [[ $(stat -c %U:%G ${CASO_LOG_DIR}) != "caso:kolla" ]]; then
+    chown caso:kolla ${CASO_LOG_DIR}
+fi
+if [[ $(stat -c %a ${CASO_LOG_DIR}) != "755" ]]; then
+    chmod 755 ${CASO_LOG_DIR}
+fi
+
+. /usr/local/bin/kolla_caso_extend_start

docker/ceph/ceph-mon/extend_start.sh

@@ -39,6 +39,16 @@ if [[ ! -e "${MON_DIR}/keyring" ]]; then
     ceph-authtool --create-keyring "${KEYRING_TMP}" --import-keyring "${KEYRING_ADMIN}"
     ceph-authtool "${KEYRING_TMP}" --import-keyring "${KEYRING_MON}"
     mkdir -p "${MON_DIR}"
-    ceph-mon --mkfs -i "${HOSTNAME}" --monmap "${MONMAP}" --keyring "${KEYRING_TMP}"
+
+    mon_stat=$(ceph mon stat --connect-timeout 1 || true)
+    mon_check=$(echo $mon_stat | awk '/mons/{print $0}' | wc -l)
+    if [[ ${mon_check} -eq 0 ]]; then
+        ceph-mon --mkfs -i "${HOSTNAME}" --keyring "${KEYRING_TMP}"
+    else
+        MONMAP_TMP="/tmp/ceph.${HOSTNAME}.monmap"
+        ceph mon getmap -o "${MONMAP_TMP}"
+        ceph-mon --mkfs -i "${HOSTNAME}" --monmap "${MONMAP_TMP}" --keyring "${KEYRING_TMP}"
+        rm "${MONMAP_TMP}"
+    fi
     rm "${KEYRING_TMP}"
 fi

docker/ceph/ceph-osd/extend_start.sh

@@ -38,6 +38,7 @@ if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then
                 sgdisk --zap-all -- "${OSD_BS_DEV}"
                 sgdisk --new=1:0:+100M --mbrtogpt -- "${OSD_BS_DEV}"
                 sgdisk --largest-new=2 --mbrtogpt -- "${OSD_BS_DEV}"
+                partprobe || true
                 sgdisk --zap-all -- "${OSD_BS_DEV}"2
             fi
 

docker/fluentd/Dockerfile.j2

@@ -15,7 +15,10 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
     {% set fluentd_packages = [
         'fluentd',
+        'gcc-c++',
         'hostname',
+        'make',
+        'ruby-devel',
         'rubygem-fluent-plugin-elasticsearch',
         'rubygem-fluent-plugin-grok-parser',
         'rubygem-fluent-plugin-kubernetes_metadata_filter',
@@ -28,6 +31,8 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     {% set fluentd_packages = [
         'g++',
         'make',
+        'ruby',
+        'ruby-dev',
         'td-agent'
     ] %}
 
@@ -69,6 +74,17 @@ RUN chmod 755 /usr/local/bin/kolla_extend_start
 
 {{ macros.install_fluent_plugins(fluentd_plugins | customizable("plugins")) }}
 
+# Build and install Fluentd output plugin for Monasca Log API
+ARG monasca_output_plugin_tag=0.1.0
+ARG monasca_output_plugin_url=https://github.com/monasca/fluentd-monasca/archive/$monasca_output_plugin_tag.tar.gz
+ADD $monasca_output_plugin_url /tmp/fluentd-monasca.tar.gz
+RUN tar -xvf /tmp/fluentd-monasca.tar.gz -C /tmp \
+    && cd /tmp/fluentd-monasca-* \
+    && gem build fluentd-monasca-output.gemspec \
+    && gem install fluentd-monasca-output-*.gem \
+    && fluent-gem install fluentd-monasca-output-*.gem \
+    && rm -rf /tmp/fluentd*
+
 {% endblock %}
 
 {% block fluentd_footer %}{% endblock %}

docker/gnocchi/gnocchi-base/Dockerfile.j2

@@ -15,8 +15,9 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'mod_ssl',
             'mod_wsgi',
             'openstack-gnocchi-common',
-            'python2-ldappool',
-            'python2-cradox'
+            'python-boto3',
+            'python2-cradox',
+            'python2-ldappool'
         ] %}
 
 {{ macros.install_packages(gnocchi_base_packages | customizable("packages")) }}
@@ -30,6 +31,7 @@ RUN sed -i -r 's,^(Listen 80),#\1,' /etc/httpd/conf/httpd.conf \
             'gnocchi-common',
             'libapache2-mod-wsgi',
             'librados-dev',
+            'python-boto3',
             'python-dev',
             'python-ldappool',
             'python-rados',
@@ -96,7 +98,7 @@ ADD gnocchi-base-archive /gnocchi-base-source
 
 {% set gnocchi_base_pip_packages = [
     'gnocchiclient',
-    '/gnocchi[keystone,mysql,file,ceph]'
+    '/gnocchi[keystone,mysql,file,ceph,s3]'
 ] %}
 
 RUN ln -s gnocchi-base-source/* gnocchi \

docker/ironic/ironic-conductor/Dockerfile.j2

@@ -60,6 +60,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'python-systemd',
             'qemu-utils',
             'shellinabox',
+            'udev',
             'xfsprogs',
         ] %}
     {% endif %}
@@ -75,6 +76,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'psmisc',
             'qemu-img-ev',
             'shellinabox',
+            'systemd',
             'util-linux',
             'xfsprogs',
         ] %}
@@ -89,6 +91,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
             'psmisc',
             'qemu-utils',
             'shellinabox',
+            'udev',
             'xfsprogs',
         ] %}
     {% endif %}

docker/kafka/Dockerfile.j2

@@ -13,7 +13,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
     ] %}
 {% elif base_distro in ['debian', 'ubuntu'] %}
     {% set kafka_packages = [
-        'default-jre-headless',
+        'openjdk-8-jre-headless',
     ] %}
 {% endif %}
 
@@ -22,7 +22,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 {% block kafka_version %}
 ENV kafka_version=1.0.2
 ENV kafka_scala_version=2.12
-ENV kafka_url=http://apache.osuosl.org/kafka/${kafka_version}/kafka_${kafka_scala_version}-${kafka_version}.tgz
+ENV kafka_url=https://archive.apache.org/dist/kafka/${kafka_version}/kafka_${kafka_scala_version}-${kafka_version}.tgz
 ENV kafka_pkg_sha512sum=4cbcdaf8ccc4efe3d1b6275f3f2c32cf8f2f1a62104b5dd0bd9e2974160ab89d85a6e1791af8b948a413b99ed696b06ea9d4299b27ea63c3f7318dabf5761144
 {% endblock %}
 

docker/keystone/keystone-fernet/fetch_fernet_tokens.py

@@ -24,7 +24,8 @@
 import os
 import sys
 
-TOKEN_PATH = '/etc/keystone/fernet-keys'
+# Adding nosec since this fails bandit B105, 'Possible hardcoded password'.
+TOKEN_PATH = '/etc/keystone/fernet-keys'  # nosec
 
 
 def json_exit(msg=None, failed=False, changed=False):

docker/logstash/Dockerfile.j2

@@ -9,16 +9,15 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
 {% if base_distro in ['centos', 'oraclelinux', 'rhel'] %}
     {% set logstash_packages = [
-        'logstash',
         'java-1.8.0-openjdk-headless'
     ] %}
 
 ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk/
 
 {% elif base_distro in ['debian', 'ubuntu'] %}
     {% set logstash_packages = [
-        'logstash',
-        'openjdk-8-jre-headless'
+        'openjdk-8-jre-headless',
+        'logrotate'
     ] %}
 
 {% if base_arch == 'x86_64' %}
@@ -36,6 +35,31 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-${java_arch}/
 {{ macros.install_packages(logstash_packages | customizable("packages")) }}
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
 
+# FIXME(dszumski): Logstash was added to Kolla for Monasca. Monasca is
+# currently tied to an old messaging version in Kafka (0.9) and doesn't
+# work with Logstash 5.x or above. When Monasca is updated to use a more
+# recent messaging version this patch should be reverted and the Monasca
+# Logstash configuration files upgraded to use the new format.
+{% if base_distro in ['centos', 'oraclelinux', 'rhel'] %}
+
+ENV logstash_rpm_url=https://download.elastic.co/logstash/logstash/packages/centos/logstash-2.4.1.noarch.rpm
+ENV logstash_rpm_sha1sum=68c5e78c3d941f06f77d638c0756b6aea715bec5
+RUN curl -sSL -o /tmp/logstash.rpm ${logstash_rpm_url} \
+    && echo "${logstash_rpm_sha1sum} /tmp/logstash.rpm" | sha1sum -c \
+    && yum -y install /tmp/logstash.rpm \
+    && rm -f /tmp/logstash.rpm
+
+{% elif base_distro in ['debian', 'ubuntu'] %}
+
+ENV logstash_deb_url=https://download.elastic.co/logstash/logstash/packages/debian/logstash-2.4.1_all.deb
+ENV logstash_deb_sha1sum=7ba3b174a3ef48a7d0945d9b5c7f12c5005abb47
+RUN curl -sSL -o /tmp/logstash.deb ${logstash_deb_url} \
+    && echo "${logstash_deb_sha1sum} /tmp/logstash.deb" | sha1sum -c \
+    && apt install /tmp/logstash.deb \
+    && rm -f /tmp/logstash.deb
+
+{% endif %}
+
 RUN chmod 755 /usr/local/bin/kolla_extend_start
 
 {% block logstash_footer %}{% endblock %}