Fix Kibana config #11

deploy/README-AI_MODELS.md

@@ -0,0 +1,40 @@
+Directory structure of `models` directory to be volume mount to [ml-api docker service][0]:
+
+
+
+```
+tree -L 2 /mnt/extra/models/                                                                                                         la-dev 
+/mnt/extra/models/
+├── bigrams.phr
+├── jbook_qexp_20201217
+│   ├── ann-index_1608230794.1036441.ann
+│   └── ann-index-vocab_1608230794.1036441.pkl
+├── ltr
+├── qexp_20201217
+│   ├── ann-index_1608230794.1036441.ann
+│   └── ann-index-vocab_1608230794.1036441.pkl
+├── sent_index_20210422
+│   ├── config
+│   ├── data.csv
+│   ├── doc_ids.txt
+│   ├── embeddings
+│   ├── embeddings.npy
+│   └── metadata.json
+├── tfidf_dictionary.dic
+├── tfidf.model
+└── transformers
+    ├── bert-base-cased-squad2
+    ├── crawl-300d-2M.vec
+    ├── distilbart-mnli-12-3
+    ├── distilbert-base-uncased-distilled-squad
+    ├── distilroberta-base
+    ├── msmarco-distilbert-base-v2
+    ├── wiki-news-300d-1M.bin -> wiki-news-300d-1M-subword.vec
+    ├── wiki-news-300d-1M-subword.bin
+    ├── wiki-news-300d-1M-subword.vec
+    └── wiki-news-300d-1M.vec
+
+10 directories, 18 files
+```
+
+[0]: ./docker-compose/services.yaml

deploy/build.sh

@@ -6,13 +6,15 @@ SCRIPT_DIR="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 REPO_DIR="$(cd -- "${SCRIPT_DIR}/../" &> /dev/null && pwd)"
 DEPLOY_CONF="${SCRIPT_DIR}/deploy.conf.sh"
 
+echo "REPO_DIR=$REPO_DIR"
+
 # make sure token is passed explicitly
-export NPM_AUTH_TOKEN="${NPM_AUTH_TOKEN}"
+export NPM_AUTH_TOKEN=${NPM_AUTH_TOKEN}
 
 source "$DEPLOY_CONF"
 
 function main() (
   compose_wrapper build
 )
 
-main
+main

deploy/deploy.conf.sh

@@ -18,19 +18,19 @@ declare -a REPO_NAMES=(\
 )
 
 declare -A REPO_URL_MAP=(\
-  [gamechanger-web]="https://github.com/dod-advana/gamechanger-web" \
-  [gamechanger-data]="https://github.com/dod-advana/gamechanger-data.git" \
-  [gamechanger-ml]="https://github.com/dod-advana/gamechanger-ml.git" \
+  [gamechanger-web]="https://github.com/neofob/gamechanger-web" \
+  [gamechanger-data]="https://github.com/neofob/gamechanger-data.git" \
+  [gamechanger-ml]="https://github.com/neofob/gamechanger-ml.git" \
   [gamechanger-neo4j-plugin]="https://github.com/dod-advana/gamechanger-neo4j-plugin.git" \
   [gamechanger-crawlers]="https://github.com/dod-advana/gamechanger-crawlers.git" \
 )
 
 declare -A REPO_TAG_MAP=(\
-  [gamechanger-web]="dev" \
-  [gamechanger-data]="hotfix/k8s-docker-update" \
-  [gamechanger-ml]="dev" \
+  [gamechanger-web]="tp-dev" \
+  [gamechanger-data]="tp-dev" \
+  [gamechanger-ml]="tp-dev" \
   [gamechanger-neo4j-plugin]="main" \
-  [gamechanger-crawlers]="dev" \
+  [gamechanger-crawlers]="4ac4f0e" \
 )
 
 declare -A REPO_DIR_MAP=(\
@@ -39,6 +39,7 @@ declare -A REPO_DIR_MAP=(\
   [gamechanger-ml]="${DEPLOY_BUILD_DIR}/gamechanger-ml" \
   [gamechanger-neo4j-plugin]="${DEPLOY_BUILD_DIR}/gamechanger-neo4j-plugin" \
   [gamechanger-crawlers]="${DEPLOY_BUILD_DIR}/gamechanger-crawlers" \
+  [gamechanger-postgres]="${DEPLOY_BUILD_DIR}/gamechanger-postgres" \
 )
 
 declare -A SERVICE_VAR_DIR_MAP=(\

deploy/docker-compose/.env

@@ -2,9 +2,10 @@ GC_WEB_IMAGE="gc-web:latest"
 GC_DATA_PIPELINES_IMAGE="gc-data-pipelines:latest"
 GC_NEO4J_IMAGE="gc-neo4j:latest"
 GC_POSTGRES_IMAGE="postgres:13.4-alpine"
-GC_REDIS_IMAGE="redis:5.0.14"
+#GC_POSTGRES_IMAGE="gc-postgres:latest"
+GC_REDIS_IMAGE="redis:7.2.0-alpine3.18"
 GC_KIBANA_IMAGE="kibana:7.10.1"
 GC_ELASTICSEARCH_IMAGE="elasticsearch:7.10.1"
 GC_S3_SERVER_IMAGE="minio/minio:RELEASE.2021-10-23T03-28-24Z"
 GC_S3_CLIENT_IMAGE="minio/mc:RELEASE.2021-10-07T04-19-58Z"
-GC_ML_API_IMAGE="gc-ml:latest"
+GC_ML_API_IMAGE="gc-ml:latest"

deploy/docker-compose/build.yaml

@@ -1,3 +1,4 @@
+---
 services:
   _build_web_image:
     image: "${GC_WEB_IMAGE}"
@@ -29,8 +30,8 @@ services:
       context: "./deploy/build/gamechanger-ml/"
       dockerfile: "./gamechangerml/api/fastapi/cpu.mlapp.Dockerfile"
       args:
-        APP_UID: 1001
-        APP_GID: 1001
+        APP_UID: 1000
+        APP_GID: 1000
     entrypoint:
       - echo
       - "[INFO] ML API image build completed."

deploy/docker-compose/common.yaml

@@ -1,3 +1,4 @@
+---
 version: '3.5'
 
 volumes:

deploy/docker-compose/kibana/config/kibana.yml

@@ -0,0 +1,28 @@
+---
+## Default Kibana configuration from Kibana base image.
+## https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts
+#
+server.name: kibana
+server.host: 0.0.0.0
+elasticsearch.hosts: [ http://elasticsearch:9200 ]
+
+monitoring.ui.container.elasticsearch.enabled: true
+monitoring.ui.container.logstash.enabled: true
+
+## X-Pack security credentials
+#
+elasticsearch.username: elasticsearch
+elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}
+
+## Encryption keys (optional but highly recommended)
+##
+## Generate with either
+##  $ docker container run --rm docker.elastic.co/kibana/kibana:7.10.1 bin/kibana-encryption-keys generate
+##  $ openssl rand -hex 32
+##
+## https://www.elastic.co/guide/en/kibana/current/using-kibana-with-security.html
+## https://www.elastic.co/guide/en/kibana/current/kibana-encryption-keys.html
+#
+#xpack.security.encryptionKey:
+#xpack.encryptedSavedObjects.encryptionKey:
+#xpack.reporting.encryptionKey:

deploy/docker-compose/services.yaml

@@ -1,14 +1,15 @@
+---
 services:
   web:
     image: "${GC_WEB_IMAGE}"
     container_name: "${COMPOSE_PROJECT_NAME:-gc}-web"
     user: "1000:1000"
     environment:
+      REACT_APP_BACKEND_URL: "http://localhost:8080"
       REACT_APP_GC_DECOUPLED: "true"
-      REACT_APP_NODE_ENV: "development"
+      REACT_APP_NODE_ENV: "production"
       REACT_APP_GLUU_SSO: "disabled"
-      REACT_APP_BACKEND_URL: "http://localhost:8080"
-      REACT_APP_MATOMO_LINK: "http://matomo"
+      REACT_APP_MATOMO_LINK: "matomo"
       REACT_APP_DATA_CATALOG_LINK: "http://data-catalog.local:8443"
       REACT_APP_CLASSIFICATION_BANNER: "UNCLASSIFIED"
       REACT_APP_CLASSIFICATION_BANNER_COLOR: "GREEN"
@@ -21,12 +22,21 @@ services:
       GAMECHANGER_DEMO_DEPLOYMENT: "true"
       GAMECHANGER_DEMO_USER: "007"
       GAMECHANGER_DISABLE_STATS_API: "true"
+      GAMECHANGER_ELASTICSEARCH_HOST: "elasticsearch"
+      GAMECHANGER_ELASTICSEARCH_PROTOCOL: "http"
+      GAMECHANGER_ELASTICSEARCH_PORT: 9200
+      GAMECHANGER_ELASTICSEARCH_INDEX: "gamechanger_original"
+      GAMECHANGER_ELASTICSEARCH_USER: "elasticsearch"
+      GAMECHANGER_ELASTICSEARCH_PASSWORD: "elasticsearch"
       S3_REGION: "us-east-1"
       S3_IS_MINIO: "true"
       S3_ACCESS_KEY: "dev-access-key"
       S3_SECRET_KEY: "dev-secret-key"
       S3_ENDPOINT: "http://s3-server:9000"
-
+      MYSQL_HOST_MATOMO: "mysql"
+      MYSQL_USER_MATOMO: "root"
+      MYSQL_PASSWORD_MATOMO: "password"
+#      CHOKIDAR_USEPOLLING: "true"
     depends_on:
       - redis
       - postgres
@@ -40,44 +50,40 @@ services:
       - "-ec"
     command:
       - |
-        mkdir -p /opt/app-root/src/secrets/
-        openssl genrsa -out /opt/app-root/src/secrets/tls_key.key 4096
-        openssl req -x509 -sha256 -days 3650 \
-          -key /opt/app-root/src/secrets/tls_key.key \
-          -out /opt/app-root/src/secrets/tls_cert.cer \
-          -subj "/CN=gamechanger.local" \
-          -addext 'subjectAltName=DNS:*.local,DNS:*.com,DNS:*'
-
-        cp /opt/app-root/src/secrets/tls_cert.cer /opt/app-root/src/secrets/ca_bundle.pem
-
-        export SAML_CERT="$$(openssl rsa -in /opt/app-root/src/secrets/tls_key.key -pubout)"
-
-        /opt/app-root/src/generateCombinedEnv.sh > ./.env
-
+        cd /opt/app-root/src/backend
         node index.js
 
   ml-api:
     image: "${GC_ML_API_IMAGE}"
     container_name: "${COMPOSE_PROJECT_NAME:-gc}-ml-api"
-    user: "1001:1001"
+    user: "1000:1000"
     depends_on:
       - redis
     environment:
-      ENV_TYPE: DEVLOCAL
+      ENV_TYPE: "PROD"
       DOWNLOAD_DEP: "false"
       REDIS_HOST: redis
       REDIS_PORT: 6379
+      ES_HOST: elasticsearch
+      ES_PORT: 9200
+      ES_USER: elasticsearch
+      ES_PASSWORD: elasticsearch
+      GC_WEB_USER: "007"
+      GC_WEB_HOST: gc-web
+      GC_WEB_PORT: 8080
+      GC_WEB_USER: "007"
     ports:
       - "5000:5000"
     volumes:
-      - "ml-api-data:/opt/app-root/src/gamechangerml/models"
+      - "/mnt/extra/models:/opt/app-root/src/gamechangerml/models:rw"
     networks:
       - app-net
 
   redis:
     image: "${GC_REDIS_IMAGE}"
     container_name: "${COMPOSE_PROJECT_NAME:-gc}-redis"
-    user: root
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
     ports:
       - "6379:6379"
     volumes:
@@ -88,7 +94,7 @@ services:
 
   neo4j:
     image: "${GC_NEO4J_IMAGE}"
-    build: 
+    build:
       context: "./deploy/build/gamechanger-neo4j-plugin/"
       dockerfile: "./docker/debian.Dockerfile"
     container_name: "${COMPOSE_PROJECT_NAME:-gc}-neo4j"
@@ -112,18 +118,23 @@ services:
     volumes:
       - "es-data:/usr/share/elasticsearch/data"
     environment:
-      - "discovery.type=single-node"
+      discovery.type: single-node
+      ELASTIC_PASSWORD: elasticsearch
+      ES_JAVA_OPTS: -Xms4096m -Xmx4096m
     networks:
       - app-net
 
   kibana:
     image: "${GC_KIBANA_IMAGE}"
     container_name: "${COMPOSE_PROJECT_NAME:-gc}-kibana"
-    user: root
     depends_on:
       - elasticsearch
     ports:
       - "5601:5601"
+    volumes:
+      - ./deploy/docker-compose/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro
+    environment:
+      - KIBANA_SYSTEM_PASSWORD=elasticsearch
     networks:
       - app-net
 
@@ -135,7 +146,7 @@ services:
       - "9000:9000"
     volumes:
       - "s3-data:/s3-data"
-    environment: 
+    environment:
       - "MINIO_ACCESS_KEY=dev-access-key"
       - "MINIO_SECRET_KEY=dev-secret-key"
       - "MINIO_REGION_NAME=us-east-1"
@@ -153,11 +164,12 @@ services:
       - "5432:5432"
     volumes:
       - "pg-data:/pg-data"
-    environment: 
+    environment:
       - "POSTGRES_USER=postgres"
       - "POSTGRES_PASSWORD=password"
       - "POSTGRES_DB=postgres"
       - "PGDATA=/pg-data"
+#      - "POSTGRES_HOST_AUTH_METHOD=trust"
     networks:
       - app-net
 
@@ -166,3 +178,37 @@ services:
     container_name: "${COMPOSE_PROJECT_NAME:-gc}-data-pipelines"
     command:
       - "true"
+
+  mysql:
+    image: mysql:8.0
+    container_name: mysql
+    environment:
+      - MYSQL_ROOT_PASSWORD=password
+      - MYSQL_DATABASE=matomo
+    command: --default-authentication-plugin=mysql_native_password
+    volumes:
+      - mysql:/var/lib/mysql
+    restart: always
+    networks:
+      - app-net
+
+  matomo:
+    image: matomo:4.3
+    container_name: matomo
+    environment:
+      - MATOMO_DATABASE_HOST=mysql
+      - MATOMO_DATABASE_USERNAME=root
+      - MATOMO_DATABASE_PASSWORD=password
+      - MATOMO_DATABASE_DBNAME=matomo
+    depends_on:
+      - mysql
+    ports:
+      - 80:80
+    volumes:
+      - matomo:/var/www/html
+    networks:
+      - app-net
+
+volumes:
+    mysql:
+    matomo: