Merge pull request #6 from maartenplieger/master

Added travis, Oauth

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,2 @@
+distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip
+

.travis.yml

@@ -0,0 +1,3 @@
+language: java
+sudo: false
+script: mvn clean verify

Dockerfile

@@ -2,6 +2,12 @@ FROM centos:7
 
 MAINTAINER Adaguc Team at KNMI <[email protected]>
 
+VOLUME /config
+VOLUME /data
+
+#TODO: perhaps host on a standard port (443)
+EXPOSE 9000
+
 RUN yum update -y && yum install -y \
     epel-release
 
@@ -24,14 +30,10 @@ RUN yum update -y && yum install -y \
     postgresql-devel \
     postgresql-server \
     gdal-devel \
-    tomcat \ 
+    tomcat \
     maven
-
-RUN mkdir /src
-
-
-
 
+RUN mkdir /src
 WORKDIR /src
 # Configure postgres
 RUN mkdir /postgresql
@@ -41,77 +43,49 @@ RUN chown postgres: /var/log/postgresql.log
 RUN runuser -l postgres -c "initdb -D /postgresql"
 
 # Install adaguc-server
+# TODO: switch to release version and/or Conda package if possible and available
 WORKDIR /src
 RUN curl -L  https://github.com/KNMI/adaguc-server/archive/master.tar.gz > adaguc-server.tar.gz
 RUN tar xvf adaguc-server.tar.gz
-WORKDIR /src/adaguc-server-master
+RUN mv /src/adaguc-server-master /src/adaguc-server
+WORKDIR /src/adaguc-server
 RUN bash compile.sh
 
 # install pywps
 WORKDIR /src
 RUN curl -L -O https://github.com/geopython/pywps/archive/pywps-3.2.5.tar.gz
 RUN tar xvf pywps-3.2.5.tar.gz
 
-# Install adaguc-services
+# Install adaguc-services from the context
 WORKDIR /src
 RUN mkdir adaguc-services
-COPY . /src/adaguc-services/
-
+COPY src/ /src/adaguc-services/src/
+COPY pom.xml /src/adaguc-services/pom.xml
 WORKDIR /src/adaguc-services
-RUN mvn package 
-
-RUN mkdir /keystore/
-RUN keytool -genkey -noprompt -keypass password -alias tomcat -keyalg RSA -storepass password -keystore /keystore/c4i_keystore.jks  -dname CN=compute-test.c3s-magic.eu
-
-# Set up data dir, this is also configured in adaguc.docker.xml
-RUN mkdir /data/
-
-#Setup directory for automatic visualization of NetCDF's
-RUN mkdir /data/adaguc-autowms
-
-#Setup directory for visualization of ADAGUC datasets
-RUN mkdir /data/adaguc-datasets
-
-RUN mkdir /data/adaguc-datasets-spaces/
-
-RUN mkdir /src/wpsoutputs
-RUN mkdir /src/adaguc-services-tmp/
-
-EXPOSE 9000
-
-
-# Install certificates
-RUN  mkdir -p /config/
-WORKDIR /config/
-# RUN curl -L https://raw.githubusercontent.com/ESGF/esgf-dist/master/installer/certs/esg_trusted_certificates.tar > esg_trusted_certificates.tar
-RUN curl -L https://raw.githubusercontent.com/ESGF/esgf-dist/master/installer/certs/esg-truststore.ts > esg-truststore.ts
-
-
-#RUN tar -xvf esg_trusted_certificates.tar
-#RUN mv esg_trusted_certificates certificates
-
+RUN mvn package
 
 # Configure adaguc-services
-COPY ./docker/adaguc-services-config.xml /root/adaguc-services-config.xml 
-ENV ADAGUC_SERVICES_CONFIG=/root/adaguc-services-config.xml 
+ENV ADAGUC_SERVICES_CONFIG=/config/adaguc-services-config.xml
 
 WORKDIR /src/adaguc-services
 CMD echo "Starting POSTGRESQL DB" && \
     runuser -l postgres -c "pg_ctl -D /postgresql -l /var/log/postgresql.log start" && \
-    sleep 1 && \ 
-    mkdir -p /data/adaguc-autowms/ && \ 
-    mkdir -p /data/adaguc-datasets/ && \ 
-    cp /src/adaguc-server-master/data/datasets/testdata.nc /data/adaguc-autowms/ && \
-    cp /src/adaguc-server-master/data/config/datasets/dataset_a.xml /data/adaguc-datasets/ && \
+    sleep 1 && \
+    mkdir -p /data/adaguc-autowms/ && \
+    mkdir -p /data/adaguc-datasets/ && \
+    mkdir -p /data/adaguc-datasets-spaces/ && \
+    mkdir -p /data/wpsoutputs/ && \
+    mkdir -p /data/adaguc-services-tmp/ && \
+    cp /src/adaguc-server/data/datasets/testdata.nc /data/adaguc-autowms/ && \
+    cp /src/adaguc-server/data/config/datasets/dataset_a.xml /data/adaguc-datasets/ && \
     echo "Configuring POSTGRESQL DB" && \
     runuser -l postgres -c "createuser --superuser adaguc" && \
     runuser -l postgres -c "psql postgres -c \"ALTER USER adaguc PASSWORD 'adaguc';\"" && \
     runuser -l postgres -c "psql postgres -c \"CREATE DATABASE adaguc;\"" && \
     echo "Starting TOMCAT Server" && \
     java -jar ./target/adaguc-services-1.0.0-SNAPSHOT.war
 
-# Build with docker build -t adagucservices:alpha .
-# docker run -it -p9000:8080 adagucservices:alpha bash
+
 
 # You can copy NetCDF's / GeoJSONS to your hosts ~/data directory. This will be served through adaguc-server, via the source=<filename> key value pair. testdata.nc is copied there by default. See example URL above.
 

README.md

@@ -1,5 +1,10 @@
 # adaguc-services
-Services for adaguc-server
+
+[![Build Status](https://api.travis-ci.org/KNMI/adaguc-services.svg?branch=master)](https://travis-ci.org/KNMI/adaguc-services)
+[![Download](https://jitpack.io/v/KNMI/adaguc-services.svg)](https://jitpack.io/#KNMI/adaguc-services)
+
+
+Services for adaguc-server and pywps
 
 For setting up development environment:
 
@@ -22,3 +27,4 @@ For creating a new package:
 
 
 
+

adaguc-services-config.xml.example

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<adaguc-services>
+  <!-- Set the environment variable ADAGUCSERVICES_DIR to the top of the folder structure (adagucservices_dir)
+  <userworkspace>{ENV.ADAGUCSERVICES_DIR}/data/adaguc-services-space</userworkspace>
+
+  <external-home-url>https://bhw451.knmi.nl:8090</external-home-url>
+
+  <basedir>{ENV.ADAGUCSERVICES_DIR}/data/adaguc-services-base</basedir>
+
+  <server>
+    <port>8090</port>
+  </server>
+
+  <security>
+    <truststorepassword>changeit</truststorepassword>
+    <truststore>{ENV.ADAGUCSERVICES_DIR}/config/ds-truststore.ts</truststore>
+    <trustrootscadirectory>{ENV.ADAGUCSERVICES_DIR}/src/.globus/certificates/</trustrootscadirectory>
+    <keystore>{ENV.ADAGUCSERVICES_DIR}/keystore/c4i_keystore.jks</keystore>
+    <keystorepassword>password</keystorepassword>
+    <keystoretype>JKS</keystoretype>
+    <keyalias>tomcat</keyalias>
+  </security>
+
+  <adaguc-server>
+    <adagucexecutable>{ENV.ADAGUCSERVICES_DIR}/src/adaguc-server/bin/adagucserver</adagucexecutable>
+    <export>ADAGUC_PATH={ENV.ADAGUCSERVICES_DIR}/src/adaguc-server/</export>
+    <export>ADAGUC_CONFIG={ENV.ADAGUCSERVICES_DIR}/src/adaguc-server/data/config/adaguc.docker.xml</export>
+    <export>ADAGUC_DATARESTRICTION=FALSE</export>
+    <export>ADAGUC_LOGFILE={ENV.ADAGUCSERVICES_DIR}/adaguc-services-tmp/adaguc.autoresource.log</export>
+    <export>ADAGUC_FONT={ENV.ADAGUCSERVICES_DIR}/src/adaguc-server/data/fonts/FreeSans.ttf</export>
+  </adaguc-server>
+
+  <pywps-server>
+    <pywpsexecutable>{ENV.ADAGUCSERVICES_DIR}/src/pywps-pywps-3.2.5/wps.py</pywpsexecutable>
+    <pywpsconfigtemplate>{ENV.ADAGUCSERVICES_DIR}/src/pywps-pywps-3.2.5/pywps/default.cfg</pywpsconfigtemplate>
+    <pywpsoutputdir>{ENV.ADAGUCSERVICES_DIR}/src/wpsoutputs</pywpsoutputdir>
+    <pywpsprocessesdir>{ENV.ADAGUCSERVICES_DIR}/src/pywps-pywps-3.2.5/pywps/processes/</pywpsprocessesdir>
+    <tmp>{ENV.ADAGUCSERVICES_DIR}/src/adaguc-services-tmp</tmp>
+    <export>ADAGUC_CONFIG={ENV.ADAGUCSERVICES_DIR}/src/adaguc-server/adagucserver.xml</export>
+    <export>ADAGUC_PATH={ENV.ADAGUCSERVICES_DIR}/src/adagucserver/</export>
+    <export>ADAGUC_TMP={ENV.ADAGUCSERVICES_DIR}/src/adaguc-services-tmp/</export>
+    <export>ADAGUC_LOGFILE={ENV.ADAGUCSERVICES_DIR}/src/adaguc-services-tmp/adagucserver-wps.log</export>
+    <export>ADAGUC_DATARESTRICTION=FALSE</export>
+    <!--<export>PATH=/src/src/adagucserver/bin/:/src/conda/bin/:/src/conda/bin/:/src/conda/bin/:/src/bin:/src/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin</export>-->
+    <!--<export>PATH=/home/c3smagic/code/KNMI/wps_prov/climexp:/home/c3smagic/code/KNMI/adagucserver/bin/:/home/c3smagic/conda/bin/:/home/c3smagic/conda/bin/:/home/c3smagic/conda/bin/:/home/c3smagic/bin:/home/c3smagic/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin</export>-->
+    <export>PATH={ENV.ADAGUCSERVICES_DIR}/src/adaguc-server/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin</export>
+    <export>PYWPS_TEMPLATES={ENV.ADAGUCSERVICES_DIR}/src/pywps-pywps-3.2.5/pywps/Templates</export>
+    <export>PORTAL_OUTPUT_PATH={ENV.ADAGUCSERVICES_DIR}/src/wpsoutputs/</export>
+    <export>USE_FONTCONFIG=False</export>
+  </pywps-server>
+
+  <basket>
+    <enabled>true</enabled>
+  </basket>
+
+  <joblist>
+    <enabled>true</enabled>
+  </joblist>
+
+  <datasetcatalog>
+    <enabled>true</enabled>
+    <catalogpath>{ENV.ADAGUCSERVICES}/data/adaguc-services-base/catalog</catalogpath>
+  </datasetcatalog>
+
+
+</adaguc-services>

docker/README.md

@@ -0,0 +1,64 @@
+# Docker setup documentation
+
+The docker container in this repo has the following volumes
+
+- `/data` , for all data, will be filled by the service while it is running
+- `/config`, for the configuration, you will have to fill this. see docker/config-template for example versionf of files when available.
+
+## Config file overview and how to get these
+Following commands should be executed inside `config` directory which will be mounted as volume inside the docker container.
+### Certificate
+
+A certificate is needed for the secure (https) connectivity. This needs to then be put into a so-called keystore so tomcat can read the certificate. Simplest option is to generate a keystore with a self-signed certificate:
+
+```sh
+    # example.com = machine you will be using to host the service. Should be a valid dns entry.
+    keytool -genkey -noprompt -keypass password -alias tomcat -keyalg RSA -storepass password -keystore keystore.jks -dname CN=example.com
+```
+
+Alternatively, you can put an existing certificate in a keystore:
+
+    command-goes-here.sh
+
+Or, use letsencrypt
+
+    letsencrypt-something.sh
+
+### Truststore
+
+adaguc-services needs to know which services to trust, for instance when connecting to an external opendap host. This similair to the root-certificates found in every web browser. If needed you can add certificates to the trust store, for instance if you generated a self-signed certificate somewhere.
+
+Simplest option is to download the truststore useb by all esgf nodes:
+
+```sh
+curl -L https://raw.githubusercontent.com/ESGF/esgf-dist/master/installer/certs/esg-truststore.ts > esg-truststore.ts
+```
+
+If you need to, you can add certificates to the truststore. Here is an example of how to add a key, where the key is first extracted from a keystore (e.g. if you are using a self-signed certificate).
+
+```sh
+# export certificate from a keystore to a file called some-certificate.pem
+keytool -export -alias tomcat -rfc -file some-certificate.pem -keystore keystore.jks -storepass password
+
+# put this certificate from some-certificate.pem into the truststore
+keytool -import -v -trustcacerts -alias adagucservicescert -file some-certificate.pem -keystore truststore.ts -storepass changeit -noprompt
+```
+
+If you already have a certificate in the truststore and would like to replace it first delete it:
+```sh
+keytool -delete -alias adagucservicescert  -keystore esg-truststore.ts -storepass changeit -noprompt
+```
+
+
+### Server Config File
+
+ADAGUC-services has a single main config file. See docker/adaguc-services-config.xml for an example suitable for usage in a docker container. The `<external-home-url>` at the top of the file always needs to be changed to the machine running the service (and be equal to the hostname in the certificate, in the example above that would be `example.com`).
+
+## Building and Running Docker container
+```sh
+docker build -t adagucservices .
+docker run -p9000:9000 -v $PWD/config:/config -v $PWD/data:/data adagucservices
+```
+
+You should be able to go with the web browser to `https://localhost:9000/wps`. You may get an authentication error.
+If you're using docker-machine check what ip address it runs on `docker machine ip <default>` and replace `localhost` with it.

docker/adaguc-services-config.xml

@@ -2,7 +2,7 @@
 <adaguc-services>
   <userworkspace>/data/adaguc-services-space</userworkspace>
 
-  <external-home-url>https://andrews-air-3.knmi.nl:9000</external-home-url>
+  <external-home-url>https://example.com:9000</external-home-url>
 
   <basedir>/src/adaguc-services-base</basedir>
 
@@ -14,7 +14,7 @@
     <truststorepassword>changeit</truststorepassword>
     <truststore>/config/esg-truststore.ts</truststore>
     <trustrootscadirectory>/src/.globus/certificates/</trustrootscadirectory>
-    <keystore>/keystore/c4i_keystore.jks</keystore>
+    <keystore>/config/keystore.jks</keystore>
     <keystorepassword>password</keystorepassword>
     <keystoretype>JKS</keystoretype>
     <keyalias>tomcat</keyalias>
@@ -32,20 +32,18 @@
   <pywps-server>
     <pywpsexecutable>/src/pywps-pywps-3.2.5/wps.py</pywpsexecutable>
     <pywpsconfigtemplate>/src/pywps-pywps-3.2.5/pywps/default.cfg</pywpsconfigtemplate>
-    <pywpsoutputdir>/src/wpsoutputs</pywpsoutputdir>
+    <pywpsoutputdir>/data/wpsoutputs</pywpsoutputdir>
     <pywpsprocessesdir>/src/pywps-pywps-3.2.5/pywps/processes/</pywpsprocessesdir>
     <tmp>/src/adaguc-services-tmp</tmp>
     <export>ADAGUC_CONFIG=/src/adaguc-server/adagucserver.xml</export>
     <export>ADAGUC_PATH=/src/adagucserver/</export>
     <export>ADAGUC_TMP=/src/adaguc-services-tmp/</export>
     <export>ADAGUC_LOGFILE=/src/adaguc-services-tmp/adagucserver-wps.log</export>
     <export>ADAGUC_DATARESTRICTION=FALSE</export>
-    <!--<export>PATH=/src/src/adagucserver/bin/:/src/conda/bin/:/src/conda/bin/:/src/conda/bin/:/src/bin:/src/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin</export>-->
-    <!--<export>PATH=/home/c3smagic/code/KNMI/wps_prov/climexp:/home/c3smagic/code/KNMI/adagucserver/bin/:/home/c3smagic/conda/bin/:/home/c3smagic/conda/bin/:/home/c3smagic/conda/bin/:/home/c3smagic/bin:/home/c3smagic/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin</export>-->
-    <export>PATH=/src/adaguc-server/bin/:/src/conda/bin/:/src/conda/bin/:/src/conda/bin/:/src/bin:/src/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin</export>
+        <export>PATH=/src/adaguc-server/bin/:{ENV.PATH}</export>
     <export>PYWPS_TEMPLATES=/src/pywps-pywps-3.2.5/pywps/Templates</export>
-    <export>PORTAL_OUTPUT_PATH=/src/wpsoutputs/</export>
+    <export>PORTAL_OUTPUT_PATH=/data/wpsoutputs/</export>
     <export>USE_FONTCONFIG=False</export>
   </pywps-server>
 
-</adaguc-services>
+</adaguc-services>