Merge pull request #212 from klips-project/contourLines

feat: init contour-lines worker

README.md

@@ -150,8 +150,8 @@ The Docker images are built automatically by a GitHub Action. For local developm
 ```shell
 # make sure you are in the root of the project
 docker image build \
-  --file src/create-polygons/Dockerfile \
-  --tag create-polygons:my-custom-tag \
+  --file src/create-contour/Dockerfile \
+  --tag create-contour:my-custom-tag \
   .
 ```
 

docker-compose.yml

@@ -97,8 +97,14 @@ services:
       context: ./
       dockerfile: ./src/create-file/Dockerfile
 
+  create-contour:
+    image: ghcr.io/klips-project/mqm-worker/create-contour:latest
+    build:
+      context: ./
+      dockerfile: ./src/create-contour/Dockerfile
+
   reclassify-geotiff:
     image: ghcr.io/klips-project/mqm-worker/reclassify-geotiff:latest
     build:
       context: ./
-      dockerfile: ./src/reclassify-geotiff/Dockerfile
+      dockerfile: ./src/reclassify-geotiff/Dockerfile

src/create-contour/Dockerfile

@@ -0,0 +1,21 @@
+FROM node:16-alpine
+
+RUN apk add --update nodejs npm
+
+COPY src/create-contour/package.json src/create-contour/package-lock.json /home/
+COPY util/start-worker.sh util/wait-for.sh /home/
+RUN chmod +x /home/start-worker.sh /home/wait-for.sh
+
+WORKDIR /home/
+RUN npm install
+COPY src/create-contour/index.js worker/
+COPY src/create-contour/get-client.js worker/
+COPY src/create-contour/add-to-table.js worker/
+COPY src/create-contour/contour.js worker/
+COPY src/workerTemplate.js .
+COPY src/create-contour/child-logger.js worker/
+COPY src/logger.js .
+
+HEALTHCHECK CMD pgrep node || exit 1
+
+CMD ["./start-worker.sh"]

src/create-polygons/README.md → src/create-contour/README.md

@@ -4,10 +4,10 @@
 Manages archiving of incoming data.
 
 ### Mechanism
-- Gets array of polygons using the contour_polygons-process
+- Gets array of lines using the contour_polygons-process
 - connects to postgres-database
 - creates new table in postgres-database (if it does not exist)
-- loops through polygons and adds them to table
+- loops through lines and adds them to table
 
 - ToDo: Delete older data
 - ToDo: Add worker to publish on geoserver in WFS
@@ -16,6 +16,7 @@ Manages archiving of incoming data.
 
 1. path to input file in cog webspace (fileUrlOnWebspace)
 2. file name (fileNameWithSuffix)
+3. Interval
 
 ## Example Worker Job JSON
 
@@ -24,10 +25,11 @@ Manages archiving of incoming data.
     "job": [
       {
         "id": 1,
-        "type": "create-polygons",
+        "type": "create-contour",
         "inputs": [
             "http://nginx/cog/dresden/dresden_temperature/dresden_20240202T0300Z.tif",
-            "dresden_20240202T0300Z.tif"
+            "dresden_20240202T0300Z.tif",
+            "2"
           ]
       }
     ]
@@ -58,5 +60,5 @@ Manages archiving of incoming data.
 ### Send example job
 
 ```bash
-cat workflows/create-polygon-test.json | rabbitmqadmin -u rabbit -p rabbit publish exchange=amq.default routing_key=dispatcher
+cat workflows/create-contour.json | rabbitmqadmin -u rabbit -p rabbit publish exchange=amq.default routing_key=dispatcher
 ```

src/create-contour/add-to-table.js

@@ -0,0 +1,32 @@
+import { getClient } from './get-client.js';
+import logger from './child-logger.js';
+
+// Add data to table
+export const addData = async (
+    datasetTimestamp,
+    contourLine,
+    region
+) => {
+    // Check if line-vector contains all necessary properties
+    if (!('geometry' in contourLine)) {
+        // timestamp of dataset not valid
+        logger.error('Object is missing geometry');
+        throw 'Object is missing geometry.';
+    }
+
+    if (!('properties' in contourLine)) {
+        // timestamp of dataset not valid
+        logger.error('Object is missing properties');
+        throw 'Object is missing properties.';
+    }
+
+    // Add to table
+    const client = await getClient();
+    // TODO process.argv can potentially be removed
+    const timestamp = process.argv[2] ?? datasetTimestamp;
+    const geom = process.argv[2] ?? contourLine.geometry;
+    const temp = process.argv[2] ?? contourLine.properties.TEMP;
+    let insertRow = await client.query(`INSERT INTO ${region}_contourLines(timestamp, geom, temp) VALUES($1);`, [timestamp, geom, temp]);
+    logger.info(`Inserted ${insertRow.rowCount} row`);
+    await client.end();
+};

src/create-contour/child-logger.js

@@ -0,0 +1,3 @@
+import { logger } from '../logger.js';
+
+export default logger.child({ type: 'create-contour' });

src/create-contour/contour.js

@@ -0,0 +1,48 @@
+import { exec } from 'child_process';
+import fs from 'fs';
+
+/**
+ * Executes a shell command and return it as a Promise.
+ * Kudos to https://ali-dev.medium.com/how-to-use-promise-with-exec-in-node-js-a39c4d7bbf77
+ *
+ * @param cmd {String} The command to execute
+ * @return {Promise<String>} A Promise returning the console output
+ */
+const execShellCommand = (cmd) => {
+    return new Promise((resolve, reject) => {
+        exec(cmd, (error, stdout, stderr) => {
+            if (error) {
+                console.warn(error);
+                reject(error);
+            }
+            resolve(stdout ? stdout : stderr);
+        });
+    });
+}
+
+/**
+ * Reclassify a GeoTIFF according to custom levels.
+ *
+ * Relies on GRASS
+ *
+ * @param {String} inputPath The path of the GeoTIFF to convert
+ * @param {String} fileName The name of the GeoTIFF
+ * @param {String} interval Elevation interval between contours
+ *
+ * @returns {Promise<String>} A Promise that resolves to the console output of the underlying GDAL process
+ *
+ * @throws If provided paths are invalid or conversion fails, an error is thrown
+ */
+const createContourLines = async (inputPath, interval) => {
+    // valdate inputPath
+    if (! await fs.existsSync(inputPath)) {
+        throw `Input file does not exist: ${inputPath}/`;
+    }
+
+    // build command for sub-process
+    const contourCmd = `gdal_contour -b 2 -a TEMP -i ${interval} -f "GeoJSON" /${inputPath} /tmp/output.geojson`;
+
+    return await execShellCommand(contourCmd);
+}
+
+export default createContourLines;

src/create-polygons/index.js → src/create-contour/index.js

@@ -7,6 +7,7 @@ import fetch from 'node-fetch';
 import dayjs from 'dayjs';
 import customParseFormat from 'dayjs/plugin/customParseFormat.js';
 import utc from 'dayjs/plugin/utc.js';
+import createContourLines from './contour.js';
 dayjs.extend(customParseFormat);
 dayjs.extend(utc);
 
@@ -16,34 +17,20 @@ const rabbitHost = process.env.RABBITHOST;
 const rabbitUser = process.env.RABBITUSER;
 const rabbitPass = process.env.RABBITPASS;
 
-const fetchPolygons = async (fileUrlOnWebspace) => {
-    const body = {
-        inputs: {
-            cogUrl: fileUrlOnWebspace,
-            interval: 1,
-            bands: [1, 2, 3]
-        }
-    };
-    const url = 'https://klips-dev.terrestris.de/processes/contour-polygons/execution';
-    const response = await fetch(url, {
-        body: JSON.stringify(body),
-        method: 'POST',
-        headers: {
-            'Content-Type': 'application/json'
-        }
-    });
-
-
-    if (!response.ok) {
-        throw new Error(`HTTP error status: ${response.status}`);
-    }
 
-    return await response.json();
-};
+const contourLinesWorker = async (workerJob, inputs) => {
+    // todo put these information into converter.ts
+    const inputPath = inputs[0];
+    const interval = inputs[2];
+
+    await createContourLines(inputPath, interval);
+
+    // array aus multiLines als geoJSON
+    // todo check if it needs a relative path
+    const contourLines = fetch("/tmp/output.geojson")
+        .then((response) => response.json())
+        .then(data => { return (data) })
 
-const polygonsWorker = async (workerJob, inputs) => {
-    // array aus multipolygonen (geoJSON?)
-    const polygons = await fetchPolygons(inputs[0]);
 
     // get region and timestamp from input (example format: langenfeld_20230629T0500Z.tif)
     const regex = /^([^_]+)_(\d{8}T\d{4}Z)/;
@@ -61,7 +48,7 @@ const polygonsWorker = async (workerJob, inputs) => {
     // get timestamp for current hour
     //const currentTimestamp = dayjs.utc().startOf('hour');
 
-    if (datasetTimestamp ) {
+    if (datasetTimestamp) {
         // timestamp of dataset not valid
         logger.info('Could not parse dataset timestamp.');
         throw 'Could not parse dataset timestamp.';
@@ -72,12 +59,11 @@ const polygonsWorker = async (workerJob, inputs) => {
     (async () => {
         const client = await getClient();
         let createTableQuery = `
-    CREATE TABLE IF NOT EXISTS ${region}_polygons(
+    CREATE TABLE IF NOT EXISTS ${region}_contourLines(
       id BIGSERIAL PRIMARY KEY NOT NULL ,
       timestamp timestamp without timezone,
       geom geometry,
       temp number,
-      band int,
     );
   `;
         const res = await client.query(createTableQuery);
@@ -88,9 +74,9 @@ const polygonsWorker = async (workerJob, inputs) => {
 
 
     // Add rows to table
-    polygons.forEach(polygon => addData(
+    contourLines.features.forEach(contourLine => addData(
         datasetTimestamp,
-        polygon,
+        contourLine,
         region
     ));
 
@@ -102,9 +88,9 @@ const polygonsWorker = async (workerJob, inputs) => {
 (async () => {
     try {
         // Initialize and start the worker process
-        await initialize(rabbitHost, rabbitUser, rabbitPass, workerQueue, resultQueue, polygonsWorker);
+        await initialize(rabbitHost, rabbitUser, rabbitPass, workerQueue, resultQueue, contourLinesWorker);
     } catch (error) {
         logger.error({ error: error }, `Problem when initializing`);
     }
 })();
-export default polygonsWorker;
+export default contourLinesWorker;

src/packagesToBuild.json

@@ -1 +1 @@
-["download-file","gunzip-file","geoserver-publish-sld","geoserver-create-and-apply-sld","geoserver-publish-layer-from-db","geonetwork-publish-metadata","send-email","dispatcher","zip-handler","upload-file", "geoserver-publish-geotiff", "geotiff-validator", "send-mattermost-message", "geoserver-publish-imagemosaic", "geoserver-create-imagemosaic-datastore", "create-file", "geotiff-optimizer", "dataset-rotation", "dataset-archive", "reclassify-geotiff"]
+["download-file","gunzip-file","geoserver-publish-sld","geoserver-create-and-apply-sld","geoserver-publish-layer-from-db","geonetwork-publish-metadata","send-email","dispatcher","zip-handler","upload-file", "geoserver-publish-geotiff", "geotiff-validator", "send-mattermost-message", "geoserver-publish-imagemosaic", "geoserver-create-imagemosaic-datastore", "create-file", "geotiff-optimizer", "dataset-rotation", "dataset-archive", "create-contour", "reclassify-geotiff"]