Refactor app setup and remove build app

.github/workflows/main.yml

@@ -45,8 +45,11 @@ jobs:
         pip install --upgrade pip
         pip install -r requirements.txt
 
+    - name: Build app 🔧
+      run: make app
+
     - name: Run make 🔧
-      run: make
+      run: make prod
 
     - name: Deploy 🚀
       uses: JamesIves/github-pages-deploy-action@v4

.gitignore

@@ -8,9 +8,7 @@ __pycache__/
 .pytest_cache
 
 /env
-/venv
+/*.sh
 
 /output
 .Rproj.user
-
-deploy*.sh

Makefile

@@ -1,13 +1,15 @@
+export NVM_DIR="${PWD}/app/nvm"
+
+.PHONY: prod dev serve assets links csvtables definitions glossary pattern protocol schema tree \
+	typos app watch clean distclean
+
 prod: assets csvtables definitions glossary pattern protocol schema tree
 
 dev: links csvtables definitions glossary pattern protocol schema tree
 
 serve:
 	python3 -m http.server -b 127.0.0.1 8080 -d output
 
-clean:
-	rm -fr output
-
 assets:
 	python3 build/assets.py
 
@@ -38,4 +40,16 @@ tree:
 typos:
 	typos --write-changes --force-exclude
 
-.PHONY: prod dev serve clean assets links csvtables definitions glossary pattern protocol schema tree typos
+app:
+	make -C app
+
+watch:
+	make -C app watch
+
+clean:
+	rm -fr output
+
+distclean:
+	rm -fr output
+	rm -fr env
+	make -C app clean

README.md

@@ -14,29 +14,30 @@ your updates will be visible at `https://protocol.isimip.org`.
 As a rule, the sector-specific text should be kept to a minimum and cover
 as much structure as possible by machine-readable code under [definitions](definitions/).
 
-The React JavaScript app is build in a separate step which are described under [app/README.md]([app/README.md]).
-
 Setup
 -----
 
 Building the protocol needs git and a recent Python version (> 3.6). The installation of Python (and its developing packages), however differs from operating system to operating system. Instructions can be found [here](https://github.com/ISI-MIP/isimip-qc/blob/main/README.md#prerequisites).
 
-If you work with different Python applications, we recommend to create a virtual environment for the protocol.
+A `Makefile` is provided to help with the installation process.
+
+If you work with different Python applications, we recommend to create a virtual environment for the protocol:
 
 ```bash
-# setup venv on Linux/macOS/Windows WSL
 python3 -m venv env
-source env/bin/activate
+source env/bin/activate  # the env needs to be sourced everytime you use a new terminal
+```
 
-# setup venv on Windows cmd
-python -m venv env
-call env\Scripts\activate.bat
+The Python requirements are installed using:
+
+```bash
+make install
 ```
 
-The Python dependencies are installed using:
+The JavaScript part of the protocol needs to be build using NodeJS and Webpack. For convenience this can be done by using only:
 
 ```bash
-pip install -r requirements.txt
+make app
 ```
 
 
@@ -45,25 +46,18 @@ Build
 
 ```bash
 make                  # should work on Linux/macOS
-make dev              # like make, but for development of the JavaScript app
+make dev              # like make, but lining the front-end assets for development
 
 make serve            # starts a http server on port :8000 so that you can access the protocol in your browser
-
-sh tools/build.sh     # Linux/macOS/WSL
-sh tools/serve.sh     # Linux/macOS/WSL, start http server
-call tools/build.cmd  # Windows cmd
-call tools/serve.cmd  # Windows cmd, start http server
 ```
 
-On Windows, a double click on `tools/build.cmd` should also build the protocol (unless you use a virtual environment).
-
 The output files are located in `output`. The files, e.g. `index.html` can opened with a web browser.
 
 
 Development server
 ------------------
 
-The command `make serve` will open a local webserver on port `:8000`. The protocol can than be accessed at http://localhost:8000 from a browser.
+The command `make serve` will open a local webserver on port `:8000`. The protocol can than be accessed at http://localhost:8080 from a browser.
 
 
 Editing
@@ -79,96 +73,58 @@ The interactive tables have the following syntax:
 
 where `number` is simply the table number to be displayed in the caption and `identifier` will not only connect the table to its definition file (see below), but will also define which JavaScript component to use. Changes of the layout of a table or the creation of new tables require work on the [app](app).
 
-The definition JSON files however can be changed without touching the JavaScript source code. Each definition is a list of JSON objects. Every object must have an attribute `specifier` which is used to refer to it in other objects/tables but also in file names. An example for a relatively simple definition file is [definitions/bias_correction.json](definitions/bias_correction.json):
+The definition YAML files however can be changed without touching the JavaScript source code. Each definition is a list of objects. Every object must have an attribute `specifier` which is used to refer to it in other objects/tables but also in file names. An example for a relatively simple definition file is [definitions/soc_scenario.yaml](definitions/soc_scenario.yaml):
 
 ```
-[
-  {
-    "specifier": "nobc",
-    "description": "Indicates that no bias correction was performed on the climate data (e.g. ocean data)."
-  },
-  {
-    "specifier": "localbc",
-    "description": "Refers to local data from weather stations used for the bias-correction in e.g. the forest sector.",
-    "sectors": [
-      "forestry"
-    ]
-  },
-  {
-    "specifier": "ewembi",
-    "description": "Refers to EWEMBI data used for the bias-correction globally on a 0.5° grid."
-  },
-  {
-    "specifier": "ewembi-isimip3basd",
-    "description": "Refers to EWEMBI data used for the bias-correction globally on a 0.5°, using improved bias-correction methods (Lange 2018, doi: 10.5194/esd-9-627-2018), and with statistical downscaling (instead of interpolation) of GCM data to the 0.5° grid prior to bias-correction."
-  }
-]
+- specifier: histsoc
+  description: >-
+    Varying direct human influences in the historical period.
+  description_note: Please label your model run `histsoc` **even if** it only partly
+    accounts for varying direct human forcings while another part of the the direct
+    human forcing is considered constant or is ignored.
+  simulation_rounds:
+  - ISIMIP3a
+  - ISIMIP3b
+
+- specifier: 1850soc
+  description: >-
+    Fixed year-1850 direct human influences (e.g. land use, nitrogen deposition and
+    fertilizer input, fishing effort).
+  description_note: Please label your simulations `1850soc` if they do not at all
+    account for historical changes in direct human forcing, but they do represent
+    constant year-1850 levels of direct human forcing for at least some direct human
+    forcings. This scenario may be thought of an approximation of pre-industrial levels
+    of human impacts.
+  simulation_rounds:
+  - ISIMIP3b
+  sectors:
+  - agriculture
+  - biodiversity
+  - biomes
+  - diarrhea
+  - fire
+  - health
+  - coastal
+  - labour
+  - lakes_global
+  - lakes_local
+  - peat
+  - permafrost
+  - water_global
+  - water_regional
+  - water_quality
+
+...
 ```
 
-Here `localbc` only applies to the `forestry` sector, while the other objects are used in every sector. The longest and most complicated definition is [definitions/variable.json](definitions/variable.json):
-
-```
-[
-  {
-    "specifier": "qtot",
-    "title": "Runoff",
-    "unit": "kg m-2 s-1",
-    "resolution": "grid cell",
-    "frequency": {
-      "biomes": "monthly",
-      "permafrost": "monthly",
-      "water_global": "daily",
-      "water_regional": "daily"
-    },
-    "comment": "Total runoff leaving the land portion of the grid cell",
-    "sectors": [
-      "water_global",
-      "water_regional",
-      "biomes",
-      "permafrost"
-    ]
-  },
-
-  ...
-]
-```
-
-Some attributes (e.g. `frequency`) can have objects as value, which the are evaluated for the particular sector. For reference, the full list for `simulation_rounds` and `sectors` are:
-
-```
-"simulation_rounds": [
-  "ISIMIP3a",
-  "ISIMIP3b"
-],
-```
-
-```
-"sectors": [
-  "agriculture",
-  "biodiversity",
-  "biomes",
-  "coastal",
-  "diarrhea",
-  "energy",
-  "forestry",
-  "health",
-  "labour",
-  "lakes_global",
-  "lakes_local",
-  "marine-fishery_global",
-  "marine-fishery_regional",
-  "permafrost",
-  "water_global",
-  "water_regional"
-]
-```
+Here `1850soc` only applies to the givem set of sectors and only to `ISIMIP3b`, while `histsoc` is used both in `ISIMIP3a` and `ISIMIP3b` and in every sector. Some attributes (e.g. `frequency` in `definitions/variable`) can have objects as value, which the are evaluated for the particular sector.
 
 In order to add a new sector, the following steps need to be taken:
 
-* Add the sector with `specifier` and `title` to `definitions/sector.json`.
-* Add `pattern/ISIMIP3a/OutputData/<sector>.json` and `pattern/ISIMIP3b/OutputData/<sector>.json` with the file patterns for the new sector.
-* Add new `variable` group(s) to `definitions/group.json`.
-* Add sector variables to `definitions/variable.json` and/or update existing variables with the new sector.
+* Add the sector with `specifier` and `title` to `definitions/sector.yaml`.
+* Add `pattern/ISIMIP3a/OutputData/<sector>.yaml` and `pattern/ISIMIP3b/OutputData/<sector>.yaml` with the file patterns for the new sector.
+* Add new `variable` group(s) to `definitions/group.yaml`.
+* Add sector variables to `definitions/variable.yaml` and/or update existing variables with the new sector.
 
 
 Printing

app/.gitignore

@@ -1,2 +1,3 @@
+nvm
 node_modules
-dist
+output

app/Makefile

@@ -1,5 +1,24 @@
-prod:
-	source ${NVM_DIR}/nvm.sh; nvm use; npm run build:prod
+export NVM_DIR=$(dir $(realpath $(lastword $(MAKEFILE_LIST))))/nvm
 
-watch:
-	source ${NVM_DIR}/nvm.sh; nvm use; npm run watch
+.PHONY: prod build watch
+
+prod: nvm node_modules
+	. "${NVM_DIR}/nvm.sh"; nvm use; npm run build:prod
+
+build: nvm node_modules
+	. "${NVM_DIR}/nvm.sh"; nvm use; npm run build
+
+watch: nvm node_modules
+	. "${NVM_DIR}/nvm.sh"; nvm use; npm run watch
+
+nvm:
+	mkdir -p ${NVM_DIR}
+	NVM_DIR=${NVM_DIR} PROFILE=/dev/null \
+		bash -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash > /dev/null'
+	. "${NVM_DIR}/nvm.sh"; nvm install
+
+node_modules:
+	. "${NVM_DIR}/nvm.sh"; nvm use; npm ci
+
+clean:
+	rm -rf nvm node_modules

app/package.json

@@ -2,9 +2,10 @@
   "name": "isimip-protocol-3",
   "version": "v1",
   "scripts": {
-    "build:prod": "webpack --config webpack/prod.config.js --mode production",
-    "build": "webpack --config webpack/dev.config.js --mode development",
-    "watch": "webpack --config webpack/dev.config.js --mode development --watch"
+    "build:prod": "webpack --config webpack.config.js --mode production",
+    "build": "webpack --config webpack.config.js --mode development",
+    "watch": "webpack --config webpack.config.js --mode development --watch",
+    "lint": "eslint --ext .js src/"
   },
   "author": "Potsdam Institute for Climate Impact Research",
   "license": "MIT",

app/src/components/Pattern.js

@@ -8,12 +8,16 @@ import SimulationRounds from './badges/SimulationRounds'
 const Pattern = ({ config, patterns }) => {
   const sectors = Object.keys(patterns[config.simulation_round])
   const simple_pattern = (pattern) => {
-    return pattern.replaceAll(/\[.*?\]\+/g, '')
-                  .replaceAll(/\([a-z\|]*?\)/g, '')
-                  .replaceAll('\\d{4}', '').replaceAll('P', '').replaceAll('?', '')
-                  .replaceAll('(<', '<').replaceAll('>)', '>')
-                  .replaceAll(/>_</g, '>@<').replaceAll('(_', '@').replaceAll(').', '.')
-                  .replaceAll('_', '-').replaceAll('@', '_')
+    return pattern
+      .replaceAll(/\[.*?\]\+/g, '')                     // remove [a-z0-9] etc.
+      .replaceAll(/\([a-z\|]*?\)/g, '')                 // ??
+      .replaceAll('\\d{4}', '')                         // remove \d{4} etc.
+      .replaceAll('\?P', '')                            // remove ?P
+      .replaceAll('?', '')                              // remove ?
+      .replaceAll('(<', '<').replaceAll('>)', '>')      // remove parentesis around (< ... >)
+      .replaceAll(/>_/g, '>@').replaceAll(/_</g, '@<')  // replace underscore between identifiers with @
+      .replaceAll('_', '-')                             // replace remaining _ with -
+      .replaceAll('@', '_')                             // replace @ with _
   }
 
   return (

app/src/utils/location.js

@@ -16,7 +16,7 @@ const parseLocation = () => {
 
   if (isEmpty(tokens)) {
     return config
-  } else if (last(tokens).match(/^\d/)) {
+  } else {
     // remove anchor id
     if (!isNil(last(tokens)) && last(tokens).match(/^\d/)) {
       tokens.pop()
@@ -58,7 +58,7 @@ const updateLocation = (config) => {
 const parseAnchor = () => {
   const tokens = splitLocationHash()
 
-  if (!isEmpty(tokens)) {
+  if (isEmpty(tokens)) {
     return null
   } else if (last(tokens).match(/^\d/)) {
     return document.getElementById(last(tokens))