Skip to content

Commit

Permalink
feat(data): validate processes files against a JSON schema. (#869)
Browse files Browse the repository at this point in the history
## 🔧 Problem

We want to be sure that the processes files generated by the data
pipeline are valid.

## 🍰 Solution

Add a [JSON schema](https://json-schema.org/) and validate our files
against it.

## 🏝️ How to test

Check that the CI's green with the new CI action, or locally using:

```
$ npm run db:validate
```
  • Loading branch information
n1k0 authored Dec 19, 2024
1 parent 0712b54 commit 0c5bbe9
Show file tree
Hide file tree
Showing 4 changed files with 262 additions and 0 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/node.js.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ jobs:
- name: Install Node dependencies
run: npm ci --prefer-offline --no-audit

- name: Check JSON db formating
run: npm run db:validate

- name: Install Python dependencies
run: pip install pipenv && pipenv install -d

Expand Down
136 changes: 136 additions & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
"encrypt": "./bin/encrypt",
"db:build": "./bin/build-db && npm run db:check",
"db:check": "elm make src/CheckDb.elm --optimize --output=check-db-app.js 1> /dev/null && node check-db.js",
"db:validate": "npx ajv validate --spec=draft2019 -c ajv-formats -s tests/processes-schema.json -d 'public/data/**/processes**.json'",
"lint:openapi": "npx swagger-cli validate openapi.yaml",
"lint:prettier": "prettier --config .prettierrc --check",
"lint:prettier:all": "npm run lint:prettier -- .",
Expand Down Expand Up @@ -69,6 +70,8 @@
"@parcel/transformer-elm": "^2.13.2",
"@parcel/transformer-image": "^2.13.2",
"@parcel/transformer-sass": "^2.13.2",
"ajv-cli": "^5.0.0",
"ajv-formats": "^3.0.1",
"bootstrap": "^5.3.3",
"concurrently": "^9.1.0",
"elm": "^0.19.1-6",
Expand Down
120 changes: 120 additions & 0 deletions tests/processes-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
{
"$schema": "https://json-schema.org/draft/2019-09/schema#",
"type": "array",
"items": {
"type": "object",
"properties": {
"alias": {
"type": ["string", "null"]
},
"categories": {
"type": "array",
"items": {
"type": "string"
}
},
"comment": {
"type": "string"
},
"density": {
"type": "number",
"minimum": 0
},
"displayName": {
"type": ["string", "null"]
},
"elec_MJ": {
"type": "number",
"minimum": 0
},
"heat_MJ": {
"type": "number",
"minimum": 0
},
"id": {
"type": "string",
"format": "uuid"
},
"sourceId": {
"type": ["string", "null"]
},
"impacts": {
"type": "object",
"properties": {
"acd": { "type": "number" },
"cch": { "type": "number" },
"etf": { "type": "number" },
"etf-c": { "type": "number" },
"fru": { "type": "number" },
"fwe": { "type": "number" },
"htc": { "type": "number" },
"htc-c": { "type": "number" },
"htn": { "type": "number" },
"htn-c": { "type": "number" },
"ior": { "type": "number" },
"ldu": { "type": "number" },
"mru": { "type": "number" },
"ozd": { "type": "number" },
"pco": { "type": "number" },
"pma": { "type": "number" },
"swe": { "type": "number" },
"tre": { "type": "number" },
"wtu": { "type": "number" },
"ecs": { "type": "number" },
"pef": { "type": "number" }
},
"required": [
"acd",
"cch",
"etf",
"etf-c",
"fru",
"fwe",
"htc",
"htc-c",
"htn",
"htn-c",
"ior",
"ldu",
"mru",
"ozd",
"pco",
"pma",
"swe",
"tre",
"wtu",
"ecs",
"pef"
]
},
"name": {
"type": "string"
},
"source": {
"type": "string"
},
"unit": {
"type": "string",
"enum": ["kg", "t⋅km", "kWh", "MJ", "L", "Item(s)", "m2", "m3"]
},
"waste": {
"type": "number",
"minimum": 0,
"maximum": 1
}
},
"required": [
"categories",
"comment",
"density",
"elec_MJ",
"heat_MJ",
"id",
"impacts",
"name",
"source",
"unit",
"waste"
]
}
}

0 comments on commit 0c5bbe9

Please sign in to comment.