Merged changes from master.

Materials-Consortia · Nov 30, 2022 · 3a908bc · 3a908bc
2 parents 4c46f55 + 105b501
commit 3a908bc
Show file tree

Hide file tree

Showing 127 changed files with 1,312 additions and 1,046 deletions.
diff --git a/.github/fly.toml b/.github/fly.toml
@@ -0,0 +1,38 @@
+# fly.toml file generated for broken-field-1976 on 2022-11-21T21:37:55Z
+
+app = "optimade"
+kill_signal = "SIGINT"
+kill_timeout = 5
+processes = []
+
+[env]
+
+[experimental]
+  allowed_public_ports = []
+  auto_rollback = true
+
+[[services]]
+  http_checks = []
+  internal_port = 5000
+  processes = ["app"]
+  protocol = "tcp"
+  script_checks = []
+  [services.concurrency]
+    hard_limit = 25
+    soft_limit = 20
+    type = "connections"
+
+  [[services.ports]]
+    force_https = true
+    handlers = ["http"]
+    port = 80
+
+  [[services.ports]]
+    handlers = ["tls", "http"]
+    port = 443
+
+  [[services.tcp_checks]]
+    grace_period = "1s"
+    interval = "15s"
+    restart_limit = 0
+    timeout = "2s"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -7,6 +7,9 @@ on:
     - master
     - 'push-action/**'
 
+env:
+  PYTEST_ADDOPTS: "--color=yes"
+
 # Cancel running workflows when additional changes are pushed
 # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-a-fallback-value
 concurrency:
@@ -57,6 +60,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
 
     - name: Run pre-commit
@@ -82,6 +86,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
 
     - name: Pass generated OpenAPI schemas through validator.swagger.io
@@ -162,7 +167,7 @@ jobs:
         ports:
         - 5432:5432
       elasticsearch:
-        image: elasticsearch:7.17.1
+        image: elasticsearch:7.17.7
         ports:
           - 9200:9200
           - 9300:9300
@@ -191,25 +196,29 @@ jobs:
         pip install -r requirements-dev.txt
         pip install -r requirements-http-client.txt
 
-    - name: Run all tests (using `mongomock`)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/
+    - name: Run non-server tests
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/ --ignore tests/server
+
+    - name: Install latest server dependencies
+      run: pip install -r requirements-server.txt
+
+    - name: Run server tests (using `mongomock`)
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongomock'
 
-
     - name: Run server tests (using a real MongoDB)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'mongodb'
 
     - name: Run server tests (using Elasticsearch)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers
       env:
         OPTIMADE_DATABASE_BACKEND: 'elastic'
 
     - name: Install adapter conversion dependencies
-      run: |
-        pip install -r requirements-client.txt
+      run: pip install -r requirements-client.txt
 
     - name: Setup environment for AiiDA
       env:
@@ -246,22 +255,22 @@ jobs:
         coverage run --append --source optimade optimade/client/cli.py \
            --filter 'nsites = 1' \
            --output-file test_get_async.json \
-           https://optimade.herokuapp.com
+           https://optimade.fly.dev
         test test_get_async.json
 
         coverage run --append --source optimade optimade/client/cli.py \
           --filter 'nsites = 1' \
           --count \
           --output-file test_count.json \
-          https://optimade.herokuapp.com
+          https://optimade.fly.dev
         test test_count.json
 
         coverage run --append --source optimade optimade/client/cli.py \
           --no-async \
           --filter 'nsites = 1' \
           --count \
           --output-file test_count_no_async.json \
-          https://optimade.herokuapp.com
+          https://optimade.fly.dev
         test test_count_no_async.json
 
         diff test_count_no_async.json test_count.json
@@ -302,6 +311,7 @@ jobs:
         pip install -U setuptools wheel
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-server.txt
         pip install -r requirements-dev.txt
         pip install -r requirements-http-client.txt
         pip install -r requirements-docs.txt

diff --git a/.github/workflows/ci_cd_updated_master.yml b/.github/workflows/ci_cd_updated_master.yml
@@ -135,23 +135,25 @@ jobs:
         sleep: 15
         force: ${{ env.FORCE_PUSH }}
 
-  deploy-heroku:
+  deploy-fly:
+    env:
+        FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+        OPTIMADE_CONFIG_FILE: /app/tests/test_config.json
+        OPTIMADE_BASE_URL: https://optimade.fly.dev
     runs-on: ubuntu-latest
     if: github.repository_owner == 'Materials-Consortia'
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
-
-    - name: Deploy to Heroku
-      uses: akhileshns/[email protected]
       with:
-          heroku_api_key: ${{secrets.HEROKU_API_KEY}}
-          heroku_app_name: "optimade"
-          heroku_email: ${{secrets.HEROKU_EMAIL}}
-          branch: ${{env.DEFAULT_REPO_BRANCH}}
-      env:
-          HD_OPTIMADE_CONFIG_FILE: /app/tests/test_config.json
-          HD_OPTIMADE_BASE_URL: https://optimade.herokuapp.com
+        fetch-depth: 0
+        submodules: true
+
+    - name: Set up Fly
+      uses: superfly/flyctl-actions/setup-flyctl@master
+
+    - name: Deploy to Fly
+      run: flyctl deploy --remote-only -c ./.github/fly.toml
 
   publish_container_image:
     name: Publish container image

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -9,7 +9,7 @@ repos:
       name: Blacken
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
     - id: check-symlinks
     - id: check-yaml
@@ -24,7 +24,7 @@ repos:
       args: [--markdown-linebreak-ext=md]
 
   - repo: https://github.com/pycqa/flake8
-    rev: '3.9.2'
+    rev: '6.0.0'
     hooks:
     - id: flake8
 
@@ -43,3 +43,16 @@ repos:
       pass_filenames: false
       files: ^optimade/.*\.py$
       description: Update the API Reference documentation whenever a Python file is touched in the code base.
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991
+    hooks:
+      - id: mypy
+        name: "MyPy"
+        additional_dependencies: ["types-all", "pydantic~=1.10"]
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: "isort"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,20 +2,60 @@
 
 ## [Unreleased](https://github.com/Materials-Consortia/optimade-python-tools/tree/HEAD)
 
-[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.4...HEAD)
+[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.20.0...HEAD)
 
-This is a hotfix release for #1335, a bug regarding chunked responses triggered when using the latest FastAPI version.
+This release continues the modularisation of the package by moving the server exceptions and warnings out into top-level modules, and removing the core dependency on FastAPI (now a server dependency only). This should allow for easier use of the `optimade.models` and `optimade.client` modules within other packages.
+
+Aside from that, the package now supports Python 3.11, and our example server is now deployed at [Fly.io](https://optimade.fly.dev) rather than Heroku.
+
+## [v0.20.0](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.20.0) (2022-11-29)
+
+[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.4...v0.20.0)
+
+This release continues the modularisation of the package by moving the server exceptions and warnings out into top-level modules, and removing the core dependency on FastAPI (now a server dependency only). This should allow for easier use of the `optimade.models` and `optimade.client` modules within other packages.
+
+Aside from that, the package now supports Python 3.11, and our example server is now deployed at [Fly.io](https://optimade.fly.dev) rather than Heroku.
+
+**Implemented enhancements:**
+
+- Add support for Python 3.11 [\#1361](https://github.com/Materials-Consortia/optimade-python-tools/issues/1361)
+- Improve test diagnostics and fix deprecated Elasticsearch calls [\#1373](https://github.com/Materials-Consortia/optimade-python-tools/pull/1373) ([ml-evs](https://github.com/ml-evs))
+- Support Python 3.11 [\#1362](https://github.com/Materials-Consortia/optimade-python-tools/pull/1362) ([ml-evs](https://github.com/ml-evs))
 
 **Fixed bugs:**
 
-- UnboundLocalError - `chunk_size` is not always set in middleware method [\#1335](https://github.com/Materials-Consortia/optimade-python-tools/issues/1335)
+- Elasticsearch pytest oneliner in the docs is no longer working [\#1377](https://github.com/Materials-Consortia/optimade-python-tools/issues/1377)
+- Remote swagger validator has changed output format [\#1370](https://github.com/Materials-Consortia/optimade-python-tools/issues/1370)
+
+**Closed issues:**
+
+- Fully isolate server code from other submodules [\#1403](https://github.com/Materials-Consortia/optimade-python-tools/issues/1403)
+- Replace https://gitlab.com/pycqa/flake8 with https://github.com/pycqa/flake8 [\#1388](https://github.com/Materials-Consortia/optimade-python-tools/issues/1388)
+- OpenAPI schema should not enforce recommended constraint on `page_number` [\#1372](https://github.com/Materials-Consortia/optimade-python-tools/issues/1372)
+- Pydantic models docs are broken on the mkdocs site with new renderer [\#1353](https://github.com/Materials-Consortia/optimade-python-tools/issues/1353)
+- Migrate away from Heroku for demo server [\#1307](https://github.com/Materials-Consortia/optimade-python-tools/issues/1307)
+- FastAPI should not be a core dependency [\#1198](https://github.com/Materials-Consortia/optimade-python-tools/issues/1198)
+
+**Merged pull requests:**
+
+- Move exceptions and warnings out of server code and separate deps [\#1405](https://github.com/Materials-Consortia/optimade-python-tools/pull/1405) ([ml-evs](https://github.com/ml-evs))
+- Complete migration from Heroku to Fly [\#1400](https://github.com/Materials-Consortia/optimade-python-tools/pull/1400) ([ml-evs](https://github.com/ml-evs))
+- Add GH actions for deploying example server to Fly [\#1396](https://github.com/Materials-Consortia/optimade-python-tools/pull/1396) ([ml-evs](https://github.com/ml-evs))
+- Support new remote swagger.io validator format [\#1371](https://github.com/Materials-Consortia/optimade-python-tools/pull/1371) ([ml-evs](https://github.com/ml-evs))
+- Do not enforce minimum value of `page_number` at model level [\#1369](https://github.com/Materials-Consortia/optimade-python-tools/pull/1369) ([ml-evs](https://github.com/ml-evs))
+- Enable `mypy` and `isort` in pre-commit & CI [\#1346](https://github.com/Materials-Consortia/optimade-python-tools/pull/1346) ([ml-evs](https://github.com/ml-evs))
+- Remove randomness from structure utils tests [\#1338](https://github.com/Materials-Consortia/optimade-python-tools/pull/1338) ([ml-evs](https://github.com/ml-evs))
+- Demote FastAPI to a server dep only [\#1199](https://github.com/Materials-Consortia/optimade-python-tools/pull/1199) ([ml-evs](https://github.com/ml-evs))
 
 ## [v0.19.4](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.4) (2022-09-19)
 
 [Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.3...v0.19.4)
 
+This is a hotfix release for #1335, a bug regarding chunked responses triggered when using the latest FastAPI version.
+
 **Fixed bugs:**
 
+- UnboundLocalError - `chunk_size` is not always set in middleware method [\#1335](https://github.com/Materials-Consortia/optimade-python-tools/issues/1335)
 - Ensure `chunk_size` is properly set when chunking responses [\#1336](https://github.com/Materials-Consortia/optimade-python-tools/pull/1336) ([ml-evs](https://github.com/ml-evs))
 
 ## [v0.19.3](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.3) (2022-09-06)

diff --git a/INSTALL.md b/INSTALL.md
@@ -101,7 +101,7 @@ These commands should be run from a local optimade-python-tools directory.
 The following command starts a local Elasticsearch v6 instance, runs the test suite, then stops and deletes the containers (required as the tests insert some data):
 
 ```shell
-docker run -d --name elasticsearch_test -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:7.17.1 \
+docker run -d --name elasticsearch_test -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:7.17.7 \
 && sleep 10 \
 && OPTIMADE_DATABASE_BACKEND="elastic" py.test; \
 docker container stop elasticsearch_test; docker container rm elasticsearch_test

diff --git a/README.md b/README.md
@@ -31,7 +31,6 @@ OPTIMADE Python tools
   <td align="center">
     <a href="https://github.com/Materials-Consortia/optimade-python-tools/actions?query=branch%3Amaster+"><img alt="Build Status" src="https://img.shields.io/github/workflow/status/Materials-Consortia/optimade-python-tools/CI%20tests?logo=github"></a><br>
     <a href="https://codecov.io/gh/Materials-Consortia/optimade-python-tools"><img alt="Codecov" src="https://img.shields.io/codecov/c/github/Materials-Consortia/optimade-python-tools?logo=codecov&logoColor=white&token=UJAtmqkZZO"></a><br>
-    <a href="https://optimade.herokuapp.com"><img alt="Heroku App Status"  src="https://heroku-shields.herokuapp.com/optimade??logo=heroku"></a>
   </td>
   <td align="center">
     <a href="https://github.com/Materials-Consortia/optimade-python-tools/pulse"><img alt="Commit Activity" src="https://img.shields.io/github/commit-activity/m/Materials-Consortia/optimade-python-tools?logo=github"></a><br>
@@ -50,7 +49,7 @@ This is to enable interoperability among databases that serve crystal structures
 This repository contains a library of tools for implementing and consuming [OPTIMADE APIs](https://www.optimade.org) using Python:
 
 1. [pydantic](https://github.com/pydantic/pydantic) data models for all [OPTIMADE entry types](https://www.optimade.org/optimade-python-tools/latest/all_models/) and endpoint responses, and a [Lark](https://github.com/lark-parser/lark) [EBNF grammar](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) implementation for the OPTIMADE filter language.
-1. A configurable reference server implementation that can make use of either MongoDB or Elasticsearch database backends out-of-the-box, and is readily extensible to other backends. Try it out on the [demo site](https://optimade.herokuapp.com)! The OpenAPI schemas of the server are used to construct the [OPTIMADE schemas](https://schemas.optimade.org/) site.
+1. A configurable reference server implementation that can make use of either MongoDB or Elasticsearch database backends out-of-the-box, and is readily extensible to other backends. Try it out on the [demo site](https://optimade.fly.dev)! The OpenAPI schemas of the server are used to construct the [OPTIMADE schemas](https://schemas.optimade.org/) site.
 1. An [OPTIMADE client](https://www.optimade.org/optimade-python-tools/latest/getting_started/client/) (`optimade-get`) that can query multiple [OPTIMADE providers](https://optimade.org/providers-dashboard) concurrently with a given filter, at the command-line or from Python code.
 1. A fuzzy API validator tool, which may be called from the shell (`optimade-validator`) or used as a GitHub Action from [optimade-validator-action](https://github.com/Materials-Consortia/optimade-validator-action); this validator is used to construct the [providers dashboard](https://optimade.org/providers-dashboard).
 

diff --git a/docs/api_reference/exceptions.md b/docs/api_reference/exceptions.md
@@ -0,0 +1,3 @@
+# exceptions
+
+::: optimade.exceptions
diff --git a/docs/api_reference/utils.md b/docs/api_reference/utils.md
@@ -0,0 +1,3 @@
+# utils
+
+::: optimade.utils
diff --git a/docs/api_reference/warnings.md b/docs/api_reference/warnings.md
@@ -0,0 +1,3 @@
+# warnings
+
+::: optimade.warnings
diff --git a/docs/concepts/validation.md b/docs/concepts/validation.md
@@ -12,8 +12,8 @@ To run the script, simply provide an OPTIMADE URL to the script at the command-l
 You can use the following to validate the Heroku deployment of our reference server:
 
 ```shell
-$ optimade-validator https://optimade.herokuapp.com/
-Testing entire implementation at https://optimade.herokuapp.com
+$ optimade-validator https://optimade.fly.dev/
+Testing entire implementation at https://optimade.fly.dev
 ...
 ```
 

diff --git a/docs/getting_started/client.md b/docs/getting_started/client.md
@@ -60,14 +60,14 @@ We can refine the search by manually specifying some URLs:
 
 === "Command line"
     ```shell
-    optimade-get --output-file results.json https://optimade.herokuapp.com https://optimade.odbx.science
+    optimade-get --output-file results.json https://optimade.fly.dev https://optimade.odbx.science
     ```
 
 === "Python"
     ```python
     from optimade.client import OptimadeClient
     client = OptimadeClient(
-        base_urls=["https://optimade.herokuapp.com", "https://optimade.odbx.science"]
+        base_urls=["https://optimade.fly.dev", "https://optimade.odbx.science"]
     )
     client.get()
     ```
@@ -97,22 +97,22 @@ At the command-line, the results of the query will be printed to `stdout`, ready
 For example:
 
 ```shell
-optimade-get --filter 'nsites = 1' --output-file results.json https://optimade.herokuapp.com
+optimade-get --filter 'nsites = 1' --output-file results.json https://optimade.fly.dev
 cat results.json
 ```
 
 has the following (truncated) output:
 
 ```json
-{  
+{
   // The endpoint that was queried
-  "structures": {  
+  "structures": {
     // The filter applied to that endpointk
-    "nsites = 1": {  
+    "nsites = 1": {
       // The base URL of the OPTIMADE API
-      "https://optimade.herokuapp.com": {  
+      "https://optimade.fly.dev": {
         // The OPTIMADE API response as if called with an infinite `page_limit`
-        "data": [  
+        "data": [
           {
             "id": "mpf_1",
             "type": "structures",
@@ -147,7 +147,7 @@ This is the same format as the cached results of the Python client:
 
 ```python
 from optimade.client import OptimadeClient
-client = OptimadeClient(base_urls="https://optimade.herokuapp.com")
+client = OptimadeClient(base_urls="https://optimade.fly.dev")
 client.get('nsites = 1')
 client.get('nsites = 2')
 print(client.all_results)
@@ -158,10 +158,10 @@ will return a dictionary with top-level keys:
 {
     "structures": {
         "nsites = 1": {
-            "https://optimade.herokuapp.com": {...}
+            "https://optimade.fly.dev": {...}
         },
         "nsites = 2": {
-            "https://optimade.herokuapp.com": {...}
+            "https://optimade.fly.dev": {...}
         }
     }
 }