Transition the project to a Poetry-based setup and revitalize basic t…

…est automation (#118) --------- Co-authored-by: Christoph Pirkl <[email protected]>
exasol · Apr 26, 2024 · 8b5b0fc · 8b5b0fc
1 parent 542fa7b
commit 8b5b0fc
Show file tree

Hide file tree

Showing 21 changed files with 2,281 additions and 189 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+poetry.lock     linguist-generated=true
diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md
@@ -8,9 +8,9 @@ assignees: ''
 ---
 
 ### Checklist
-In order to improve the time, how long a bugfix does take, please make sure you address as much as possible of the following checklist.
+In order to speedup fixing the bug, please make sure you address as many items from the following checklist as possible.
 
-- [ ] I have reproduced the issue with at least the latest released version of \<PROJECT-NAME\>.
+- [ ] I have reproduced the issue with the latest released version of pyexasol.
 - [ ] I have added references to issues that sound similar in section _Related Issues_.
 
 **Tip: Adding screenshots, outputs and logs might help to improve clarity.**
@@ -62,7 +62,7 @@ Screenshot: If applicable, add screenshots to help explaining your problem.
 Describe the system and the used configuration.
 -->
 **Desktop:**
- - OS: [e.g. Linux (Fedora 35), Linux (Ubuntu 20.01), MacOs (...), Windows (...)]
+ - OS: [e.g. Linux (Fedora 35), Linux (Ubuntu 20.01), macOS (...), Windows (...)]
  - Python Version [e.g. Python 3.8.0]
  - Package Version [e.g. 22]
 

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -0,0 +1,49 @@
+name: CI Checks
+
+on: 
+  workflow_call:
+
+
+jobs:
+
+  unit-tests:
+    name: Unit Tests
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+
+    steps:
+    - name: SCM Checkout
+      uses: actions/checkout@v4
+
+    - name: Setup Python & Poetry Environment
+      uses: exasol/python-toolbox/.github/actions/[email protected]
+      with:
+        python-version: ${{ matrix.python-version }} 
+
+    - name: Run Tests
+      run: |
+        poetry run nox -s unit-tests
+
+  integration-tests:
+    name: Integration Tests
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+
+    steps:
+    - name: SCM Checkout
+      uses: actions/checkout@v4
+
+    - name: Setup Python & Poetry Environment
+      uses: exasol/python-toolbox/.github/actions/[email protected]
+      with:
+        python-version: ${{ matrix.python-version }} 
+
+    - name: Run Tests
+      run: |
+        poetry run nox -s integration-tests
diff --git a/.github/workflows/ci-master.yml b/.github/workflows/ci-master.yml
@@ -0,0 +1,20 @@
+name: Continuous Integration (Master)
+
+on:
+  push:
+    branches:
+      - "master"
+      - "main"
+  schedule:
+    # “At 00:00 on every 7th day-of-month from 1 through 31.” (https://crontab.guru)
+    - cron: "0 0 1/7 * *"
+
+jobs:
+  verify:
+    uses: ./.github/workflows/checks.yml
+
+  examples:
+    uses: ./.github/workflows/examples.yml
+
+  ssl_cert:
+    uses: ./.github/workflows/ssl_cert.yml
diff --git a/.github/workflows/ci-pr.yml b/.github/workflows/ci-pr.yml
@@ -0,0 +1,14 @@
+name: Continuous Integration (PR)
+
+on:
+  pull_request:
+
+jobs:
+  verify:
+    uses: ./.github/workflows/checks.yml
+
+  examples:
+    uses: ./.github/workflows/examples.yml
+
+  ssl_cert:
+    uses: ./.github/workflows/ssl_cert.yml
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -1,95 +1,81 @@
 name: Examples
 
 on:
-  push:
-    paths:
-      - "**.py"
-      - "**.yml"
-  pull_request:
-
-  workflow_dispatch:
+  workflow_call:
 
 jobs:
   examples:
+    name: Run Examples
+    # For all containers with versions 7.1.x and older versions 8.y.z of docker-db,
+    # there will be a crash if UDFs (User Defined Functions) are utilized.
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - python-version: "3.11"
-            exasol-tag: latest-7.1
-            exasol-port: 8563
-
-          - python-version: "3.11"
-            exasol-tag: latest-7.0
-            exasol-port: 8563
-
-          - python-version: "3.7"
-            exasol-tag: latest-6.2
-            exasol-port: 8888
-    name: "Build with Python ${{ matrix.python-version }} and Exasol ${{ matrix.exasol-tag }}"
-
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
-    services:
-      exasol:
-        image: exasol/docker-db:${{ matrix.exasol-tag }}
-        ports:
-          - 8563:${{ matrix.exasol-port }}
-        options: --privileged
-
+        python-version: ["3.11"]
     steps:
-      - name: "Checkout repo (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Checkout repo"
         uses: actions/checkout@v2
 
-      - name: "Setup Python (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        uses: actions/setup-python@v2
+      - name: "Setup Python & Poetry Environment"
+        uses: exasol/python-toolbox/.github/actions/python[email protected]
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: "Install dependencies (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: pip install -e .[pandas,ujson,rapidjson,orjson,examples]
+      - name: "Install all extras"
+        run: poetry install --all-extras
+
+      - name: "Start test db"
+        run: >
+          poetry run itde spawn-test-environment --environment-name test 
+          --database-port-forward 8563 --bucketfs-port-forward 2580 
+          --docker-db-image-version 7.1.17 --db-mem-size 4GB
 
-      - name: "Wait for Exasol container (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: python examples/_wait_container.py
+      - name: "Wait for Exasol container (UDF extraction)"
+        run: poetry run python examples/_wait_container.py
+
+      - name: "Run examples"
+        run: |
+          poetry run python examples/a00_prepare.py
+          poetry run python examples/a01_basic.py
+          poetry run python examples/a02_fetch_tuple.py
+          poetry run python examples/a03_fetch_dict.py
+          poetry run python examples/a04_fetch_mapper.py
+          poetry run python examples/a05_formatting.py
+          poetry run python examples/a06_transaction.py
+          poetry run python examples/a07_exceptions.py
+          poetry run python examples/a08_ext.py
+          poetry run python examples/a09_abort_query.py
+          poetry run python examples/a10_context_manager.py
+          poetry run python examples/a11_insert_multi.py
+          poetry run python examples/a12_meta.py
+          poetry run python examples/a13_meta_nosql.py
+          poetry run python examples/b01_pandas.py
+          poetry run python examples/b02_import_export.py
+          poetry run python examples/b03_parallel_export.py
+          poetry run python examples/b04_parallel_import.py
+          poetry run python examples/b05_parallel_export_import.py
+          poetry run python examples/b06_http_transport_errors.py
+          poetry run python examples/c01_redundancy.py
+          poetry run python examples/c02_edge_case.py
+          poetry run python examples/c03_db2_compat.py
+          poetry run python examples/c04_encryption.py
+          poetry run python examples/c05_session_params.py
+          poetry run python examples/c06_local_config.py
+          poetry run python examples/c07_profiling.py
+          poetry run python examples/c08_snapshot_transactions.py
+          poetry run python examples/c09_script_output.py
+          poetry run python examples/c10_overload.py
+          poetry run python examples/c11_quote_ident.py
+          poetry run python examples/c12_thread_safety.py
+          poetry run python examples/c13_dsn_parsing.py
+          poetry run python examples/c14_http_proxy.py
+          poetry run python examples/c15_garbage_collection.py
+          poetry run python examples/j01_rapidjson.py
+          poetry run python examples/j02_ujson.py
+          poetry run python examples/j03_orjson.py
 
-      - name: "Run examples (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Stop test db"
         run: |
-          python examples/a00_prepare.py
-          python examples/a01_basic.py
-          python examples/a02_fetch_tuple.py
-          python examples/a03_fetch_dict.py
-          python examples/a04_fetch_mapper.py
-          python examples/a05_formatting.py
-          python examples/a06_transaction.py
-          python examples/a07_exceptions.py
-          python examples/a08_ext.py
-          python examples/a09_abort_query.py
-          python examples/a10_context_manager.py
-          python examples/a11_insert_multi.py
-          python examples/a12_meta.py
-          python examples/a13_meta_nosql.py
-          python examples/b01_pandas.py
-          python examples/b02_import_export.py
-          python examples/b03_parallel_export.py
-          python examples/b04_parallel_import.py
-          python examples/b05_parallel_export_import.py
-          python examples/b06_http_transport_errors.py
-          python examples/c01_redundancy.py
-          python examples/c02_edge_case.py
-          python examples/c03_db2_compat.py
-          python examples/c04_encryption.py
-          python examples/c05_session_params.py
-          python examples/c06_local_config.py
-          python examples/c07_profiling.py
-          python examples/c08_snapshot_transactions.py
-          python examples/c09_script_output.py
-          python examples/c10_overload.py
-          python examples/c11_quote_ident.py
-          python examples/c12_thread_safety.py
-          python examples/c13_dsn_parsing.py
-          python examples/c14_http_proxy.py
-          python examples/c15_garbage_collection.py
-          python examples/j01_rapidjson.py
-          python examples/j02_ujson.py
-          python examples/j03_orjson.py
+          docker kill db_container_test
diff --git a/.github/workflows/ssl_cert.yml b/.github/workflows/ssl_cert.yml
@@ -1,58 +1,43 @@
 name: SSL certificate test
 
 on:
-  # Workflow is disabled due to incompatible changes in "test_container_ssl"
-  #
-  # push:
-  #  paths:
-  #    - "**.py"
-  #    - "**.yml"
-  #  branches:
-  #    - master
-  pull_request:
-
-  workflow_dispatch:
+  workflow_call:
 
 jobs:
   ssl_cert:
+    name: Build SSL-Cert
+    # For all containers with versions 7.1.x and older versions 8.y.z of docker-db,
+    # there will be a crash if UDFs (User Defined Functions) are utilized.
+    runs-on: ubuntu-20.04
+    timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - python-version: "3.11"
-            exasol-tag: 7.1.6
-    name: "Build SSL-Cert with Python ${{ matrix.python-version }} and Exasol ${{ matrix.exasol-tag }}"
-
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-
+        python-version: ["3.11"]
     steps:
-      - name: "Checkout repo (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Checkout repo"
         uses: actions/checkout@v2
 
-      - name: "Setup Python (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        uses: actions/setup-python@v2
+      - name: "Setup Python & Poetry Environment"
+        uses: exasol/python-toolbox/.github/actions/python[email protected]
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: "Install dependencies (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: pip install -e .[pandas]
-
-      - name: "Checkout test environment (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        uses: actions/checkout@v2
-        with:
-          repository: exasol/integration-test-docker-environment
-          path: './test_env'
+      - name: "Install all extras"
+        run: poetry install --all-extras
 
-      - name: "Spawn EXASOL environemnt (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: ./start-test-env spawn-test-environment --environment-name ssl --docker-db-image-version=${{ matrix.exasol-tag }} --database-port-forward 8888 --create-certificates
-        working-directory: ./test_env
+      - name: "Start test db"
+        run: >
+          poetry run itde spawn-test-environment --environment-name ssl
+          --database-port-forward 8888 --bucketfs-port-forward 2580 
+          --docker-db-image-version 7.1.17 --db-mem-size 4GB
+          --create-certificates
 
-      - name: "Pull certificate from container (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: docker cp test_container_ssl:/certificates/rootCA.crt .
+      - name: "Pull certificate from container"
+        run: docker cp db_container_ssl:/certificates/rootCA.crt .
 
-      - name: "Setup custom host (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
+      - name: "Setup custom host"
         run: echo "127.0.0.1 exasol-test-database" | sudo tee -a /etc/hosts
 
-      - name: "Run test (Python ${{ matrix.python-version }} Exasol ${{ matrix.exasol-tag }})"
-        run: python examples/_ssl_test.py
+      - name: "Run test"
+        run: poetry run python examples/_ssl_test.py
diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,5 @@ build/
 .vscode
 
 .DS_Store
+
+.build_output/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,12 @@
 # Changelog
 
+## [Unreleased]
+
+- Switch packaging and project workflows to poetry
+- Drop support for python 3.7
+- Drop support for Exasol 6.x
+- Drop support for Exasol 7.0.x
+
 ## [0.25.2] - 2023-03-14
 
 - Fix deprecation warning by setting SSLContext protocol.

diff --git a/MANIFEST.in b/MANIFEST.in
diff --git a/examples/_config.py b/examples/_config.py
@@ -3,4 +3,4 @@
 dsn = os.environ.get('EXAHOST', 'localhost:8563')
 user = os.environ.get('EXAUID', 'SYS')
 password = os.environ.get('EXAPWD', 'exasol')
-schema = os.environ.get('EXASCHEMA', 'PYEXASOL_TEST')
+schema = os.environ.get('EXASCHEMA', 'TEST')
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,3 +16,5 @@ build/
		.vscode

		.DS_Store

		.build_output/