Merge branch 'main' into edgarrmondragon/feat/nested-nullable-props

meltano · Oct 3, 2024 · 8222ff4 · 8222ff4
2 parents a3f5b46 + 9bf104e
commit 8222ff4
Show file tree

Hide file tree

Showing 95 changed files with 3,835 additions and 756 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml
@@ -13,7 +13,7 @@ body:
     attributes:
       label: Singer SDK Version
       description: Version of the library you are using
-      placeholder: "0.40.0"
+      placeholder: "0.41.0"
     validations:
       required: true
   - type: checkboxes

diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -1,8 +1,8 @@
-griffe==1.2.0
+griffe==1.3.1
 pip==24.2
 poetry==1.8.3
 poetry-plugin-export==1.8.0
-poetry-dynamic-versioning==1.4.0
+poetry-dynamic-versioning==1.4.1
 pre-commit==3.8.0
 nox==2024.4.15
 nox-poetry==1.0.3
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -65,7 +65,7 @@ jobs:
         name: Packages
         path: dist
     - name: Publish
-      uses: pypa/[email protected].0
+      uses: pypa/[email protected].2
 
   upload-to-release:
     name: Upload files to release

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -90,6 +90,12 @@ jobs:
         pipx inject nox nox-poetry
         nox --version
 
+    - uses: actions/cache@v4
+      if: matrix.session == 'tests'
+      with:
+        path: http_cache.sqlite
+        key: http_cache-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.sqlalchemy }}
+
     - name: Run Nox
       env:
         SQLALCHEMY_VERSION: ${{ matrix.sqlalchemy }}
@@ -100,7 +106,7 @@ jobs:
       if: always() && (matrix.session == 'tests')
       with:
         include-hidden-files: true
-        name: coverage-data-nox_${{ matrix.session }}-${{ matrix.os }}-py${{ matrix.python-version }}_sqlalchemy_${{ matrix.sqlalchemy }}
+        name: coverage-data-nox_-${{ matrix.os }}-py${{ matrix.python-version }}_sqlalchemy_${{ matrix.sqlalchemy }}
         path: ".coverage.*"
 
   tests-external:

diff --git a/.github/workflows/version_bump.yml b/.github/workflows/version_bump.yml
@@ -6,6 +6,7 @@ on:
       dry_run:
         description: "Run the action without creating a PR or release draft"
         required: true
+        default: false
         type: boolean
       bump:
         description: "Version bump type"
@@ -84,7 +85,7 @@ jobs:
 
     - name: Create Pull Request
       if: ${{ github.event.inputs.dry_run == 'false' }}
-      uses: peter-evans/create-pull-request@v6
+      uses: peter-evans/create-pull-request@v7
       id: create-pull-request
       with:
         token: ${{ secrets.MELTYBOT_GITHUB_AUTH_TOKEN }}

diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,6 @@
+# HTTP cache
+http_cache.sqlite
+
 # Local Poetry configuration file
 
 poetry.toml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -43,14 +43,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.2
+  rev: 0.29.3
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.3
+  rev: v0.6.8
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,41 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v0.41.0 (2024-10-02)
+
+### ✨ New
+
+- [#2667](https://github.com/meltano/sdk/issues/2667) Support stream aliasing of `BATCH` messages via stream maps -- _**Thanks @ReubenFrankel!**_
+- [#2651](https://github.com/meltano/sdk/issues/2651) SQL taps now emit schemas with `maxLength` when applicable
+- [#2618](https://github.com/meltano/sdk/issues/2618) Developers can now more easily override the mapping from SQL column type to JSON schema
+
+### 🐛 Fixes
+
+- [#2697](https://github.com/meltano/sdk/issues/2697) All HTTP timeout exceptions are now retried in REST and GraphQL streams
+- [#2683](https://github.com/meltano/sdk/issues/2683) A clear error message is now emitted when flattening is enabled but `flattening_max_depth` is not set
+- [#2665](https://github.com/meltano/sdk/issues/2665) Mapped datetime values are now typed as `date-time` strings in the schema message -- _**Thanks @gregkoutsimp!**_
+- [#2663](https://github.com/meltano/sdk/issues/2663) Properties dropped using `None` or `__NULL__` in stream maps are now also removed from the schema `required` array
+
+### ⚙️ Under the Hood
+
+- [#2696](https://github.com/meltano/sdk/issues/2696) Use tox without installing Poetry explicitly in workflows
+- [#2654](https://github.com/meltano/sdk/issues/2654) Added a generic `FileStream` (still in active development!)
+- [#2695](https://github.com/meltano/sdk/issues/2695) Update dependencies in templates
+- [#2661](https://github.com/meltano/sdk/issues/2661) Drop support for Python 3.8 in templates
+- [#2670](https://github.com/meltano/sdk/issues/2670) Deprecated `Faker` class in stream maps
+- [#2666](https://github.com/meltano/sdk/issues/2666) Remove non-functional `record-flattening` capability -- _**Thanks @ReubenFrankel!**_
+- [#2652](https://github.com/meltano/sdk/issues/2652) Renamed `SQLConnector.type_mapping` to `SQLConnector.sql_to_jsonschema`
+- [#2647](https://github.com/meltano/sdk/issues/2647) Use future `warnings.deprecated` decorator
+
+### 📚 Documentation Improvements
+
+- [#2658](https://github.com/meltano/sdk/issues/2658) Added more versions when stuff changed or was added
+
+### 📦 Packaging changes
+
+- [#2694](https://github.com/meltano/sdk/issues/2694) Removed unused backport `importlib_resources` dependency in tap template
+- [#2664](https://github.com/meltano/sdk/issues/2664) Added a constraint on setuptools <= 70.3.0 to fix incompatibility with some dependencies
+
 ## v0.40.0 (2024-09-02)
 
 ### ✨ New

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/build.yml
@@ -42,4 +42,4 @@ jobs:
     - name: Publish
       ## TODO: create a trusted publisher on PyPI
       ## https://docs.pypi.org/trusted-publishers/
-      uses: pypa/gh-action-pypi-publish@v1.9.0
+      uses: pypa/gh-action-pypi-publish@v1.10.2
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.github/workflows/test.yml
@@ -24,16 +24,16 @@ on:
     - tox.ini
   workflow_dispatch:
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
   pytest:
     runs-on: ubuntu-latest
-    env:
-      GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }}
     strategy:
       fail-fast: false
       matrix:
         python-version:
-        - "3.8"
         - "3.9"
         - "3.10"
         - "3.11"
@@ -44,13 +44,7 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: {{ '${{ matrix.python-version }}' }}
-    - name: Install Poetry
-      run: |
-        pip install poetry
-    - name: Install dependencies
-      run: |
-        poetry env use {{ '${{ matrix.python-version }}' }}
-        poetry install
-    - name: Test with pytest
+    - run: pipx install tox
+    - name: Run Tox
       run: |
-        poetry run pytest
+        tox -e $(echo py{{ '${{ matrix.python-version }}' }} | tr -d .)
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/.pre-commit-config.yaml
@@ -18,19 +18,19 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.3
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.6
+  rev: v0.6.8
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.1
+  rev: v1.11.2
   hooks:
   - id: mypy
diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/pyproject.toml
@@ -16,7 +16,6 @@ keywords = [
 classifiers = [
     "Intended Audience :: Developers",
     "Operating System :: OS Independent",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -30,13 +29,13 @@ packages = [
 {%- endif %}
 
 [tool.poetry.dependencies]
-python = ">=3.8"
-singer-sdk = { version="~=0.40.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
+python = ">=3.9"
+singer-sdk = { version="~=0.41.0"{{ ', extras = ["faker"]' if cookiecutter.faker_extra }} }
 fs-s3fs = { version = "~=1.1.1", optional = true }
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=8"
-singer-sdk = { version="~=0.40.0", extras = ["testing"] }
+singer-sdk = { version="~=0.41.0", extras = ["testing"] }
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]
@@ -49,8 +48,7 @@ python_version = "3.12"
 warn_unused_configs = true
 
 [tool.ruff]
-src = ["{{cookiecutter.library_name}}"]
-target-version = "py38"
+target-version = "py39"
 
 [tool.ruff.lint]
 ignore = [
@@ -64,14 +62,11 @@ select = ["ALL"]
 [tool.ruff.lint.flake8-annotations]
 allow-star-arg-any = true
 
-[tool.ruff.lint.isort]
-known-first-party = ["{{cookiecutter.library_name}}"]
-
 [tool.ruff.lint.pydocstyle]
 convention = "google"
 
 [build-system]
-requires = ["poetry-core==1.9.0"]
+requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]

diff --git a/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini b/cookiecutter/mapper-template/{{cookiecutter.mapper_id}}/tox.ini
@@ -1,19 +1,12 @@
 # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
 
 [tox]
-envlist = py{38,39,310,311,312}
-isolated_build = true
+envlist = py3{9,10,11,12}
+requires =
+    tox>=4.19
 
 [testenv]
-allowlist_externals = poetry
+deps =
+    pytest
 commands =
-    poetry install -v
-    poetry run pytest
-
-[testenv:pytest]
-# Run the python tests.
-# To execute, run `tox -e pytest`
-envlist = py{38,39,310,311,312}
-commands =
-    poetry install -v
-    poetry run pytest
+    pytest {posargs}
diff --git a/...cutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py b/...cutter/mapper-template/{{cookiecutter.mapper_id}}/{{cookiecutter.library_name}}/mapper.py
@@ -3,14 +3,13 @@
 from __future__ import annotations
 
 import typing as t
-from typing import TYPE_CHECKING
 
 import singer_sdk.typing as th
 from singer_sdk import _singerlib as singer
 from singer_sdk.mapper import PluginMapper
 from singer_sdk.mapper_base import InlineMapper
 
-if TYPE_CHECKING:
+if t.TYPE_CHECKING:
     from pathlib import PurePath
 
 

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/build.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/build.yml
@@ -42,4 +42,4 @@ jobs:
     - name: Publish
       ## TODO: create a trusted publisher on PyPI
       ## https://docs.pypi.org/trusted-publishers/
-      uses: pypa/gh-action-pypi-publish@v1.9.0
+      uses: pypa/gh-action-pypi-publish@v1.10.2
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.github/workflows/test.yml
@@ -24,16 +24,16 @@ on:
     - tox.ini
   workflow_dispatch:
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
   pytest:
     runs-on: ubuntu-latest
-    env:
-      GITHUB_TOKEN: {{ '${{secrets.GITHUB_TOKEN}}' }}
     strategy:
       fail-fast: false
       matrix:
         python-version:
-        - "3.8"
         - "3.9"
         - "3.10"
         - "3.11"
@@ -44,13 +44,7 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: {{ '${{ matrix.python-version }}' }}
-    - name: Install Poetry
-      run: |
-        pip install poetry
-    - name: Install dependencies
-      run: |
-        poetry env use {{ '${{ matrix.python-version }}' }}
-        poetry install
-    - name: Test with pytest
+    - run: pipx install tox
+    - name: Run Tox
       run: |
-        poetry run pytest
+        tox -e $(echo py{{ '${{ matrix.python-version }}' }} | tr -d .)
diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/.pre-commit-config.yaml
@@ -18,20 +18,20 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.29.1
+  rev: 0.29.3
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.5.6
+  rev: v0.6.8
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]
   - id: ruff-format
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.11.1
+  rev: v1.11.2
   hooks:
   - id: mypy
     additional_dependencies:

diff --git a/cookiecutter/tap-template/{{cookiecutter.tap_id}}/README.md b/cookiecutter/tap-template/{{cookiecutter.tap_id}}/README.md
@@ -122,7 +122,7 @@ Now you can test and orchestrate using Meltano:
 # Test invocation:
 meltano invoke {{ cookiecutter.tap_id }} --version
 # OR run a test `elt` pipeline:
-meltano elt {{ cookiecutter.tap_id }} target-jsonl
+meltano run {{ cookiecutter.tap_id }} target-jsonl
 ```
 
 ### SDK Dev Guide