Merge branch 'develop'

.env.dependencies

@@ -28,7 +28,9 @@ MONGO_INITDB_ROOT_USERNAME=onetl
 MONGO_INITDB_ROOT_PASSWORD=E4j7h!9A
 
 # S3
+MINIO_ACCESS_KEY=onetl
 MINIO_ROOT_USER=onetl
+MINIO_SECRET_KEY=woh3fogh3Biereu3quee1aidu9theiro
 MINIO_ROOT_PASSWORD=woh3fogh3Biereu3quee1aidu9theiro
 
 # SFTP

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,24 @@
+<!-- Thank you for your contribution! -->
+<!-- Unless your change is trivial, please create an issue to discuss the change before creating a PR -->
+<!-- See https://github.com/MobileTeleSystems/onetl/blob/develop/CONTRIBUTING.rst for help on Contributing -->
+<!-- PLEASE DO **NOT** put issue ids in the PR title! Instead, add a descriptive title and put ids in the body -->
+
+## Change Summary
+
+<!-- Please give a short summary of the changes. -->
+
+## Related issue number
+
+<!-- Are there any issues opened that will be resolved by merging this change? -->
+<!-- WARNING: please use "fix #123" style references so the issue is closed when this PR is merged. -->
+
+## Checklist
+
+* [ ] Commit message and PR title is comprehensive
+* [ ] Keep the change as small as possible
+* [ ] Unit and integration tests for the changes exist
+* [ ] Tests pass on CI and coverage does not decrease
+* [ ] Documentation reflects the changes where applicable
+* [ ] `docs/changelog/next_release/<pull request or issue id>.<change type>.rst` file added describing change
+  (see [docs/changelog/README.rst](https://github.com/MobileTeleSystems/onetl/blob/develop/docs/changelog/README.rst) for details.)
+* [ ] My PR is ready to review.

.github/workflows/changelog.yml

@@ -0,0 +1,63 @@
+name: Changelog
+
+on:
+  pull_request:
+    types: [opened, synchronize, labeled, unlabeled, reopened]
+    branches-ignore:
+    - dependabot/**
+    - pre-commit-ci-update-config
+    - master
+
+env:
+  DEFAULT_PYTHON: '3.11'
+
+permissions:
+  contents: read
+
+jobs:
+  check-changelog:
+    name: Changelog Entry Check
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+      id: python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.DEFAULT_PYTHON }}
+
+    - name: Cache pip
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-${{ hashFiles('requirements/core.txt', 'requirements/docs.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-${{ hashFiles('requirements/core.txt', 'requirements/docs.txt') }}
+          ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-changelog-
+
+    - name: Upgrade pip
+      run: python -m pip install --upgrade pip setuptools wheel
+
+    - name: Install dependencies
+      run: |
+        pip install -I  -r requirements/core.txt -r requirements/docs.txt
+
+    - name: Check changelog entry exists
+      run: |
+        if [ ! -s docs/changelog/next_release/${{ github.event.pull_request.number }}.*.rst ]; then
+            echo "Please add corresponding file 'docs/changelog/next_release/<issue number>.<change type>.rst' with changes description"
+            exit 1
+        fi
+
+    - name: Validate changelog
+      run: |
+        # Fetch the pull request' base branch so towncrier will be able to
+        # compare the current branch with the base branch.
+        git fetch --no-tags origin +refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}
+        towncrier check --compare-with origin/${{ github.base_ref }}
+        towncrier --draft

.github/workflows/codeql-analysis.yml

@@ -5,8 +5,8 @@ on:
     branches:
     - develop
   pull_request:
-    branches:
-    - develop
+    branches-ignore:
+    - master
   workflow_dispatch:
 
 concurrency:

.github/workflows/data/base/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/base/tracked.txt

@@ -0,0 +1,15 @@
+.github/workflows/tests.yml
+.github/workflows/get-matrix.yml
+.github/workflows/data/base/**
+requirements/core.txt
+requirements/tests/base.txt
+onetl/db_connection/db_connection.py
+onetl/db_connection/jdbc*.py
+onetl/db_connection/dialect_mixins/*
+onetl/file_connection/file_connection.py
+onetl/core/**
+onetl/hwm/**
+onetl/impl/**
+onetl/strategy/**
+onetl/exception.py
+.env.local

.github/workflows/data/clickhouse/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/clickhouse/matrix.yml

@@ -0,0 +1,38 @@
+min: &min
+  spark-version: 2.3.1
+  python-version: '3.7'
+  java-version: 8
+  os: ubuntu-latest
+
+max: &max
+  spark-version: 3.4.0
+  python-version: '3.11'
+  java-version: 17
+  os: ubuntu-latest
+
+latest: &latest
+  spark-version: latest
+  python-version: '3.11'
+  java-version: 17
+  os: ubuntu-latest
+
+matrix:
+  small:
+  - clickhouse-image: clickhouse/clickhouse-server
+    clickhouse-version: 23.2.6-alpine
+    <<: *max
+  full:
+  # the lowest supported Clickhouse version by JDBC driver
+  - clickhouse-image: yandex/clickhouse-server
+    clickhouse-version: '20.7'
+    <<: *min
+  - clickhouse-image: clickhouse/clickhouse-server
+    clickhouse-version: 23.2.6-alpine
+    <<: *max
+  nightly:
+  - clickhouse-image: yandex/clickhouse-server
+    clickhouse-version: '20.7'
+    <<: *min
+  - clickhouse-image: clickhouse/clickhouse-server
+    clickhouse-version: latest-alpine
+    <<: *latest

.github/workflows/data/clickhouse/tracked.txt

@@ -0,0 +1 @@
+**/clickhouse*

.github/workflows/data/core/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/core/matrix.yml

@@ -0,0 +1,22 @@
+min: &min
+  spark-version: 2.3.1
+  python-version: '3.7'
+  java-version: 8
+  os: ubuntu-latest
+
+max: &max
+  spark-version: 3.4.0
+  python-version: '3.11'
+  java-version: 17
+  os: ubuntu-latest
+
+latest: &latest
+  spark-version: latest
+  python-version: '3.11'
+  java-version: 17
+  os: ubuntu-latest
+
+matrix:
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *max, *latest]

.github/workflows/data/core/tracked.txt

@@ -0,0 +1,4 @@
+onetl/hooks/**
+onetl/plugins/**
+onetl/_internal.py
+onetl/log.py

.github/workflows/data/ftp/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/ftp/matrix.yml

@@ -0,0 +1,19 @@
+min: &min
+  python-version: '3.7'
+  os: ubuntu-latest
+
+max: &max
+  python-version: '3.11'
+  os: ubuntu-latest
+
+matrix:
+  small:
+  # chonjay21/ftps image has only latest tag
+  - ftp-version: latest
+    <<: *max
+  full: &full
+  - ftp-version: latest
+    <<: *min
+  - ftp-version: latest
+    <<: *max
+  nightly: *full

.github/workflows/data/ftp/tracked.txt

@@ -0,0 +1,3 @@
+**/ftp*
+docker-compose.yml
+.env.dependencies

.github/workflows/data/ftps/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/ftps/matrix.yml

@@ -0,0 +1,19 @@
+min: &min
+  python-version: '3.7'
+  os: ubuntu-latest
+
+max: &max
+  python-version: '3.11'
+  os: ubuntu-latest
+
+matrix:
+  small:
+  # chonjay21/ftps image has only latest tag
+  - ftps-version: latest
+    <<: *max
+  full: &full
+  - ftps-version: latest
+    <<: *min
+  - ftps-version: latest
+    <<: *max
+  nightly: *full

.github/workflows/data/ftps/tracked.txt

@@ -0,0 +1,3 @@
+**/ftp*
+docker-compose.yml
+.env.dependencies

.github/workflows/data/greenplum/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/greenplum/matrix.yml

@@ -0,0 +1,24 @@
+min: &min
+  # Spark 2.3.0 does not support passing ivysettings.xml
+  spark-version: 2.3.1
+  python-version: '3.7'
+  java-version: 8
+  os: ubuntu-latest
+
+max: &max
+  # Greenplum connector does not support Spark 3.3+
+  spark-version: 3.2.3
+  python-version: '3.10'
+  java-version: 11
+  os: ubuntu-latest
+
+matrix:
+  small:
+  - greenplum-version: 6.8
+    <<: *max
+  full: &full
+  - greenplum-version: 6.1
+    <<: *min
+  - greenplum-version: 6.8
+    <<: *max
+  nightly: *full

.github/workflows/data/greenplum/tracked.txt

@@ -0,0 +1 @@
+**/greenplum*

.github/workflows/data/hdfs/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/hdfs/matrix.yml

@@ -0,0 +1,18 @@
+min: &min
+  python-version: '3.7'
+  os: ubuntu-latest
+
+max: &max
+  python-version: '3.11'
+  os: ubuntu-latest
+
+matrix:
+  small:
+  - hadoop-version: hadoop3-hdfs
+    <<: *max
+  full: &full
+  - hadoop-version: hadoop2-hdfs
+    <<: *min
+  - hadoop-version: hadoop3-hdfs
+    <<: *max
+  nightly: *full

.github/workflows/data/hdfs/tracked.txt

@@ -0,0 +1,4 @@
+**/hdfs*
+onetl/connection/kerberos_helpers.py
+docker-compose.yml
+.env.dependencies

.github/workflows/data/hive/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/hive/matrix.yml

@@ -0,0 +1,22 @@
+min: &min
+  spark-version: 2.3.1
+  python-version: '3.7'
+  java-version: 8
+  os: ubuntu-latest
+
+max: &max
+  spark-version: 3.4.0
+  python-version: '3.11'
+  java-version: 17
+  os: ubuntu-latest
+
+latest: &latest
+  spark-version: latest
+  python-version: '3.11'
+  java-version: 17
+  os: ubuntu-latest
+
+matrix:
+  small: [*max]
+  full: [*min, *max]
+  nightly: [*min, *latest]

.github/workflows/data/hive/tracked.txt

@@ -0,0 +1 @@
+**/hive*

.github/workflows/data/mongodb/ignored.txt

@@ -0,0 +1 @@
+docs

.github/workflows/data/mongodb/matrix.yml

@@ -0,0 +1,33 @@
+min: &min
+  # MongoDB connector does not support Spark 2
+  spark-version: 3.2.3
+  python-version: '3.7'
+  java-version: 8
+  os: ubuntu-latest
+
+max: &max
+  spark-version: 3.4.0
+  python-version: '3.11'
+  java-version: 17
+  os: ubuntu-latest
+
+latest: &latest
+  spark-version: latest
+  python-version: '3.11'
+  java-version: 17
+  os: ubuntu-latest
+
+matrix:
+  small:
+  - mongodb-version: 6.0.5
+    <<: *max
+  full:
+  - mongodb-version: 4.0.0
+    <<: *min
+  - mongodb-version: 6.0.5
+    <<: *max
+  nightly:
+  - mongodb-version: 4.0.0
+    <<: *min
+  - mongodb-version: latest
+    <<: *latest

.github/workflows/data/mongodb/tracked.txt

@@ -0,0 +1 @@
+**/mongodb*

.github/workflows/data/mssql/ignored.txt

@@ -0,0 +1 @@
+docs