Merge branch 'main' into pr/ggit12/2731

.azure-pipelines.yml

@@ -1,101 +1,117 @@
 trigger:
-- master
+- main
+- "*.*.x"
 
 variables:
-  python.version: '3.11'
-  PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
-  PYTEST_ADDOPTS: '-v --color=yes --nunit-xml=nunit/test-results.xml'
-  ANNDATA_DEV: no
-  RUN_COVERAGE: no
+  python.version: '3.12'
+  PYTEST_ADDOPTS: '-v --color=yes --internet-tests --nunit-xml=test-data/test-results.xml'
   TEST_EXTRA: 'test-full'
-  PRERELEASE_DEPENDENCIES: no
+  DEPENDENCIES_VERSION: "latest"  # |"pre-release" | "minimum-version"
+  TEST_TYPE: "standard" # | "coverage"
 
 jobs:
 - job: PyTest
   pool:
     vmImage: 'ubuntu-22.04'
   strategy:
     matrix:
-      Python3.9:
-        python.version: '3.9'
-      Python3.11: {}
-      minimal_tests:
+      Python3.10:
+        python.version: '3.10'
+      Python3.12: {}
+      minimal_dependencies:
         TEST_EXTRA: 'test-min'
       anndata_dev:
-        ANNDATA_DEV: yes
-        RUN_COVERAGE: yes
-        PRERELEASE_DEPENDENCIES: yes
+        DEPENDENCIES_VERSION: "pre-release"
+        TEST_TYPE: "coverage"
+      minimum_versions:
+        python.version: '3.10'
+        DEPENDENCIES_VERSION: "minimum-version"
+        TEST_TYPE: "coverage"
+
 
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '$(python.version)'
     displayName: 'Use Python $(python.version)'
 
+  - script: |
+      python -m pip install --upgrade uv
+      echo "##vso[task.setvariable variable=uv_cache_dir]`uv cache dir`"
+    displayName: 'Install tools'
+
   - task: Cache@2
     inputs:
       key: '"python $(python.version)" | "$(Agent.OS)" | pyproject.toml'
       restoreKeys: |
-        python | "$(Agent.OS)"
-        python
-      path: $(PIP_CACHE_DIR)
+        "python" | "$(Agent.OS)"
+        "python"
+      path: $(uv_cache_dir)
     displayName: Cache pip packages
 
+
+  - task: Cache@2
+    inputs:
+      key: '"pytest"'
+      restoreKeys: '"pytest"'
+      path: $(System.DefaultWorkingDirectory)/.pytest_cache/d
+    displayName: Cache pytest data
+
   - script: |
       export MPLBACKEND="agg"
       echo $MPLBACKEND
     displayName: 'Set env'
 
-  - script: |
-      python -m pip install --upgrade pip
-      pip install wheel coverage
-      pip install .[dev,$(TEST_EXTRA)]
+  - script: uv pip install --system --compile 'scanpy[dev,$(TEST_EXTRA)] @ .'
     displayName: 'Install dependencies'
-    condition: eq(variables['PRERELEASE_DEPENDENCIES'], 'no')
+    condition: eq(variables['DEPENDENCIES_VERSION'], 'latest')
 
-  - script: |
-      python -m pip install --pre --upgrade pip
-      pip install --pre wheel coverage
-      pip install --pre .[dev,$(TEST_EXTRA)]
+  - script: >
+      uv pip install --system --compile --pre
+      "scanpy[dev,$(TEST_EXTRA)] @ ."
+      "anndata[dev,test] @ git+https://github.com/scverse/anndata"
     displayName: 'Install dependencies release candidates'
-    condition: eq(variables['PRERELEASE_DEPENDENCIES'], 'yes')
+    condition: eq(variables['DEPENDENCIES_VERSION'], 'pre-release')
 
   - script: |
-      pip install -v "anndata[dev,test] @ git+https://github.com/scverse/anndata"
-    displayName: 'Install development anndata'
-    condition: eq(variables['ANNDATA_DEV'], 'yes')
-
-  - script: |
-      pip list
-    displayName: 'Display installed versions'
+      uv pip install --system --compile tomli packaging
+      deps=`python3 ci/scripts/min-deps.py pyproject.toml --extra dev test`
+      uv pip install --system --compile $deps "scanpy @ ."
+    displayName: 'Install dependencies minimum version'
+    condition: eq(variables['DEPENDENCIES_VERSION'], 'minimum-version')
 
   - script: pytest
     displayName: 'PyTest'
-    condition: eq(variables['RUN_COVERAGE'], 'no')
+    condition: eq(variables['TEST_TYPE'], 'standard')
 
   - script: |
-      coverage run -m pytest
-      coverage xml
+      pytest --cov --cov-report=xml --cov-context=test
     displayName: 'PyTest (coverage)'
-    condition: eq(variables['RUN_COVERAGE'], 'yes')
+    condition: eq(variables['TEST_TYPE'], 'coverage')
 
-  - task: PublishCodeCoverageResults@1
+  - task: PublishCodeCoverageResults@2
     inputs:
       codeCoverageTool: Cobertura
-      summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
+      summaryFileLocation: 'test-data/coverage.xml'
       failIfCoverageEmpty: true
-    condition: eq(variables['RUN_COVERAGE'], 'yes')
+    condition: eq(variables['TEST_TYPE'], 'coverage')
 
   - task: PublishTestResults@2
     condition: succeededOrFailed()
     inputs:
-      testResultsFiles: 'nunit/test-results.xml'
+      testResultsFiles: 'test-data/test-results.xml'
       testResultsFormat: NUnit
-      testRunTitle: 'Publish test results for Python $(python.version)'
+      testRunTitle: 'Publish test results for $(Agent.JobName)'
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      pathToPublish: '.pytest_cache/d/debug'
+      artifactName: debug-data
+    condition: eq(variables['TEST_TYPE'], 'coverage')
 
   - script: bash <(curl -s https://codecov.io/bash)
     displayName: 'Upload to codecov.io'
-    condition: eq(variables['RUN_COVERAGE'], 'yes')
+    condition: eq(variables['TEST_TYPE'], 'coverage')
 
 - job: CheckBuild
   pool:
@@ -104,8 +120,8 @@ jobs:
 
   - task: UsePythonVersion@0
     inputs:
-      versionSpec: '3.11'
-    displayName: 'Use Python 3.11'
+      versionSpec: '3.12'
+    displayName: 'Use Python 3.12'
 
   - script: |
       python -m pip install --upgrade pip

.codecov.yml

@@ -1,17 +1,16 @@
 # Based on pydata/xarray
 codecov:
-  require_ci_to_pass: no
+  require_ci_to_pass: false
 
 coverage:
   status:
     project:
       default:
-        # Require 1% coverage, i.e., always succeed
-        target: 1
-    patch: false
+        # Require 75% coverage
+        target: 75
     changes: false
 
 comment:
   layout: "diff, flags, files"
   behavior: once
-  require_base: no
+  require_base: false

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -1,7 +1,9 @@
 name: Bug report
 description: Scanpy doesn’t do what it should? Please help us fix it!
 #title: ...
-labels: ['Bug 🐛']
+type: Bug
+labels:
+- Triage 🩺
 #assignees: []
 body:
 - type: checkboxes
@@ -14,7 +16,7 @@ body:
       required: true
     - label: I have confirmed this bug exists on the latest version of scanpy.
       required: true
-    - label: (optional) I have confirmed this bug exists on the master branch of scanpy.
+    - label: (optional) I have confirmed this bug exists on the main branch of scanpy.
       required: false
 - type: markdown
   attributes:

.github/ISSUE_TEMPLATE/enhancement-request.yml

@@ -1,7 +1,9 @@
 name: Enhancement request
 description: Anything you’d like to see in scanpy?
 #title: ...
-labels: ['Enhancement ✨']
+type: Enhancement
+labels:
+- Triage 🩺
 #assignees: []
 body:
 - type: dropdown
@@ -12,6 +14,7 @@ body:
     - 'Additional function parameters / changed functionality / changed defaults?'
     - 'New analysis tool: A simple analysis tool you have been using and are missing in `sc.tools`?'
     - 'New plotting function: A kind of plot you would like to seein `sc.pl`?'
+    - 'Improved documentation or error message?'
     - 'Other?'
   validations:
     required: true

.github/workflows/benchmark.yml

@@ -0,0 +1,59 @@
+name: Benchmark
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  benchmark:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -e {0} # -e to fail on error
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python: ["3.12"]
+        os: [ubuntu-latest]
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python }}
+      ASV_DIR: "./benchmarks"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: blob:none
+
+      - name: Fetch main branch for `asv run`’s hash
+        run: git fetch origin main:main
+        if: ${{ github.ref_name != 'main' }}
+
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: 'pip'
+
+      - name: Cache datasets
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache
+          key: benchmark-state-${{ hashFiles('benchmarks/**') }}
+
+      - name: Install dependencies
+        run: pip install 'asv>=0.6.4'
+
+      - name: Configure ASV
+        working-directory: ${{ env.ASV_DIR }}
+        run: asv machine --yes
+
+      - name: Quick benchmark run
+        working-directory: ${{ env.ASV_DIR }}
+        run: asv run --dry-run --quick --show-stderr --verbose HEAD^!

.github/workflows/check-pr.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - master
     types:
       # milestone changes
       - milestoned
@@ -46,14 +45,17 @@ jobs:
     needs: check-milestone
     if: ${{ needs.check-milestone.outputs.no-relnotes-reason == '' && !contains(github.event.pull_request.labels.*.name, 'Development Process 🚀') }}
     steps:
-      - uses: actions/checkout@v3
-      - name: Find out if relevant release notes are modified
-        uses: dorny/paths-filter@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: blob:none
+      - name: Find out if a relevant release fragment is added
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           filters: | # this is intentionally a string
-            relnotes: 'docs/release-notes/${{ github.event.pull_request.milestone.title }}.md'
-      - name: Check if relevant release notes are modified
+            relnotes: 'docs/release-notes/${{ github.event.pull_request.number }}.*.md'
+      - name: Check if a relevant release fragment is added
         uses: flying-sheep/check@v1
         with:
           success: ${{ steps.changes.outputs.relnotes }}

.github/workflows/publish.yml

@@ -11,8 +11,11 @@ jobs:
     permissions:
       id-token: write # to authenticate as Trusted Publisher to pypi.org
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: blob:none
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.x"
           cache: "pip"

.gitignore

@@ -1,5 +1,4 @@
 # Scanpy outfiles
-/cache/
 /data/
 /write/
 
@@ -12,16 +11,13 @@
 /docs/jupyter_execute
 
 # tests
-/*coverage*
-/nunit/
-/.cache/
-/.pytest_cache/
-/scanpy/tests/test*.h5ad
-/scanpy/tests/**/*actual.png
-/scanpy/tests/**/*failed-diff.png
+/*cache/
+/test-data/
+/tests/test*.h5ad
+/tests/**/*actual.png
+/tests/**/*failed-diff.png
 
 # Environment management
-/hatch.toml
 /Pipfile
 /Pipfile.lock
 /requirements*.lock
@@ -31,10 +27,8 @@
 
 # Python build files
 __pycache__/
-/scanpy/_version.py
+/ci/scanpy-min-deps.txt
 /dist/
-/build/
-/scanpy.egg-info/
 /*-env/
 /env-*/
 /environment.yml
@@ -47,4 +41,7 @@ Thumbs.db
 
 # IDEs and editors
 /.idea/
-/.vscode/
+
+# asv benchmark files
+/benchmarks/.asv
+/benchmarks/data/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "notebooks"]
+	path = notebooks
+	url = https://github.com/scverse/scanpy-tutorials/