Setup ruff format and check (#1744)

- Format python files with ruff
- Update ruff to 0.8.5
- Run format check on CI

.github/workflows/ci-python.yml

@@ -39,6 +39,10 @@ jobs:
         run: poetry install --with dev --all-extras --no-root
         working-directory: python-package
 
+      - name: Format check
+        run: poetry run ruff format --check .
+        working-directory: python-package
+
       - name: Setup config
         env:
           GOOGLE_CREDENTIALS: ${{ secrets.GCP_SA_KEY }}

python-package/basedosdados/backend.py

@@ -383,7 +383,12 @@ def _get_dataset_id_from_name(self, gcp_dataset_id):
         response = self._execute_query(query=query, variables=variables)
         r = {} if response is None else self._simplify_response(response)
         if r.get("allCloudtable") != []:
-            return r.get("allCloudtable")["items"][0].get("table").get("dataset").get("_id")
+            return (
+                r.get("allCloudtable")["items"][0]
+                .get("table")
+                .get("dataset")
+                .get("_id")
+            )
         msg = f"{gcp_dataset_id} not found. Please create the metadata first in {self.graphql_url}"
         logger.info(msg)
         return None

python-package/basedosdados/constants.py

@@ -1,6 +1,7 @@
 """
 Constants for the project.
 """
+
 __all__ = ["config", "constants"]
 
 from dataclasses import dataclass

python-package/basedosdados/download/metadata.py

@@ -1,6 +1,7 @@
 """
 Functions to get metadata from BD's API
 """
+
 from functools import wraps
 
 from basedosdados.backend import Backend

python-package/basedosdados/upload/connection.py

@@ -2,7 +2,6 @@
 Module for managing BigQuery Connections.
 """
 
-
 from typing import Union
 
 import google.auth

python-package/basedosdados/upload/storage.py

@@ -402,7 +402,8 @@ def delete_table(self, mode="staging", bucket_name=None, not_found_ok=False):
             )
         # Divides table_blobs list for maximum batch request size
         table_blobs_chunks = [
-            table_blobs[i : i + 999] for i in range(0, len(table_blobs), 999)  # noqa
+            table_blobs[i : i + 999]
+            for i in range(0, len(table_blobs), 999)  # noqa
         ]
 
         for i, source_table in enumerate(

python-package/basedosdados/upload/utils.py

@@ -1,4 +1,4 @@
-""" Utilities functions for Upload sub-module"""
+"""Utilities functions for Upload sub-module"""
 
 import os
 from pathlib import Path

python-package/pyproject.toml

@@ -33,7 +33,7 @@ tqdm = "^4"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^3.3.3"
-ruff = "^0.4.3"
+ruff = "^0.8.5"
 semgrep = "^1.36.0"
 taskipy = "^1.12.0"
 
@@ -51,6 +51,11 @@ all = ["gql", "pandavro", "requests-toolbelt"]
 avro = ["pandavro"]
 upload = ["gql", "requests-toolbelt"]
 
+[tool.ruff]
+exclude = [
+    "tests/outdated"
+]
+
 [tool.pytest.ini_options]
 pythonpath = "."
 addopts = [

python-package/tests/test_backend.py

@@ -13,6 +13,7 @@ def test_get_dataset():
     assert len(out) != 0
     assert out["page_total"] > 100
 
+
 def test_get_dataset_with_input():
     """
     Test dataset output with input
@@ -25,6 +26,7 @@ def test_get_dataset_with_input():
     assert len(out) != 0
     assert out["page_size"] > 1
 
+
 def test_get_tables():
     """
     Test tables output without parameters
@@ -38,6 +40,7 @@ def test_get_tables():
     assert out["page_size"] >= 10
     assert out["page_total"] > 79
 
+
 def test_get_tables_with_input():
     """
     Test tables output with input
@@ -51,6 +54,7 @@ def test_get_tables_with_input():
     assert out["page_size"] >= 10
     assert out["page_total"] == 0
 
+
 def test_get_columns():
     """
     Test columns output without parameters
@@ -64,6 +68,7 @@ def test_get_columns():
     assert out["page_size"] > 1
     assert out["page_total"] > 3000
 
+
 def test_get_columns_with_input():
     """
     Test columns output with input
@@ -77,6 +82,7 @@ def test_get_columns_with_input():
     assert out["page_size"] >= 10
     assert out["page_total"] > 0
 
+
 def test_search():
     """
     Test search output without parameters
@@ -89,6 +95,7 @@ def test_search():
     assert len(out) != 0
     assert out["page_size"] > 1
 
+
 def test_search_with_input():
     """
     Test search output with input
@@ -101,25 +108,29 @@ def test_search_with_input():
     assert len(out) != 0
     assert out["page_size"] >= 10
 
+
 def test_get_dataset_config():
     """
     Test get_dataset_config output
     """
     backend = Backend()
 
-    out = backend.get_dataset_config(dataset_id='br_me_rais')
+    out = backend.get_dataset_config(dataset_id="br_me_rais")
     assert isinstance(out, dict)
     assert len(out) > 0
     assert out != ""
     assert out != "None" or "null" or "NaN" or "inf" or "-inf" or "nan"
 
+
 def test_get_table_config():
     """
     Test get_dataset_config output
     """
     backend = Backend()
 
-    out = backend.get_table_config(dataset_id='br_me_rais', table_id='microdados_estabelecimentos')
+    out = backend.get_table_config(
+        dataset_id="br_me_rais", table_id="microdados_estabelecimentos"
+    )
     print(out)
     print(len(out))
     print(type(out))
@@ -128,27 +139,30 @@ def test_get_table_config():
     assert out != ""
     assert out != "None" or "null" or "NaN" or "inf" or "-inf" or "nan"
 
+
 def test_get_dataset_id_from_name():
- """
-   Test get dataset id from name
- """
- backend = Backend()
+    """
+    Test get dataset id from name
+    """
+    backend = Backend()
+
+    out = backend._get_dataset_id_from_name("br_me_rais")
+    assert isinstance(out, str)
+    assert len(out) > 30
+    assert out == "3e7c4d58-96ba-448e-b053-d385a829ef00"
 
- out = backend._get_dataset_id_from_name('br_me_rais')
- assert isinstance(out, str)
- assert len(out) > 30
- assert out == '3e7c4d58-96ba-448e-b053-d385a829ef00'
 
 def test_get_table_id_from_name():
- """
-   Test get table id from name
- """
- backend = Backend()
+    """
+    Test get table id from name
+    """
+    backend = Backend()
+
+    out = backend._get_table_id_from_name("br_me_rais", "microdados_estabelecimentos")
+    assert isinstance(out, str)
+    assert len(out) > 35
+    assert out == "86b69f96-0bfe-45da-833b-6edc9a0af213"
 
- out = backend._get_table_id_from_name('br_me_rais', 'microdados_estabelecimentos')
- assert isinstance(out, str)
- assert len(out) > 35
- assert out == '86b69f96-0bfe-45da-833b-6edc9a0af213'
 
 def test_execute_query():
     """
@@ -198,6 +212,7 @@ def test_execute_query():
     assert len(out) != 0
     assert out["allDataset"]["page_total"] > 100
 
+
 def test_simplify_response():
     """
     Test simplify query

python-package/tests/test_download/test_download.py

@@ -111,13 +111,14 @@ def test_read_sql_invalid_billing_project_id():
             from_file=True,
         )
 
+
 @pytest.mark.skip(reason="TODO: Refactor the exceptions that are thrown in read_sql")
 def test_read_sql_inexistent_project():
     """
     Test if the `read_sql` function raises an error when the billing project id is not valid.
     """
 
-    # this is the exception throw BaseDosDadosAccessDeniedException 
+    # this is the exception throw BaseDosDadosAccessDeniedException
     with pytest.raises(GenericGBQException) as excinfo:
         read_sql(
             query="select * from `asedosdados.br_ibge_pib.municipio` limit 10",