Skip to content

Commit

Permalink
Fix all Ruff linting issues across the codebase
Browse files Browse the repository at this point in the history
  • Loading branch information
rcboufleur committed Jan 17, 2025
1 parent 13f1aa4 commit c4115fd
Show file tree
Hide file tree
Showing 16 changed files with 870 additions and 356 deletions.
39 changes: 39 additions & 0 deletions python/lsst/consdb/efd_transform/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""Provides a structured framework for processing and transforming data from the (EFD).
Overview:
---------
This module offers tools for accessing, transforming, and managing EFD data, supporting
workflows and integration with the LSST ecosystem. It includes capabilities for
data retrieval, transformation pipelines, configuration management, and schema
generation.
Components:
-----------
- **Data Access**: The `dao` subpackage provides Data Access Objects (DAOs) to interact
with specific database tables such as `exposure_efd` and `visit_efd`.
- **Data Transformation**: Includes utilities for applying structured transformations
to EFD data, including summarization and restructuring.
- **Configuration Handling**: Supports loading and validating instrument configurations
through YAML files for flexible setup and operation.
- **Schema Generation**: Automates the creation of database schemas based on predefined
instrument configurations.
- **Queue Management**: Implements tools for task scheduling and queue-based workflows.
Submodules:
-----------
- `dao`: Contains DAOs for database interactions.
- `config_model`: Defines models for validating YAML configurations.
- `generate_schema`: Includes schema generation utilities.
- `summary`: Provides tools for summarizing EFD data.
- `transform`: Manages transformation pipelines.
- `transform_efd`: Contains specialized transformation methods.
- `queue_manager`: Handles task queue management.
Configuration Files:
--------------------
The module uses YAML files for instrument-specific configurations:
- `config_LATISS.yaml`: Configuration for LATISS.
- `config_LSSTComCam.yaml`: Configuration for LSSTComCam.
- `config_LSSTComCamSim.yaml`: Configuration for LSSTComCamSim.
"""
63 changes: 59 additions & 4 deletions python/lsst/consdb/efd_transform/config_model.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,55 @@
"""Defines the configuration models used in the application."""

from typing import Dict, List, Optional, Union

from pydantic import BaseModel, model_validator

TABLES = ["exposure_efd", "exposure_efd_unpivoted", "visit1_efd", "visit1_efd_unpivoted"]
TABLES = [
"exposure_efd",
"exposure_efd_unpivoted",
"visit1_efd",
"visit1_efd_unpivoted",
]
UNPIVOTED_TABLES = ["exposure_efd_unpivoted", "visit1_efd_unpivoted"]


class Field(BaseModel):
"""Represents a field with a name.
Attributes
----------
name (str): The name of the field.
"""

name: str


class Topic(BaseModel):
"""Represents a topic with a name and associated fields.
Attributes
----------
name (str): The name of the topic.
fields (List[Field]): A list of fields associated with the topic.
"""

name: str
fields: List[Field]


class Column(BaseModel):
"""Represents a column with a name and optional tables.
Attributes
----------
name (str): The name of the column.
tables (Optional[List[str]]): A list of table names where the column
exists.
"""

name: str
tables: Optional[List[str]] = TABLES
store_unpivoted: Optional[bool] = False
Expand All @@ -31,16 +65,37 @@ class Column(BaseModel):

@model_validator(mode="after")
def validate_tables_when_unpivoted(cls, model):
"""Validate the tables attribute when unpivoting is required.
Args:
----
model (Column): The column model to validate.
Returns:
-------
Column: The validated column model.
"""
# Here 'model' is an instance of 'Column'
if model.store_unpivoted:
invalid_tables = [table for table in model.tables if table not in UNPIVOTED_TABLES]
invalid_tables = [
table for table in model.tables if table not in UNPIVOTED_TABLES
]
if invalid_tables:
raise ValueError(
f"When 'store_unpivoted' is True, only {UNPIVOTED_TABLES} are allowed. "
f"Invalid tables provided: {invalid_tables}"
f"When 'store_unpivoted' is True, only {UNPIVOTED_TABLES} "
f"are allowed. Invalid tables provided: {invalid_tables}"
)
return model


class ConfigModel(BaseModel):
"""Represents the configuration model containing a list of columns.
Attributes
----------
columns (List[Column]): A list of columns in the configuration.
"""

columns: List[Column]
16 changes: 16 additions & 0 deletions python/lsst/consdb/efd_transform/dao/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
"""Provides tools for interacting with the database access objects (DAOs).
Submodules include:
- base: Defines the foundational DAO components such as database engine management
and query execution.
- butler: Provides data access methods for querying dimensions using a Butler object.
- exposure_efd: Handles operations related to the "exposure_efd" table.
- exposure_efd_unpivoted: Manages data for the "exposure_efd_unpivoted" table.
- influxdb: Interfaces with the InfluxDB API for time-series data queries.
- transformd: Manages transformed EFD scheduler data.
- visit_efd: Handles data operations for the "visit1_efd" table.
- visit_efd_unpivoted: Accesses data for the "visit1_efd_unpivoted" table.
These modules collectively facilitate database operations, data transformation, and
efficient retrieval of time-series and structured data.
"""
Loading

0 comments on commit c4115fd

Please sign in to comment.