From e673fc273df5786848f4a9c4bc0e4e26cf3767ed Mon Sep 17 00:00:00 2001 From: Dimitris Papagiannis Date: Tue, 14 May 2024 14:59:53 +0200 Subject: [PATCH] Added method for changing run class, fix for method edit lumisections, update tests. (#3) * Adapted advanced rr usage tests, added extra function * Added a method for changing the run class * Update CI, cleanup test * Fix for edit_rr_lumisections, cleanup, readme * Verbose tests * Added a test for edit_rr_lumisections * Cleanup instruction --- .github/workflows/test_package.yaml | 11 ++- .gitignore | 7 +- .vscode/launch.json | 102 +++++++++--------------- README.md | 29 +++---- pytest.ini | 4 + runregistry/__init__.py | 2 +- runregistry/runregistry.py | 114 +++++++++++++++++++++------ runregistry/utils.py | 19 ++++- testing-requirements.txt | 3 +- tests/advanced_rr_operations.py | 38 --------- tests/test_advanced_rr_operations.py | 88 +++++++++++++++++++++ tests/test_client.py | 4 +- 12 files changed, 261 insertions(+), 160 deletions(-) create mode 100644 pytest.ini delete mode 100644 tests/advanced_rr_operations.py create mode 100644 tests/test_advanced_rr_operations.py diff --git a/.github/workflows/test_package.yaml b/.github/workflows/test_package.yaml index 80795fc..cbaf807 100644 --- a/.github/workflows/test_package.yaml +++ b/.github/workflows/test_package.yaml @@ -4,16 +4,15 @@ on: [push] jobs: build: - runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -30,9 +29,9 @@ jobs: # # default set of ruff rules with GitHub Annotations # ruff --format=github --target-version=py37 . - name: Test with pytest - env: + env: SSO_CLIENT_ID: ${{ secrets.SSO_CLIENT_ID }} SSO_CLIENT_SECRET: ${{ secrets.SSO_CLIENT_SECRET }} ENVIRONMENT: ${{ vars.ENVIRONMENT }} run: | - pytest tests \ No newline at end of file + pytest tests -s diff --git a/.gitignore b/.gitignore index 2ff0c4e..0b77ca1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ /certs - # Created by https://www.gitignore.io/api/python,pycharm+all # Edit at https://www.gitignore.io/?templates=python,pycharm+all @@ -86,7 +85,7 @@ modules.xml *.ipr # Sonarlint plugin - .idea/sonarlint +.idea/sonarlint ### Python ### # Byte-compiled / optimized / DLL files @@ -209,4 +208,6 @@ dmypy.json ### Python Patch ### .venv/ -# End of https://www.gitignore.io/api/python,pycharm+all \ No newline at end of file +# End of https://www.gitignore.io/api/python,pycharm+all + +.env* diff --git a/.vscode/launch.json b/.vscode/launch.json index de126a5..78246ca 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,66 +1,40 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: Current File (Integrated Terminal)", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "debugStdLib": true, - "env": { - "ENVIRONMENT": "staging" - } - }, - { - "name": "Python: Remote Attach", - "type": "python", - "request": "attach", - "port": 5678, - "host": "localhost", - "pathMappings": [ - { - "localRoot": "${workspaceFolder}", - "remoteRoot": "." - } - ] - }, - { - "name": "Python: Module", - "type": "python", - "request": "launch", - "module": "enter-your-module-name-here", - "console": "integratedTerminal" - }, - { - "name": "Python: Django", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/manage.py", - "console": "integratedTerminal", - "args": ["runserver", "--noreload", "--nothreading"], - "django": true - }, - { - "name": "Python: Flask", - "type": "python", - "request": "launch", - "module": "flask", - "env": { - "FLASK_APP": "app.py" - }, - "args": ["run", "--no-debugger", "--no-reload"], - "jinja": true - }, - { - "name": "Python: Current File (External Terminal)", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "externalTerminal" - } - ] + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File (Integrated Terminal)", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false, + "env": { + "ENVIRONMENT": "staging" + } + }, + { + "name": "Python: Module", + "type": "debugpy", + "request": "launch", + "module": "enter-your-module-name-here", + "console": "integratedTerminal" + }, + { + "name": "Python: Current File (External Terminal)", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "externalTerminal" + }, + { + "name": "Python: Update runs", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/update_specific_runs.py", + "args": ["319528"] + } + ] } diff --git a/README.md b/README.md index abfaf4a..b768b23 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Python version>=3.6 is required for this package. A virtual environment is also required, if you are in lxplus you should run the following commands: ```bash -virtualenv -p `which python3` venv +virtualenv -p $(which python3) venv source venv/bin/activate ``` @@ -32,27 +32,27 @@ pip install runregistry ## Authentication Prerequisites > **Warning** -> Grid certificates have been deprecated by CERN. As of version `1.0.0`, the `runregistry` -> client only works with a client ID and a secret. +> Grid certificates have been deprecated by CERN. As of version `1.0.0`, the `runregistry` +> client only works with a client ID and a secret. -You will need to create an SSO registration for your application which is going to be using the runregistry API client. +You will need to create an SSO registration for your application which is going to be using the runregistry API client. Instructions on how to do it can be found on the [`cernrequests`](https://github.com/CMSTrackerDPG/cernrequests) GitHub page. -Once you have a client ID and a secret, you will need to store them in a file named `.env`. A [sample file](.env_sample) is provided so that you can edit it and rename it to `.env`. +Once you have a client ID and a secret, you will need to store them in a file named `.env`. A [sample file](.env_sample) is provided so that you can edit it and rename it to `.env`. Alternatively, you can run `export SSO_CLIENT_ID=...` and `export SSO_CLIENT_SECRET=...` on the same terminal that you will be running your python script in. ## Usage -### Get a single run (get_run): +### Get a single run (get_run) ```python import runregistry run = runregistry.get_run(run_number=328762) ``` -### Query several runs (get_runs): +### Query several runs (get_runs) ```python import runregistry @@ -396,7 +396,7 @@ You can also manipulate runs via API: ```python runregistry.reset_RR_attributes_and_refresh_runs(run=362761) ``` -3. Move runs from one state to another: +3. Move runs from one state to another: ```python runregistry.move_runs("OPEN", "SIGNOFF", run=362761) ``` @@ -416,20 +416,17 @@ python3 -m twine upload --skip-existing --repository pypi dist/* ``` Instructions from [here](https://packaging.python.org/en/latest/tutorials/packaging-projects/). -## Testing +## Running the tests ### Locally -> **TODO** -> Remove the qa environment after migration. +You will be needing a file named `.env` with the following variables: -You will be needing a file named `.env` with the following variables ```bash SSO_CLIENT_ID= SSO_CLIENT_SECRET= -ENVIRONMENT=qa +ENVIRONMENT=development ``` -While most of the tests work on the development deployment, some fail and need the production one. This is the reason we are setting `ENVIRONMENT=qa`. ```bash python3 -m venv venv @@ -458,8 +455,8 @@ No*. Our recommendation is to query Run Registry only for data that RR is responsible for. -*It's not that you can't, it's just that this puts extra burden on the application, making it slow for everyone. +*It's not that you can't, it's just that this puts extra burden on the application, making it slow for everyone. ### Is the token stored somewhere and reused? -No, almost every function call gets a new token. This is not ideal, and it may be improved in the future. \ No newline at end of file +No, almost every function call gets a new token. This is not ideal, and it may be improved in the future. \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..87b78bc --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +log_level = INFO +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/runregistry/__init__.py b/runregistry/__init__.py index 6440e8d..af98058 100644 --- a/runregistry/__init__.py +++ b/runregistry/__init__.py @@ -2,4 +2,4 @@ # To update: # pip install wheel && pip install twine && python setup.py bdist_wheel && twine upload --skip-existing dist/* -__version__ = "1.0.0" +__version__ = "1.1.0" diff --git a/runregistry/runregistry.py b/runregistry/runregistry.py index b63d431..499d1d3 100644 --- a/runregistry/runregistry.py +++ b/runregistry/runregistry.py @@ -1,17 +1,25 @@ import os +import time import json import requests -import time from dotenv import load_dotenv from cernrequests import get_api_token, get_with_token -from runregistry.utils import transform_to_rr_run_filter, transform_to_rr_dataset_filter +from runregistry.utils import ( + transform_to_rr_run_filter, + transform_to_rr_dataset_filter, + __parse_runs_arg, +) load_dotenv() + # Silence unverified HTTPS warning: # urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) PAGE_SIZE = 50 +# Offline table +WAITING_DQM_GUI_CONSTANT = "waiting dqm gui" + staging_cert = "" staging_key = "" api_url = "" @@ -360,7 +368,7 @@ def move_runs(from_, to_, run=None, runs=[], **kwargs): move run/runs from one state to another """ if not run and not runs: - print("move_runs(): no 'run' and 'runs' arguments were provided, return") + print("move_runs(): no 'run' and 'runs' arguments were provided") return states = ["SIGNOFF", "OPEN", "COMPLETED"] @@ -388,7 +396,7 @@ def move_runs(from_, to_, run=None, runs=[], **kwargs): for run_number in runs: payload = json.dumps({"run_number": run_number}) answer = requests.post(url, headers=headers, data=payload).json() - answers += [answer] + answers.append(answer) return answers @@ -398,7 +406,7 @@ def make_significant_runs(run=None, runs=[], **kwargs): mark run/runs significant """ if not run and not runs: - print("move_runs(): no 'run' and 'runs' arguments were provided, return") + print("make_significant_runs(): no 'run' and 'runs' arguments were provided") return url = "%s/runs/mark_significant" % (api_url) @@ -413,32 +421,50 @@ def make_significant_runs(run=None, runs=[], **kwargs): for run_number in runs: data = {"run_number": run} answer = requests.post(url, headers=headers, json=data) - answers += [answer] + answers.append(answer) return answers -def reset_RR_attributes_and_refresh_runs(run=None, runs=[], **kwargs): +def reset_RR_attributes_and_refresh_runs(runs=[], **kwargs): """ reset RR attributes and refresh run/runs """ - if not run and not runs: - print("move_runs(): no 'run' and 'runs' arguments were provided, return") + runs = __parse_runs_arg(runs) + if not runs: + print( + "reset_RR_attributes_and_refresh_runs(): no 'runs' arguments were provided" + ) return + headers = _get_headers(token=_get_token()) + answers = [] + for run_number in runs: + url = "%s/runs/reset_and_refresh_run/%d" % (api_url, run_number) + answer = requests.post(url, headers=headers) + answers.append(answer) - url = "%s/runs/reset_and_refresh_run" % (api_url) + return answers - headers = _get_headers(token=_get_token()) - if run: - url = "%s/runs/reset_and_refresh_run/%d" % (api_url, run) - return requests.post(url, headers=headers) +def manually_refresh_components_statuses_for_runs(runs=[], **kwargs): + """ + Refreshes all components statuses for the runs specified that have not been + changed by shifters. + """ + runs = __parse_runs_arg(runs) + + if not runs: + print( + "manually_refresh_components_statuses_for_runs(): no 'runs' arguments were provided, return" + ) + return + headers = _get_headers(token=_get_token()) answers = [] for run_number in runs: - url = "%s/runs/reset_and_refresh_run/%d" % (api_url, run_number) + url = "%s/runs/refresh_run/%d" % (api_url, run_number) answer = requests.post(url, headers=headers) - answers += [answer] + answers.append(answer) return answers @@ -452,7 +478,7 @@ def edit_rr_lumisections( comment="", cause="", dataset_name="online", - **kwargs + **kwargs, ): """ WIP edit RR lumisections attributes @@ -460,7 +486,7 @@ def edit_rr_lumisections( states = ["GOOD", "BAD", "STANDBY", "EXCLUDED", "NONSET"] if status not in states: print( - "move_runs(): get status '", + "edit_rr_lumisections(): get status '", status, "', while allowed statuses are ", states, @@ -485,18 +511,15 @@ def edit_rr_lumisections( "component": component, } ) - return requests.post(url, headers=headers, data=payload) - - -# Offline table -WAITING_DQM_GUI_CONSTANT = "waiting dqm gui" + return requests.put(url, headers=headers, data=payload) def move_datasets( from_, to_, dataset_name, workspace="global", run=None, runs=[], **kwargs ): """ - move offline dataset/datasets from one state to another + Move offline dataset/datasets from one state to another. + Requires a privileged token. """ if not run and not runs: print("move_datasets(): no 'run' and 'runs' arguments were provided, return") @@ -528,9 +551,48 @@ def move_datasets( answers = [] for run_number in runs: payload = json.dumps( - {"run_number": run, "dataset_name": dataset_name, "workspace": workspace} + { + "run_number": run_number, + "dataset_name": dataset_name, + "workspace": workspace, + } ) answer = requests.post(url, headers=headers, data=payload).json() - answers += [answer] + answers.append(answer) return answers + + +def change_run_class(run_numbers, new_class): + """ + Method for changing the class of a run (or runs), + e.g. from "Commissioning22" to "Cosmics22". + Requires a privileged token. + """ + headers = _get_headers(token=_get_token()) + + def _execute_request_for_single_run(run_number, new_class): + payload = json.dumps({"class": new_class}) + return requests.put( + url="%s/manual_run_edit/%s/class" % (api_url, run_number), + headers=headers, + data=payload, + ) + + if not isinstance(new_class, str): + raise Exception('Invalid input for "new_class"') + answers = [] + if isinstance(run_numbers, list): + for run_number in run_numbers: + if not isinstance(run_number, int): + raise Exception( + "Invalid run number value found in run_numbers. Please provide a list of numbers." + ) + answers.append(_execute_request_for_single_run(run_number, new_class)) + elif isinstance(run_numbers, int): + answers.append(_execute_request_for_single_run(run_numbers, new_class)) + else: + raise Exception( + 'Invalid input for "run_numbers". Please provide a list of numbers.' + ) + return answers diff --git a/runregistry/utils.py b/runregistry/utils.py index 1dc15d3..ed50842 100644 --- a/runregistry/utils.py +++ b/runregistry/utils.py @@ -1,5 +1,3 @@ -import json - from runregistry.attributes import ( run_table_attributes, run_triplet_attributes, @@ -11,6 +9,23 @@ ) +def __parse_runs_arg(runs): + """ + Helper function to parse runs arguments. + Returns a list. + """ + if isinstance(runs, int): + return [runs] + elif isinstance(runs, str): + try: + runs = int(runs) + return [runs] + except: + return [] + elif isinstance(runs, list): + return runs + + def transform_to_rr_run_filter(run_filter): """ Transforms a filter to a compatible filter that RR back end understands. diff --git a/testing-requirements.txt b/testing-requirements.txt index fbcfd7a..51baf05 100644 --- a/testing-requirements.txt +++ b/testing-requirements.txt @@ -1,3 +1,4 @@ pytest>=3.6 pytest-cov -codecov \ No newline at end of file +codecov +-e . \ No newline at end of file diff --git a/tests/advanced_rr_operations.py b/tests/advanced_rr_operations.py deleted file mode 100644 index 10876d5..0000000 --- a/tests/advanced_rr_operations.py +++ /dev/null @@ -1,38 +0,0 @@ -import sys, os - -sys.path.append(os.path.dirname(os.path.realpath("../runregistry"))) - -import runregistry - -runregistry.setup("development") - -# answer = runregistry.edit_rr_lumisections(363534, 101, 111, "gem", "GOOD", "API TEST") -# print( answer, answer.text ) -# exit() - -answer = runregistry.move_datasets( - "waiting dqm gui", - "OPEN", - "/PromptReco/Commissioning2021/DQM", - run=362874, - workspace="global", -) -print(answer, answer.text) - -answer = runregistry.move_datasets( - "OPEN", - "SIGNOFF", - "/PromptReco/Commissioning2021/DQM", - run=362874, - workspace="ctpps", -) -print(answer, answer.text) - -answer = runregistry.make_significant_runs(run=362761) -print(answer, answer.text) - -answer = runregistry.reset_RR_attributes_and_refresh_runs(run=362761) -print(answer, answer.text) - -answer = runregistry.move_runs("OPEN", "SIGNOFF", run=362761) -print(answer, answer.text) diff --git a/tests/test_advanced_rr_operations.py b/tests/test_advanced_rr_operations.py new file mode 100644 index 0000000..15d4272 --- /dev/null +++ b/tests/test_advanced_rr_operations.py @@ -0,0 +1,88 @@ +import pytest +import logging +import runregistry + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def setup_runregistry(): + logger.info("Connecting to development runregistry") + runregistry.setup("development") + + +def test_move_datasets(setup_runregistry): + answer = runregistry.move_datasets( + from_=runregistry.WAITING_DQM_GUI_CONSTANT, + to_="OPEN", + dataset_name="/PromptReco/Commissioning2021/DQM", + run=362874, + workspace="global", + ) + # TODO: Run also with a token that has permission + assert answer.status_code == 401 + answer = runregistry.move_datasets( + from_="OPEN", + to_="SIGNOFF", + dataset_name="/PromptReco/Commissioning2021/DQM", + run=362874, + workspace="ctpps", + ) + # Requires permission + assert answer.status_code == 401 + + +def test_make_significant_runs(setup_runregistry): + # Get latest run in dev runregistry and make it significant + run = runregistry.get_runs(limit=1, filter={})[0] + answer = runregistry.make_significant_runs(run=run["run_number"]) + # requires permission + assert answer.status_code == 401 + + +def test_reset_RR_attributes_and_refresh_runs_signed_off(setup_runregistry): + answers = runregistry.reset_RR_attributes_and_refresh_runs(runs=362761) + # Cannot refresh runs which are not open + assert all( + [ + answer.status_code == 500 and "Run must be in state OPEN" in answer.text + for answer in answers + ] + ) + + +def test_manually_refresh_components_statuses_for_runs_open(setup_runregistry): + run = runregistry.get_runs(limit=1, filter={})[0] + answers = runregistry.manually_refresh_components_statuses_for_runs( + runs=run["run_number"] + ) + assert all([answer.status_code == 200 for answer in answers]) + + +def test_reset_RR_attributes_and_refresh_runs_open(setup_runregistry): + run = runregistry.get_runs(limit=1, filter={})[0] + answers = runregistry.reset_RR_attributes_and_refresh_runs(runs=run["run_number"]) + assert all([answer.status_code == 200 for answer in answers]) + + +def test_manually_refresh_components_statuses_for_runs_signed_off(setup_runregistry): + answers = runregistry.manually_refresh_components_statuses_for_runs(runs=362761) + # Cannot refresh runs which are not open + assert all( + [ + answer.status_code == 500 and "Run must be in state OPEN" in answer.text + for answer in answers + ] + ) + + +def test_move_runs(setup_runregistry): + answer = runregistry.move_runs("OPEN", "SIGNOFF", run=362761) + # Requires permission + assert answer.status_code == 401 + + +def test_edit_rr_lumisections(setup_runregistry): + answer = runregistry.edit_rr_lumisections(380326, 0, 1, "castor-castor", "GOOD") + # Requires permission + assert answer.status_code == 401 diff --git a/tests/test_client.py b/tests/test_client.py index 3d3d7d9..4bcf721 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,5 +1,3 @@ -import os - import pytest import json @@ -49,7 +47,7 @@ def test_get_runs(): # Gets runs that contain lumisections that classified DT as GOOD AND lumsiections that classified hcal as STANDBY filter_run = { "run_number": {"and": [{">": 309000}, {"<": 310000}]}, - "dt-dt": "GOOD" + "dt-dt": "GOOD", # 'hcal': 'STANDBY' }