diff --git a/tests/data/result/adviser/adviser-220131050913-b7c4ef0fc532ac45 b/tests/data/result/adviser/adviser-220131050913-b7c4ef0fc532ac45 new file mode 100644 index 000000000..011cc5f5e --- /dev/null +++ b/tests/data/result/adviser/adviser-220131050913-b7c4ef0fc532ac45 @@ -0,0 +1,162 @@ +{ + "metadata": { + "analyzer": "thoth-adviser", + "analyzer_version": "0.50.0", + "arguments": { + "advise": { + "beam_width": 25000, + "constraints": "input/constraints.txt", + "count": 1, + "dev": false, + "labels": "input/labels.json", + "library_usage": "input/library_usage.json", + "limit": 1, + "limit_latest_versions": -1, + "no_pretty": false, + "output": "/mnt/workdir/adviser-220131050913-b7c4ef0fc532ac45", + "pipeline": null, + "plot": null, + "predictor": "AUTO", + "predictor_config": {}, + "prescription": [ + "/opt/app-root/src/prescriptions.pickle" + ], + "recommendation_type": "latest", + "requirements": "input/Pipfile", + "requirements_format": "pipenv", + "requirements_locked": "input/Pipfile.lock", + "runtime_environment": "input/runtime_environment.json", + "seed": 42, + "user_stack_scoring": true + }, + "thoth-adviser": { + "metadata": { + "kebechet_metadata": null, + "origin": "https://github.com/thoth-station/ps-cv", + "re_run_adviser_id": null, + "source_type": null, + "thoth.adviser": { + "justification": null, + "stack_info": null + } + }, + "verbose": false + } + }, + "datetime": "2022-01-31T05:15:49.969966", + "distribution": { + "codename": "Ootpa", + "id": "rhel", + "like": "fedora", + "version": "8.3", + "version_parts": { + "build_number": "", + "major": "8", + "minor": "3" + } + }, + "document_id": "adviser-220131050913-b7c4ef0fc532ac45", + "duration": 376, + "hostname": "adviser-220131050913-b7c4ef0fc532ac45-1876565397", + "os_release": { + "id": "rhel", + "name": "Red Hat Enterprise Linux", + "platform_id": "platform:el8", + "redhat_bugzilla_product": "Red Hat Enterprise Linux 8", + "redhat_bugzilla_product_version": "8.3", + "redhat_support_product": "Red Hat Enterprise Linux", + "redhat_support_product_version": "8.3", + "version": "8.3 (Ootpa)", + "version_id": "8.3" + }, + "python": { + "api_version": 1013, + "implementation_name": "cpython", + "major": 3, + "micro": 3, + "minor": 8, + "releaselevel": "final", + "serial": 0 + }, + "thoth_deployment_name": "ocp4-stage", + "timestamp": 1643606149 + }, + "result": { + "error": true, + "error_msg": "Resolver was killed as allocated memory has been exceeded (OOM) - https://thoth-station.ninja/j/oom", + "parameters": { + "beam_width": 25000, + "constraints": "input/constraints.txt", + "count": 1, + "dev": false, + "labels": {}, + "library_usage": null, + "limit": 1, + "limit_latest_versions": -1, + "no_pretty": false, + "output": "/mnt/workdir/adviser-220131050913-b7c4ef0fc532ac45", + "pipeline": null, + "plot": null, + "predictor": "AUTO", + "predictor_config": "{}", + "prescription": [ + "/opt/app-root/src/prescriptions.pickle" + ], + "project": { + "constraints": [], + "requirements": { + "dev-packages": {}, + "packages": { + "jupyter-tensorboard": "*", + "keras": "*", + "opencv-python": "*", + "pillow": "*", + "tensorboard": "*", + "tensorflow": ">=2.0.0" + }, + "requires": { + "python_version": "3.8" + }, + "source": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "requirements_locked": null, + "runtime_environment": { + "base_image": null, + "cuda_version": null, + "cudnn_version": null, + "hardware": { + "cpu_family": null, + "cpu_model": null, + "gpu_model": null + }, + "labels": null, + "mkl_version": null, + "name": "ps-cv-tensorflow", + "openblas_version": null, + "openmpi_version": null, + "operating_system": { + "name": "ubi", + "version": "8" + }, + "platform": "linux-x86_64", + "python_version": "3.8", + "recommendation_type": null + } + }, + "recommendation_type": "latest", + "requirements": "input/Pipfile", + "requirements_format": "pipenv", + "requirements_locked": "input/Pipfile.lock", + "runtime_environment": "input/runtime_environment.json", + "seed": 42, + "user_stack_scoring": true + }, + "report": null + } +} diff --git a/tests/test_ceph.py b/tests/test_ceph.py index 6a8a785af..5306d8409 100644 --- a/tests/test_ceph.py +++ b/tests/test_ceph.py @@ -18,9 +18,11 @@ """This is the tests.""" +import json import pytest from moto import mock_s3 +from thoth.storages import AdvisersCacheStore from thoth.storages import CephStore from thoth.storages.exceptions import NotFoundError @@ -59,6 +61,7 @@ _ENV = {**CEPH_INIT_ENV} _BUCKET_PREFIX = "some-prefix/" +_DEPLOYMENT_NAME = "thoth-test-deployment" @pytest.fixture(name="adapter") @@ -79,6 +82,14 @@ def _fixture_connected_adapter(): yield connected_adapter +@pytest.fixture(name="connected_cache_adapter") +def _fixture_connected_cache_adapter(): + """Retrieve a connected cache adapter to Ceph.""" + adapter = AdvisersCacheStore(**CEPH_INIT_KWARGS, prefix=_BUCKET_PREFIX, deployment_name=_DEPLOYMENT_NAME) + with connected_ceph_adapter(adapter, raw_ceph=False) as connected_cache_adapter: + yield connected_cache_adapter + + class TestCephStore(ThothStoragesTest): """Testing class for Ceph operations.""" @@ -181,3 +192,19 @@ def connect(self, adapter): assert not adapter.is_connected() adapter.connect() assert adapter.is_connected() + + +class TestCephCache(ThothStoragesTest): + """Test retrieving properties from cached documents.""" + + def test_retrieve_document_ttl(self, connected_cache_adapter): + """Test retrieving TTL from a cached document.""" + assert connected_cache_adapter.document_exists("adviser-220131050913-b7c4ef0fc532ac45") is False + with open("tests/data/result/adviser/adviser-220131050913-b7c4ef0fc532ac45", "r") as advise_result: + advise_file = json.load(advise_result) + connected_cache_adapter.store_document(advise_file) + assert connected_cache_adapter.document_exists("adviser-220131050913-b7c4ef0fc532ac45") is True + + document_ttl = connected_cache_adapter.retrieve_document_ttl("adviser-220131050913-b7c4ef0fc532ac45") + eps = 10 + assert abs(14400.0 - document_ttl) <= eps diff --git a/thoth/storages/ceph_cache.py b/thoth/storages/ceph_cache.py index d070ce827..774f0d59b 100644 --- a/thoth/storages/ceph_cache.py +++ b/thoth/storages/ceph_cache.py @@ -18,6 +18,7 @@ """A base class for implementing caches based on Ceph.""" from datetime import datetime +from datetime import timedelta from datetime import timezone from .exceptions import CacheMiss @@ -45,12 +46,11 @@ def retrieve_document_ttl(self, document_id: str) -> float: self.retrieve_document_record(document_id) # Uses UTC time to be environment agnostic (no timezone) - time_lived = ( - datetime.now() - - self.ceph.retrieve_document_attr(object_key=document_id, attr="LastModified").replace(tzinfo=timezone.utc) - ).total_seconds() + time_lived = datetime.now(timezone.utc) - self.ceph.retrieve_document_attr( + object_key=document_id, attr="LastModified" + ).replace(tzinfo=timezone.utc) - if time_lived <= 14400.0: - return time_lived + if time_lived <= timedelta(hours=4): + return (timedelta(hours=4) - time_lived).total_seconds() return 0.0