diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f4253c03..8d2305267 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,11 +9,17 @@ FROM roflcoopter/${ARCH}-ffmpeg:${FFMPEG_VERSION} as ffmpeg FROM roflcoopter/${ARCH}-wheels:${WHEELS_VERSION} as wheels # Build GPAC -FROM ubuntu:${UBUNTU_VERSION} AS gpac +FROM roflcoopter/${ARCH}-base:${BASE_VERSION} as gpac ENV \ DEBIAN_FRONTEND=noninteractive +RUN \ + if [ "$ARCH" = "armhf" ] || \ + [ "$ARCH" = "rpi3" ] || \ + [ "$ARCH" = "aarch64" ] || \ + [ "$ARCH" = "jetson-nano" ]; then echo "Crossbuilding!" && cross-build-start; fi + RUN \ apt-get update && apt-get install -y --no-install-recommends \ build-essential \ diff --git a/rootfs/etc/cont-init.d/40-set-env-vars b/rootfs/etc/cont-init.d/40-set-env-vars index 5ee39f623..3a1cb7cdb 100644 --- a/rootfs/etc/cont-init.d/40-set-env-vars +++ b/rootfs/etc/cont-init.d/40-set-env-vars @@ -49,7 +49,7 @@ export HOME=/home/abc printf "/home/abc" > /var/run/environment/HOME # Find latest version of postgresql -export PG_VERSION=$(pg_config --version | awk '{print $2}' | awk -F'.' '{print $1}') +export PG_VERSION=$(psql --version | awk '{print $3}' | awk -F'.' '{print $1}') export PG_BIN="/usr/lib/postgresql/$PG_VERSION/bin" printf "$PG_VERSION" > /var/run/environment/PG_VERSION printf "$PG_BIN" > /var/run/environment/PG_BIN diff --git a/tests/components/storage/test_tier_handler.py b/tests/components/storage/test_tier_handler.py index ec90f0e33..47759c5c4 100644 --- a/tests/components/storage/test_tier_handler.py +++ b/tests/components/storage/test_tier_handler.py @@ -6,9 +6,20 @@ import pytest from sqlalchemy import select -from viseron.components.storage.const import CONFIG_RECORDER +from viseron import Viseron +from viseron.components.storage import Storage +from viseron.components.storage.const import ( + COMPONENT as STORAGE_COMPONENT, + CONFIG_RECORDER, +) from viseron.components.storage.models import Recordings -from viseron.components.storage.tier_handler import SegmentsTierHandler, handle_file +from viseron.components.storage.tier_handler import ( + RecordingsTierHandler, + SegmentsTierHandler, + ThumbnailTierHandler, + find_next_tier_segments, + handle_file, +) from viseron.domains.camera.const import CONFIG_LOOKBACK from tests.common import BaseTestWithRecordings @@ -54,15 +65,24 @@ def test_handle_file_move( @dataclass -class MockQueryResult: +class MockRecordingsQueryResult: """Mock query result.""" - recording_id: int + recording_id: int | None file_id: int path: str tier_path: str +@dataclass +class MockFilesQueryResult: + """Mock query result.""" + + id: int + path: str + tier_path: str + + def _get_tier_config(events: bool, continuous: bool): """Get tier config for test.""" max_age_events = None @@ -139,12 +159,12 @@ def test__check_tier( "viseron.components.storage.tier_handler.handle_file" ): mock_get_recordings_to_move.return_value = [ - MockQueryResult(1, 1, "/tmp/test1.mp4", "/tmp/"), - MockQueryResult(1, 2, "/tmp/test2.mp4", "/tmp/"), + MockRecordingsQueryResult(1, 1, "/tmp/test1.mp4", "/tmp/"), + MockRecordingsQueryResult(1, 2, "/tmp/test2.mp4", "/tmp/"), ] mock_get_files_to_move.return_value = [ - MockQueryResult(1, 1, "/tmp/test1.mp4", "/tmp/"), - MockQueryResult(1, 2, "/tmp/test2.mp4", "/tmp/"), + MockFilesQueryResult(1, "/tmp/test1.mp4", "/tmp/"), + MockFilesQueryResult(2, "/tmp/test2.mp4", "/tmp/"), ] tier_handler._check_tier( # pylint: disable=protected-access self._get_db_session @@ -167,3 +187,203 @@ def test__check_tier( recordings = session.execute(stmt).scalars().fetchall() assert len(recordings) == recordings_amount assert recordings[0].id == first_recording_id + + @pytest.mark.parametrize( + "tiers_config, recording_id, force_delete, next_tier_index, " + "move_thumbnail_called, move_event_clip_called", + [ + ( # Test that check_tier deletes the file if next tier is None + [_get_tier_config(events=True, continuous=True)], + 1, + True, + None, + True, + True, + ), + # Test that check_tier deletes the file if its not part of a recording and + # next tier does not store continuous + ( + [ + _get_tier_config(events=True, continuous=True), + _get_tier_config(events=True, continuous=False), + ], + None, + True, + None, + False, + False, + ), + # Test that check_tier moves the file if its part of a recording and + # the next tier stores events + ( + [ + _get_tier_config(events=True, continuous=True), + _get_tier_config(events=True, continuous=False), + ], + 1, + False, + 1, + True, + True, + ), + # Test that check_tier moves the file to the correct tier when the next tier + # does not store events but the next next tier does + ( + [ + _get_tier_config(events=True, continuous=True), + _get_tier_config(events=False, continuous=False), + _get_tier_config(events=True, continuous=False), + _get_tier_config(events=False, continuous=True), + ], + 1, + False, + 2, + True, + True, + ), + ], + ) + def test__check_tier_next_tier4( + self, + vis: Viseron, + tiers_config, + recording_id: int, + force_delete: bool, + next_tier_index: int | None, + move_thumbnail_called: bool, + move_event_clip_called: bool, + ): + """Test that check_tier finds the correct tier.""" + mock_camera = Mock() + mock_camera.identifier = "test" + mock_camera.config = {CONFIG_RECORDER: {CONFIG_LOOKBACK: 5}} + + tier_handlers = [] + for i, tier_config in enumerate(tiers_config): + tier_handler = SegmentsTierHandler( + vis, + mock_camera, + i, + "recorder", + "segments", + tier_config, + None, + ) + tier_handlers.append(tier_handler) + recordings_tier_handler = MagicMock(spec=RecordingsTierHandler) + thumbnail_tier_handler = MagicMock(spec=ThumbnailTierHandler) + vis.data[STORAGE_COMPONENT].camera_tier_handlers = { + "test": { + "recorder": [ + { + "segments": tier_handler, + "thumbnails": thumbnail_tier_handler, + "recordings": recordings_tier_handler, + } + for tier_handler in tier_handlers + ] + } + } + + with patch( + "viseron.components.storage.tier_handler.get_recordings_to_move" + ) as mock_get_recordings_to_move, patch( + "viseron.components.storage.tier_handler.get_files_to_move" + ) as mock_get_files_to_move, patch( + "viseron.components.storage.tier_handler.handle_file" + ) as mock_handle_file: + mock_get_recordings_to_move.return_value = [ + MockRecordingsQueryResult(recording_id, 1, "/tmp/test1.mp4", "/tmp/"), + ] + mock_get_files_to_move.return_value = [ + MockFilesQueryResult(1, "/tmp/test1.mp4", "/tmp/"), + ] + tier_handlers[0]._check_tier( # pylint: disable=protected-access + self._get_db_session + ) + mock_handle_file.assert_called_once_with( + self._get_db_session, + tier_handlers[0]._storage, # pylint: disable=protected-access + tier_handlers[0]._camera.identifier, # pylint: disable=protected-access + tier_handlers[0].tier, + tier_handlers[next_tier_index].tier if next_tier_index else None, + "/tmp/test1.mp4", + "/tmp/", + tier_handlers[0]._logger, # pylint: disable=protected-access + force_delete=force_delete, + ) + if move_thumbnail_called: + thumbnail_tier_handler.move_thumbnail.assert_called_once_with( + 1, tier_handlers[next_tier_index].tier if next_tier_index else None + ) + if move_event_clip_called: + recordings_tier_handler.move_event_clip.assert_called_once_with( + 1, tier_handlers[next_tier_index].tier if next_tier_index else None + ) + + +def test_find_next_tier_segments(vis: Viseron): + """Test find_next_tier_segments.""" + mock_storage = Mock(spec=Storage) + mock_camera = Mock() + mock_camera.identifier = "test_camera" + mock_camera.config = {CONFIG_RECORDER: {CONFIG_LOOKBACK: 5}} + + tier_handler_0 = SegmentsTierHandler( + vis, + mock_camera, + 0, + "recorder", + "segments", + _get_tier_config(events=True, continuous=True), + None, + ) + tier_handler_1 = SegmentsTierHandler( + vis, + mock_camera, + 1, + "recorder", + "segments", + _get_tier_config(events=False, continuous=False), + None, + ) + tier_handler_2 = SegmentsTierHandler( + vis, + mock_camera, + 2, + "recorder", + "segments", + _get_tier_config(events=True, continuous=False), + None, + ) + + tier_handler_3 = SegmentsTierHandler( + vis, + mock_camera, + 3, + "recorder", + "segments", + _get_tier_config(events=False, continuous=True), + None, + ) + + mock_camera.identifier = "test_camera" + mock_storage.camera_tier_handlers = { + "test_camera": { + "recorder": [ + {"segments": tier_handler_0}, + {"segments": tier_handler_1}, + {"segments": tier_handler_2}, + {"segments": tier_handler_3}, + ] + } + } + + result = find_next_tier_segments(mock_storage, 0, mock_camera, "events") + assert result == tier_handler_2 + + result = find_next_tier_segments(mock_storage, 0, mock_camera, "continuous") + assert result == tier_handler_3 + + result = find_next_tier_segments(mock_storage, 2, mock_camera, "events") + assert result is None diff --git a/tests/components/webserver/common.py b/tests/components/webserver/common.py index 205bdf681..1df4b5852 100644 --- a/tests/components/webserver/common.py +++ b/tests/components/webserver/common.py @@ -81,8 +81,8 @@ def setUp(self) -> None: def tearDown(self) -> None: """Tear down the test.""" + super().tearDown() self.vis.shutdown() - return super().tearDown() def get_app(self): """Get the application. diff --git a/viseron/__init__.py b/viseron/__init__.py index ff64d9437..1133166ce 100644 --- a/viseron/__init__.py +++ b/viseron/__init__.py @@ -536,7 +536,6 @@ def join( | multiprocessing.process.BaseProcess, ) -> None: thread_or_process.join(timeout=10) - time.sleep(0.5) # Wait for process to exit properly if thread_or_process.is_alive(): LOGGER.error(f"{thread_or_process.name} did not exit in time") if isinstance(thread_or_process, multiprocessing.Process): diff --git a/viseron/components/storage/tier_handler.py b/viseron/components/storage/tier_handler.py index 9ce4ffccf..97ba28848 100644 --- a/viseron/components/storage/tier_handler.py +++ b/viseron/components/storage/tier_handler.py @@ -8,7 +8,7 @@ from datetime import datetime, timedelta from queue import Queue from threading import Lock, Timer -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal from sqlalchemy import Result, delete, insert, select, update from sqlalchemy.exc import IntegrityError @@ -132,6 +132,11 @@ def first_tier(self) -> bool: """Return if first tier.""" return self._tier_id == 0 + @property + def tier_id(self) -> int: + """Return tier id.""" + return self._tier_id + def add_file_handler(self, path: str, pattern: str): """Add file handler to webserver.""" self._logger.debug(f"Adding handler for /files{pattern}") @@ -364,87 +369,142 @@ def initialize(self) -> None: self._events_min_age, ] + self._events_enabled = any(self._events_params) + self._continuous_enabled = any(self._continuous_params) + self.add_file_handler(self._path, rf"{self._path}/(.*.m4s$)") self.add_file_handler(self._path, rf"{self._path}/(.*.mp4$)") + @property + def events_enabled(self) -> bool: + """Return if events are enabled.""" + return self._events_enabled + + @property + def continuous_enabled(self) -> bool: + """Return if continuous is enabled.""" + return self._continuous_enabled + + def _get_events_file_ids(self, session: Session) -> Result[Any] | list: + if self._events_enabled: + return get_recordings_to_move( + session, + self._tier_id, + self._camera.identifier, + self._camera.recorder.lookback, + self._events_max_bytes, + self._events_min_age, + self._events_min_bytes, + self._events_max_age, + ) + return [] + + def _get_continuous_file_ids(self, session: Session) -> Result[Any] | list: + if self._continuous_enabled: + return get_files_to_move( + session, + self._category, + self._subcategory, + self._tier_id, + self._camera.identifier, + self._continuous_max_bytes, + self._continuous_min_age, + self._continuous_min_bytes, + self._continuous_max_age, + ) + return [] + def _check_tier(self, get_session: Callable[[], Session]) -> None: - events_enabled = False - continuous_enabled = False - events_file_ids: Result[Any] | list = [] - continuous_file_ids: Result[Any] | list = [] with get_session() as session: - if any(self._events_params): - events_enabled = True - events_file_ids = get_recordings_to_move( - session, - self._tier_id, - self._camera.identifier, - self._camera.recorder.lookback, - self._events_max_bytes, - self._events_min_age, - self._events_min_bytes, - self._events_max_age, - ) - - if any(self._continuous_params): - continuous_enabled = True - continuous_file_ids = get_files_to_move( - session, - self._category, - self._subcategory, - self._tier_id, - self._camera.identifier, - self._continuous_max_bytes, - self._continuous_min_age, - self._continuous_min_bytes, - self._continuous_max_age, - ) + # Convert to list since we iterate over it twice and the Result closes + # after the first loop + events_file_ids = list(self._get_events_file_ids(session)) + continuous_file_ids = self._get_continuous_file_ids(session) - events_file_ids = list(events_file_ids) # A file can be in multiple recordings, so we need to keep track of which # files we have already processed using processed_paths processed_paths = [] - if events_enabled and not continuous_enabled: + events_next_tier = None + continuous_next_tier = None + if self._events_enabled and not self._continuous_enabled: + events_next_tier = find_next_tier_segments( + self._storage, self._tier_id, self._camera, "events" + ) for file in events_file_ids: if file.path in processed_paths: continue + force_delete = bool(file.recording_id is None) handle_file( get_session, self._storage, self._camera.identifier, self._tier, - self._next_tier, + events_next_tier.tier if events_next_tier else None, file.path, file.tier_path, self._logger, + force_delete, ) processed_paths.append(file.path) - elif continuous_enabled and not events_enabled: + elif self._continuous_enabled and not self._events_enabled: + continuous_next_tier = find_next_tier_segments( + self._storage, self._tier_id, self._camera, "continuous" + ) for file in continuous_file_ids: handle_file( get_session, self._storage, self._camera.identifier, self._tier, - self._next_tier, + continuous_next_tier.tier if continuous_next_tier else None, file.path, file.tier_path, self._logger, ) else: overlap = files_to_move_overlap(events_file_ids, continuous_file_ids) + events_next_tier = find_next_tier_segments( + self._storage, self._tier_id, self._camera, "events" + ) + continuous_next_tier = find_next_tier_segments( + self._storage, self._tier_id, self._camera, "continuous" + ) for file in overlap: if file.path in processed_paths: continue + + force_delete = False + next_tier = None + # If the file is not part of a recording, and no succeeding tiers + # store continuous recordings we can delete the file + if file.recording_id is None and continuous_next_tier is None: + force_delete = True + # If no succeeding tier stores either events or continuous + # recordings, we can delete the file + elif events_next_tier is None and continuous_next_tier is None: + force_delete = True + elif events_next_tier and continuous_next_tier is None: + next_tier = events_next_tier + elif continuous_next_tier and events_next_tier is None: + next_tier = continuous_next_tier + elif events_next_tier and continuous_next_tier: + # Find the lowest tier_id for the two next tiers + next_tier = ( + events_next_tier + if events_next_tier.tier_id < continuous_next_tier.tier_id + else continuous_next_tier + ) + handle_file( get_session, self._storage, self._camera.identifier, self._tier, - self._next_tier, + next_tier.tier if next_tier else None, file.path, file.tier_path, self._logger, + force_delete=force_delete, ) processed_paths.append(file.path) @@ -467,7 +527,10 @@ def _check_tier(self, get_session: Callable[[], Session]) -> None: self._category ][self._tier_id]["thumbnails"] ) - thumbnail_tier_handler.move_thumbnail(recording_id) + thumbnail_tier_handler.move_thumbnail( + recording_id, + events_next_tier.tier if events_next_tier else None, + ) # Signal to the recordings tier that the recording has been moved if recording_ids: @@ -480,10 +543,13 @@ def _check_tier(self, get_session: Callable[[], Session]) -> None: self._category ][self._tier_id]["recordings"] ) - recordings_tier_handler.move_event_clip(recording_id) + recordings_tier_handler.move_event_clip( + recording_id, + events_next_tier.tier if events_next_tier else None, + ) # Delete recordings from Recordings table if this is the last tier - if recording_ids and self._next_tier is None: + if recording_ids and events_next_tier is None: self._logger.debug("Deleting recordings: %s", recording_ids) with get_session() as _session: stmt = delete(Recordings).where(Recordings.id.in_(recording_ids)) @@ -542,7 +608,9 @@ def _on_created(self, event: FileCreatedEvent) -> None: ) super()._on_created(event) - def move_thumbnail(self, recording_id: int) -> None: + def move_thumbnail( + self, recording_id: int, next_tier: dict[str, Any] | None + ) -> None: """Move thumbnail to next tier.""" with self._storage.get_session() as session: sel = select(Recordings).where(Recordings.id == recording_id) @@ -552,7 +620,7 @@ def move_thumbnail(self, recording_id: int) -> None: self._storage, self._camera.identifier, self._tier, - self._next_tier, + next_tier, recording.thumbnail_path, self._tier[CONFIG_PATH], self._logger, @@ -604,7 +672,9 @@ def _on_created(self, event: FileCreatedEvent) -> None: self._update_clip_path(event) super()._on_created(event) - def move_event_clip(self, recording_id: int) -> None: + def move_event_clip( + self, recording_id: int, next_tier: dict[str, Any] | None + ) -> None: """Move event clip to next tier.""" with self._storage.get_session() as session: sel = ( @@ -621,7 +691,7 @@ def move_event_clip(self, recording_id: int) -> None: self._storage, self._camera.identifier, self._tier, - self._next_tier, + next_tier, recording.clip_path, self._tier[CONFIG_PATH], self._logger, @@ -629,6 +699,27 @@ def move_event_clip(self, recording_id: int) -> None: session.commit() +def find_next_tier_segments( + storage: Storage, + tier_id: int, + camera: AbstractCamera, + file_type: Literal["events", "continuous"], +) -> SegmentsTierHandler | None: + """Find the next tier for segments.""" + next_tier = None + for tier in storage.camera_tier_handlers[camera.identifier]["recorder"][ + tier_id + 1 : + ]: + segments_tier_handler: SegmentsTierHandler = tier["segments"] + if segments_tier_handler.events_enabled and file_type == "events": + next_tier = segments_tier_handler + break + if segments_tier_handler.continuous_enabled and file_type == "continuous": + next_tier = segments_tier_handler + break + return next_tier + + def handle_file( get_session: Callable[..., Session], storage: Storage, @@ -638,13 +729,14 @@ def handle_file( path: str, tier_path: str, logger: logging.Logger, + force_delete: bool = False, ) -> None: """Move file if there is a succeeding tier, else delete the file.""" if path in storage.camera_requested_files_count[camera_identifier].filenames: logger.debug("File %s is recently requested, skipping", path) return - if next_tier is None: + if force_delete or next_tier is None: delete_file(get_session, path, logger) else: new_path = path.replace(tier_path, next_tier[CONFIG_PATH], 1) diff --git a/viseron/components/webserver/__init__.py b/viseron/components/webserver/__init__.py index 3179d7b30..e3ce31fbc 100644 --- a/viseron/components/webserver/__init__.py +++ b/viseron/components/webserver/__init__.py @@ -281,7 +281,8 @@ def register_websocket_command(self, handler) -> None: def run(self) -> None: """Start ioloop.""" self._ioloop.start() - self._ioloop.close() + self._ioloop.close(True) + LOGGER.debug("IOLoop closed") def stop(self) -> None: """Stop ioloop.""" @@ -309,4 +310,4 @@ def stop(self) -> None: self._httpserver.stop() LOGGER.debug("Stopping IOloop") - self._ioloop.stop() + self._ioloop.add_callback(self._ioloop.stop)