Export history 1311 (#1338)

* Log missing files to export database, #1311

* Working on diff

* Working on diff

* Working on my own dictdiff

* Restored from main

* Added tolerance

* Added docstring

* Don't add missing

* Added action

* Fixed test failure

* Added get_/set_history

* Added --history

* Fixed dict diff

* Added --repair

* Added --check

* Added tests for --check, --repair

* Added test for --history

* Refactored test

* Updated --migrate-photos-library to fallback to filesize if no fingerprint

* Removed retry on _open_db

osxphotos/cli/exportdb.py

@@ -2,13 +2,14 @@
 
 import json
 import pathlib
+import re
 import sys
 from textwrap import dedent
 
 import click
 from rich import print
 
-from osxphotos._constants import OSXPHOTOS_EXPORT_DB
+from osxphotos._constants import OSXPHOTOS_EXPORT_DB, UUID_PATTERN
 from osxphotos._version import __version__
 from osxphotos.export_db import (
     MAX_EXPORT_RESULTS_DATA_ROWS,
@@ -28,6 +29,7 @@
     export_db_update_signatures,
     export_db_vacuum,
 )
+from osxphotos.sqlite_utils import sqlite_check_integrity, sqlite_repair_db
 from osxphotos.utils import pluralize
 
 from .cli_params import THEME_OPTION, TIMESTAMP_OPTION, VERBOSE_OPTION
@@ -48,6 +50,9 @@
 @click.command(name="exportdb")
 @click.option("--version", is_flag=True, help="Print export database version and exit.")
 @click.option("--vacuum", is_flag=True, help="Run VACUUM to defragment the database.")
+@click.option(
+    "--create", metavar="VERSION", help="Create a new export database with VERSION."
+)
 @click.option(
     "--check-signatures",
     is_flag=True,
@@ -101,6 +106,12 @@
     nargs=1,
     help="Print information about UUID contained in the database.",
 )
+@click.option(
+    "--history",
+    metavar="FILE_PATH_OR_UUID",
+    nargs=1,
+    help="Print history of FILE_PATH_OR_UUID contained in the database.",
+)
 @click.option(
     "--delete-uuid",
     metavar="UUID",
@@ -135,9 +146,17 @@
     type=(TemplateString(), click.IntRange(-(MAX_EXPORT_RESULTS_DATA_ROWS - 1), 0)),
 )
 @click.option(
-    "--migrate",
+    "--upgrade",
     is_flag=True,
-    help="Migrate (if needed) export database to current version.",
+    help="Upgrade (if needed) export database to current version.",
+)
+@click.option("--check", is_flag=True, help="Check export database for errors.")
+@click.option(
+    "--repair",
+    is_flag=True,
+    help="Repair export database. "
+    "This may be useful if the export database is corrupted and osxphotos reports "
+    "'database disk image is malformed' errors. ",
 )
 @click.option(
     "--sql",
@@ -175,25 +194,29 @@
 @click.argument("export_db", metavar="EXPORT_DATABASE", type=click.Path(exists=True))
 def exportdb(
     append,
+    check,
     check_signatures,
+    create,
     dry_run,
     export_db,
     export_dir,
     info,
     errors,
     last_errors,
     last_run,
-    migrate,
     migrate_photos_library,
+    repair,
     report,
     save_config,
     sql,
     theme,
     timestamp,
     touch_file,
     update_signatures,
+    upgrade,
     uuid_files,
     uuid_info,
+    history,
     delete_uuid,
     delete_file,
     vacuum,
@@ -215,7 +238,7 @@ def exportdb(
     if export_db.is_dir():
         # assume it's the export folder
         export_db = export_db / OSXPHOTOS_EXPORT_DB
-        if not export_db.is_file():
+        if not export_db.is_file() and not create:
             rich_echo_error(
                 f"[error]Error: {OSXPHOTOS_EXPORT_DB} missing from {export_db.parent}[/error]"
             )
@@ -226,10 +249,13 @@ def exportdb(
     sub_commands = [
         bool(cmd)
         for cmd in [
+            check,
             check_signatures,
+            create,
             info,
             last_run,
-            migrate,
+            upgrade,
+            repair,
             report,
             save_config,
             sql,
@@ -263,6 +289,49 @@ def exportdb(
             )
         sys.exit(0)
 
+    if create:
+        if pathlib.Path(export_db).exists():
+            rich_echo_error(
+                f"[error]Error: export database {export_db} already exists[/error]"
+            )
+            sys.exit(1)
+
+        if not "4.3" <= create <= OSXPHOTOS_EXPORTDB_VERSION:
+            rich_echo_error(
+                f"[error]Error: invalid version number {create}: must be between >= 4.3, <= {OSXPHOTOS_EXPORTDB_VERSION}[/]"
+            )
+            sys.exit(1)
+
+        try:
+            ExportDB(export_db, export_dir, create)
+        except Exception as e:
+            rich_echo_error(f"[error]Error: {e}[/error]")
+            sys.exit(1)
+        else:
+            rich_echo(f"Created export database [filepath]{export_db}[/]")
+            sys.exit(0)
+
+    if check:
+        errors = sqlite_check_integrity(export_db)
+        if not errors:
+            rich_echo(f"Ok: [filepath]{export_db}[/]")
+            sys.exit(0)
+        else:
+            rich_echo_error(f"[error]Errors: [filepath]{export_db}[/][/error]")
+            for error in errors:
+                rich_echo_error(error)
+            sys.exit(1)
+
+    if repair:
+        try:
+            sqlite_repair_db(export_db)
+        except Exception as e:
+            rich_echo_error(f"[error]Error: {e}[/error]")
+            sys.exit(1)
+        else:
+            rich_echo(f"Ok: [filepath]{export_db}[/]")
+            sys.exit(0)
+
     if vacuum:
         try:
             start_size = pathlib.Path(export_db).stat().st_size
@@ -423,6 +492,24 @@ def exportdb(
                 )
             sys.exit(0)
 
+    if history:
+        # get history for a file or uuid
+        exportdb = ExportDB(export_db, export_dir)
+        if re.match(UUID_PATTERN, history):
+            kwargs = {"uuid": history}
+        else:
+            kwargs = {"filepath": history}
+        try:
+            history_list = exportdb.get_history(**kwargs)
+        except Exception as e:
+            rich_echo_error(f"[error]Error: {e}[/error]")
+            sys.exit(1)
+
+        for item in history_list:
+            rich_echo(f"[date]{item[0]}[/], [filepath]{item[1]}[/], [uuid]{item[2]}[/]")
+
+        sys.exit(0)
+
     if delete_uuid:
         # delete a uuid from the export database
         exportdb = ExportDB(export_db, export_dir)
@@ -465,11 +552,11 @@ def exportdb(
         rich_echo(f"Wrote report to [filepath]{report_filename}[/]")
         sys.exit(0)
 
-    if migrate:
+    if upgrade:
         exportdb = ExportDB(export_db, export_dir)
         if upgraded := exportdb.was_upgraded:
             rich_echo(
-                f"Migrated export database [filepath]{export_db}[/] from version [num]{upgraded[0]}[/] to [num]{upgraded[1]}[/]"
+                f"Upgraded export database [filepath]{export_db}[/] from version [num]{upgraded[0]}[/] to [num]{upgraded[1]}[/]"
             )
         else:
             rich_echo(

osxphotos/dictdiff.py

@@ -0,0 +1,96 @@
+"""Perform diffs of dictionaries; this is optimized for osxphotos and is not a general purpose diff tool"""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+EPSILON = sys.float_info.epsilon
+
+
+def _compare(a: Any, b: Any, tolerance: float):
+    """Compare two values, return True if equal, False if not"""
+    if tolerance is None:
+        return a == b
+    elif isinstance(a, (int, float)) and isinstance(b, (int, float)):
+        return abs(a - b) <= tolerance
+    else:
+        return a == b
+
+
+def dictdiff(
+    d1: dict[Any, Any],
+    d2: dict[Any, Any],
+    tolerance: float = EPSILON,
+) -> list[list[Any, str, tuple[Any, ...]]]:
+    """Perform recursive diff of two dictionaries
+
+    Args:
+        d1: first dictionary
+        d2: second dictionary
+        tolerance: tolerance for comparing floats
+
+    Returns:
+        list of differences in the form of a list of tuples of the form:
+        [path, change_type, (old_value, new_value)]
+        where:
+            path: path to the key in the dictionary
+            change_type: one of "added", "removed", "changed"
+            old_value: old value of the key
+            new_value: new value of the key
+    """
+    return _dictdiff(d1, d2, tolerance)
+
+
+def _dictdiff(
+    d1: dict[Any, Any],
+    d2: dict[Any, Any],
+    tolerance: float = EPSILON,
+    path: str = "",
+) -> list[list[Any, str, tuple[Any, ...]]]:
+    """Perform recursive diff of two dictionaries
+
+    Args:
+        d1: first dictionary
+        d2: second dictionary
+        tolerance: tolerance for comparing floats
+        path: path to current key in dictionary (used for recursion)
+
+    Returns:
+        list of differences in the form of a list of tuples of the form:
+        [path, change_type, (old_value, new_value)]
+        where:
+            path: path to the key in the dictionary
+            change_type: one of "added", "removed", "changed"
+            old_value: old value of the key
+            new_value: new value of the key
+    """
+    diffs = []
+    for k in d1.keys():
+        new_path = f"{path}[{k}]" if path else k
+        if k not in d2:
+            diffs.append([new_path, "removed", (d1[k],)])
+        elif isinstance(d1[k], dict) and isinstance(d2[k], dict):
+            diffs.extend(_dictdiff(d1[k], d2[k], tolerance, new_path))
+        elif isinstance(d1[k], (list, set, tuple)) and isinstance(
+            d2[k], (list, set, tuple)
+        ):
+            added = []
+            removed = []
+            try:
+                added = set(d2[k]) - set(d1[k])
+                removed = set(d1[k]) - set(d2[k])
+            except TypeError:
+                # can't compare sets of unhashable types
+                if d1[k] != d2[k]:
+                    diffs.append([new_path, "changed", (d1[k], d2[k])])
+            if added:
+                diffs.append([new_path, "added", list(added)])
+            if removed:
+                diffs.append([new_path, "removed", list(removed)])
+        elif not _compare(d1[k], d2[k], tolerance):
+            diffs.append([new_path, "changed", (d1[k], d2[k])])
+    for k in set(d2.keys()) - set(d1.keys()):
+        new_path = f"{path}[{k}]" if path else k
+        diffs.append([new_path, "added", (d2[k],)])
+    return diffs