Bug fixing

- Explicitly store the last processed frame number in the database
- De-duplicate output for the watcher script

.gitignore

@@ -22,6 +22,7 @@ compile_commands.json
 *.kdev4
 
 .idea
+*.egg-info/
 
 # swap
 [._]*.s[a-w][a-z]

include/ocs/database.h

@@ -34,6 +34,8 @@ class database {
    std::int64_t get_starting_frame_number() const;
    bool is_frame_processed(std::int64_t frame_num) const;
 
+   void store_last_frame_number(std::int64_t frame_num) const;
+
 private:
    static void db_update(db::database &db, int from);
 
@@ -46,6 +48,7 @@ class database {
    statement_ptr_t add_text_entry_;
    statement_ptr_t get_starting_frame_number_;
    statement_ptr_t is_frame_number_present_;
+   statement_ptr_t store_last_frame_number_;
 
    mutable std::recursive_mutex database_mutex_{};
 };

src/database.cpp

@@ -15,6 +15,7 @@ using db_t = ocs::db::database;
 #define OCS_IDL_INCLUDE
 #include "db/updates/v0.inl"
 #include "db/updates/v1.inl"
+#include "db/updates/v2.inl"
 
 // Note: should always be last
 #include "db/updates/update.inl"
@@ -28,6 +29,7 @@ database::database(std::string db_path)
 }
 
 void database::store(const ocr::ocr_result &result) {
+   store_last_frame_number(result.frame_number);
    if (result.entries.empty()) {
       return;
    }
@@ -72,11 +74,7 @@ std::int64_t database::get_starting_frame_number() const {
    stmt->reset();
    stmt->evaluate();
 
-   if (stmt->is_null(0)) {
-      return 0;
-   } else {
-      return stmt->get_int64(0);
-   }
+   return stmt->get_int64(0) + 1;
 }
 
 bool database::is_frame_processed(std::int64_t frame_num) const {
@@ -90,10 +88,27 @@ bool database::is_frame_processed(std::int64_t frame_num) const {
    return stmt->get_int(0) != 0;
 }
 
+void database::store_last_frame_number(std::int64_t frame_num) const {
+   std::lock_guard lock{database_mutex_};
+
+   static std::int64_t max_frame_number = -1;
+   if (frame_num <= max_frame_number) {
+      return;
+   }
+
+   auto &stmt = store_last_frame_number_;
+
+   stmt->reset();
+   stmt->bind("pnum", frame_num);
+   stmt->evaluate();
+
+   max_frame_number = frame_num;
+}
+
 void database::prepare_statements() {
    // clang-format off
    get_starting_frame_number_ = std::make_unique<db::statement>("get_starting_frame_number");
-   get_starting_frame_number_->prepare(db_, R"sql(SELECT MAX(frame_num)+1 FROM ocr_entries;)sql", true);
+   get_starting_frame_number_->prepare(db_, R"sql(SELECT last_processed_frame FROM metadata;)sql", true);
 
    add_text_entry_ = std::make_unique<db::statement>("add_text_entry");
    add_text_entry_->prepare(db_,
@@ -105,5 +120,9 @@ void database::prepare_statements() {
    is_frame_number_present_->prepare(db_,
    R"sql(SELECT COUNT(*) FROM ocr_entries WHERE frame_num = :pnum;)sql", true);
 
+   store_last_frame_number_ = std::make_unique<db::statement>("store_last_frame_number");
+   store_last_frame_number_->prepare(db_,
+   R"sql(UPDATE metadata SET last_processed_frame=:pnum;)sql", true);
+
    // clang-format on
 }

src/db/updates/update.inl

@@ -6,7 +6,7 @@
 #error Internal use only
 #endif
 
-const int ocs::database::CURRENT_DB_VERSION = 2;
+const int ocs::database::CURRENT_DB_VERSION = 3;
 
 void ocs::database::db_update(db_t &db, int from) {
    switch (from) {
@@ -16,6 +16,9 @@ void ocs::database::db_update(db_t &db, int from) {
       case 1:
          return update_v1(db);
 
+      case 2:
+         return update_v2(db);
+
       default:
          throw std::runtime_error("Unknown database version: " + std::to_string(from));
    }

src/db/updates/v2.inl

@@ -0,0 +1,43 @@
+//
+// Created by Dennis Sitelew on 28.12.22.
+//
+
+#ifndef OCS_IDL_INCLUDE
+#error Internal use only
+#endif
+
+namespace {
+
+void update_v2(db_t &db) {
+   std::int64_t current_max = 0;
+
+   // Get the maximal frame number
+   auto max_stmt = std::make_unique<db::statement>("get_max_frame_num");
+   max_stmt->prepare(db, R"sql(SELECT MAX(frame_num) FROM ocr_entries;)sql", false);
+   max_stmt->reset();
+
+   auto code = max_stmt->evaluate();
+   if (code == SQLITE_ROW && !max_stmt->is_null(0)) {
+      current_max = max_stmt->get_int64(0);
+   }
+
+   // Add the frame number column
+   auto sql = R"sql(
+BEGIN TRANSACTION;
+
+ALTER TABLE metadata
+ADD COLUMN last_processed_frame INT DEFAULT(0);
+
+COMMIT;
+)sql";
+   ocs::db::statement::exec(db, sql);
+
+   // Update the frame number itself
+   std::string update = "UPDATE metadata SET last_processed_frame=";
+   update += std::to_string(current_max);
+   update += ';';
+
+   ocs::db::statement::exec(db, update.c_str());
+}
+
+} // namespace

tools/ocs-watcher/ocsw/runner.py

@@ -90,10 +90,16 @@ def _ocr_file(self, path) -> bool:
         ]
         proc = subprocess.Popen(args, creationflags=flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
+        last_output = ''
+
+        def get_optional_text(stream) -> str:
+            return stream.decode('utf-8') if stream is not None else ''
+
         def handle_outputs(timeout):
             outs, errs = proc.communicate(timeout=timeout)
-            if outs is not None and len(outs) != 0:
-                print(outs.decode('utf-8'), end='', sep='', flush=True)
+            out_text = get_optional_text(outs)
+            if len(out_text) != 0:
+                print(out_text, end='', sep='', flush=True)
 
             if proc.returncode != 0:
                 error = errs.decode("utf-8").strip()
@@ -102,11 +108,15 @@ def handle_outputs(timeout):
                 return False
 
         def run_for_output():
+            nonlocal last_output
             while proc.poll() is None:
                 try:
                     handle_outputs(timeout=self.OUTPUT_POLLING_INTERVAL)
                 except subprocess.TimeoutExpired as e:
-                    print(e.stdout.decode('utf-8'), end='', sep='', flush=True)
+                    text = get_optional_text(e.stdout)
+                    if len(text) != 0:
+                        print(text[len(last_output):], end='', sep='', flush=True)
+                    last_output = text
 
         try:
             print(f'Running OCR for {path}')