Merge pull request #1460 from groue/dev/notify-changes

Explicit change notifications

GRDB/Core/Database+Schema.swift

@@ -57,6 +57,14 @@ extension Database {
             case let .attached(name): return "\(name).sqlite_master"
             }
         }
+
+        /// The name of the master sqlite table, without the schema name.
+        var unqualifiedMasterTableName: String { // swiftlint:disable:this inclusive_language
+            switch self {
+            case .main, .attached: return "sqlite_master"
+            case .temp: return "sqlite_temp_master"
+            }
+        }
     }
 
     /// The identifier of a database table or view.
@@ -658,9 +666,17 @@ extension Database {
     /// attached database.
     func canonicalTableName(_ tableName: String) throws -> String? {
         for schemaIdentifier in try schemaIdentifiers() {
+            // Regular tables
             if let result = try schema(schemaIdentifier).canonicalName(tableName, ofType: .table) {
                 return result
             }
+
+            // Master table (sqlite_master, sqlite_temp_master)
+            // swiftlint:disable:next inclusive_language
+            let masterTableName = schemaIdentifier.unqualifiedMasterTableName
+            if tableName.lowercased() == masterTableName.lowercased() {
+                return masterTableName
+            }
         }
         return nil
     }
@@ -1367,7 +1383,7 @@ struct SchemaObject: Hashable, FetchableRecord {
 
 /// All objects in a database schema (tables, views, indexes, triggers).
 struct SchemaInfo: Equatable {
-    private var objects: Set<SchemaObject>
+    let objects: Set<SchemaObject>
 
     /// Returns whether there exists a object of given type with this name
     /// (case-insensitive).

GRDB/Core/Database.swift

@@ -79,6 +79,7 @@ let SQLITE_TRANSIENT = unsafeBitCast(OpaquePointer(bitPattern: -1), to: sqlite3_
 /// - ``add(transactionObserver:extent:)``
 /// - ``remove(transactionObserver:)``
 /// - ``afterNextTransaction(onCommit:onRollback:)``
+/// - ``notifyChanges(in:)``
 /// - ``registerAccess(to:)``
 ///
 /// ### Collations
@@ -820,6 +821,65 @@ public final class Database: CustomStringConvertible, CustomDebugStringConvertib
         }
     }
 
+    /// Notifies that some changes were performed in the provided
+    /// database region.
+    ///
+    /// This method makes it possible to notify undetected changes, such as
+    /// changes performed by another process, changes performed by
+    /// direct calls to SQLite C functions, or changes to the
+    /// database schema.
+    /// See <doc:GRDB/TransactionObserver#Dealing-with-Undetected-Changes>
+    /// for a detailed list of undetected database modifications.
+    ///
+    /// It triggers active transaction observers (``TransactionObserver``).
+    /// In particular, ``ValueObservation`` that observe the input `region`
+    /// will fetch and notify a fresh value.
+    ///
+    /// For example:
+    ///
+    /// ```swift
+    /// try dbQueue.write { db in
+    ///     // Notify observers that some changes were performed in the database
+    ///     try db.notifyChanges(in: .fullDatabase)
+    ///
+    ///     // Notify observers that some changes were performed in the player table
+    ///     try db.notifyChanges(in: Player.all())
+    ///
+    ///     // Equivalent alternative
+    ///     try db.notifyChanges(in: Table("player"))
+    /// }
+    /// ```
+    ///
+    /// This method has no effect when called from a read-only
+    /// database access.
+    ///
+    /// > Caveat: Individual rowids in the input region are ignored.
+    /// > Notifying a change to a specific rowid is the same as notifying a
+    /// > change in the whole table:
+    /// >
+    /// > ```swift
+    /// > try dbQueue.write { db in
+    /// >     // Equivalent
+    /// >     try db.notifyChanges(in: Player.all())
+    /// >     try db.notifyChanges(in: Player.filter(id: 1))
+    /// > }
+    /// > ```
+    public func notifyChanges(in region: some DatabaseRegionConvertible) throws {
+        // Don't do anything when read-only, because read-only transactions
+        // are not notified. We don't want to notify transactions observers
+        // of changes, and have them wait for a commit notification that
+        // will never come.
+        if !isReadOnly, let observationBroker {
+            let eventKinds = try region
+                .databaseRegion(self)
+                // Use canonical table names for case insensitivity of the input.
+                .canonicalTables(self)
+                .impactfulEventKinds(self)
+
+            try observationBroker.notifyChanges(withEventsOfKind: eventKinds)
+        }
+    }
+
     /// Extends the `region` argument with the database region selected by all
     /// statements executed by the closure, and all regions explicitly tracked
     /// with the ``registerAccess(to:)`` method.

GRDB/Core/DatabaseRegion.swift

@@ -169,16 +169,16 @@ public struct DatabaseRegion {
         // the observed region, we optimize database observation.
         //
         // And by canonicalizing table names, we remove views, and help the
-        // `isModified` methods.
+        // `isModified` methods. (TODO: is this comment still accurate?
+        // Isn't it about providing TransactionObserver.observes() with
+        // real tables names, instead?)
         try ignoringInternalSQLiteTables().canonicalTables(db)
     }
 
     /// Returns a region only made of actual tables with their canonical names.
-    /// Canonical names help the `isModified` methods.
     ///
-    /// This method removes views (assuming no table exists with the same name
-    /// as a view).
-    private func canonicalTables(_ db: Database) throws -> DatabaseRegion {
+    /// This method removes views.
+    func canonicalTables(_ db: Database) throws -> DatabaseRegion {
         guard let tableRegions else { return .fullDatabase }
         var region = DatabaseRegion()
         for (table, tableRegion) in tableRegions {
@@ -233,6 +233,44 @@ extension DatabaseRegion {
         }
         return tableRegion.contains(rowID: event.rowID)
     }
+
+    /// Returns an array of all event kinds that can impact this region.
+    ///
+    /// - precondition: the region is canonical.
+    func impactfulEventKinds(_ db: Database) throws -> [DatabaseEventKind] {
+        if let tableRegions {
+            return try tableRegions.flatMap { (table, tableRegion) -> [DatabaseEventKind] in
+                let tableName = table.rawValue // canonical table name
+                let columnNames: Set<String>
+                if let columns = tableRegion.columns {
+                    columnNames = Set(columns.map(\.rawValue))
+                } else {
+                    columnNames = try Set(db.columns(in: tableName).map(\.name))
+                }
+
+                return [
+                    DatabaseEventKind.delete(tableName: tableName),
+                    DatabaseEventKind.insert(tableName: tableName),
+                    DatabaseEventKind.update(tableName: tableName, columnNames: columnNames),
+                ]
+            }
+        } else {
+            // full database
+            return try db.schemaIdentifiers().flatMap { schemaIdentifier in
+                let schema = try db.schema(schemaIdentifier)
+                return try schema.objects
+                    .filter { $0.type == .table }
+                    .flatMap { table in
+                        let columnNames = try Set(db.columns(in: table.name).map(\.name))
+                        return [
+                            DatabaseEventKind.delete(tableName: table.name),
+                            DatabaseEventKind.insert(tableName: table.name),
+                            DatabaseEventKind.update(tableName: table.name, columnNames: columnNames),
+                        ]
+                    }
+            }
+        }
+    }
 }
 
 extension DatabaseRegion: Equatable {

GRDB/Core/DatabaseRegionObservation.swift

@@ -161,6 +161,11 @@ private class DatabaseRegionObserver: TransactionObserver {
         region.isModified(byEventsOfKind: eventKind)
     }
 
+    func databaseDidChange() {
+        isChanged = true
+        stopObservingDatabaseChangesUntilNextTransaction()
+    }
+
     func databaseDidChange(with event: DatabaseEvent) {
         if region.isModified(by: event) {
             isChanged = true

GRDB/Core/TransactionObserver.swift

@@ -282,6 +282,20 @@ class DatabaseObservationBroker {
         }
     }
 
+    func notifyChanges(withEventsOfKind eventKinds: [DatabaseEventKind]) throws {
+        // Support for stopObservingDatabaseChangesUntilNextTransaction()
+        SchedulingWatchdog.current!.databaseObservationBroker = self
+        defer {
+            SchedulingWatchdog.current!.databaseObservationBroker = nil
+        }
+
+        for observation in transactionObservations where observation.isEnabled {
+            if eventKinds.contains(where: { observation.observes(eventsOfKind: $0) }) {
+                observation.databaseDidChange()
+            }
+        }
+    }
+
     // MARK: - Statement execution
 
     /// Returns true if there exists some transaction observer interested in
@@ -782,6 +796,13 @@ public protocol TransactionObserver: AnyObject {
     /// from being applied on the observed tables.
     func observes(eventsOfKind eventKind: DatabaseEventKind) -> Bool
 
+    /// Called when the database was modified in some unspecified way.
+    ///
+    /// This method allows a transaction observer to handle changes that are
+    /// not automatically detected. See <doc:GRDB/TransactionObserver#Dealing-with-Undetected-Changes>
+    /// and ``Database/notifyChanges(in:)`` for more information.
+    func databaseDidChange()
+
     /// Called when the database is changed by an insert, update, or
     /// delete event.
     ///
@@ -857,6 +878,9 @@ extension TransactionObserver {
     public func databaseWillChange(with event: DatabasePreUpdateEvent) { }
     #endif
 
+    /// The default implementation does nothing.
+    public func databaseDidChange() { }
+
     /// Prevents the observer from receiving further change notifications until
     /// the next transaction.
     ///
@@ -889,7 +913,7 @@ extension TransactionObserver {
         guard let broker = SchedulingWatchdog.current?.databaseObservationBroker else {
             fatalError("""
                 stopObservingDatabaseChangesUntilNextTransaction must be called \
-                from the databaseDidChange method
+                from the `databaseDidChange()` or `databaseDidChange(with:)` methods
                 """)
         }
         broker.disableUntilNextTransaction(transactionObserver: self)
@@ -942,6 +966,11 @@ final class TransactionObservation {
     }
     #endif
 
+    func databaseDidChange() {
+        guard isEnabled else { return }
+        observer?.databaseDidChange()
+    }
+
     func databaseDidChange(with event: DatabaseEvent) {
         guard isEnabled else { return }
         observer?.databaseDidChange(with: event)

GRDB/Documentation.docc/DatabaseSharing.md

@@ -228,9 +228,7 @@ In applications that use the background modes supported by iOS, post `resumeNoti
 
 <doc:DatabaseObservation> features are not able to detect database changes performed by other processes.
 
-Whenever you need to notify other processes that the database has been changed, you will have to use a cross-process notification mechanism such as [NSFileCoordinator] or [CFNotificationCenterGetDarwinNotifyCenter].
-
-You can trigger those notifications automatically with ``DatabaseRegionObservation``:
+Whenever you need to notify other processes that the database has been changed, you will have to use a cross-process notification mechanism such as [NSFileCoordinator] or [CFNotificationCenterGetDarwinNotifyCenter]. You can trigger those notifications automatically with ``DatabaseRegionObservation``:
 
 ```swift
 // Notify all changes made to the database
@@ -246,6 +244,8 @@ let observer = try observation.start(in: dbPool) { db in
 }
 ```
 
+The processes that observe the database can catch those notifications, and deal with the notified changes. See <doc:GRDB/TransactionObserver#Dealing-with-Undetected-Changes> for some related techniques.
+
 [NSFileCoordinator]: https://developer.apple.com/documentation/foundation/nsfilecoordinator
 [CFNotificationCenterGetDarwinNotifyCenter]: https://developer.apple.com/documentation/corefoundation/1542572-cfnotificationcentergetdarwinnot
 [WAL mode]: https://www.sqlite.org/wal.html

GRDB/Documentation.docc/Extension/DatabaseRegionObservation.md

@@ -9,6 +9,7 @@
 The only changes that are not notified are:
 
 - Changes performed by external database connections.
+- Changes performed by SQLite statements that are not compiled and executed by GRDB.
 - Changes to the database schema, changes to internal system tables such as `sqlite_master`.
 - Changes to [`WITHOUT ROWID`](https://www.sqlite.org/withoutrowid.html) tables.
 

GRDB/Documentation.docc/Extension/TransactionObserver.md

@@ -25,13 +25,7 @@ Database changes are notified to the ``databaseDidChange(with:)`` callback. This
 
 Transaction completions are notified to the ``databaseWillCommit()-7mksu``, ``databaseDidCommit(_:)`` and ``databaseDidRollback(_:)`` callbacks.
 
-The only changes and transactions that are not notified are:
-
-- Read-only transactions.
-- Changes and transactions performed by external database connections.
-- Changes to the database schema, changes to internal system tables such as `sqlite_master`.
-- Changes to [`WITHOUT ROWID`](https://www.sqlite.org/withoutrowid.html) tables.
-- The deletion of duplicate rows triggered by [`ON CONFLICT REPLACE`](https://www.sqlite.org/lang_conflict.html) clauses (this last exception might change in a future release of SQLite).
+> Important: Some changes and transactions are not automatically notified. See <doc:GRDB/TransactionObserver#Dealing-with-Undetected-Changes> below.
 
 Notified changes are not actually written to disk until the transaction commits, and the `databaseDidCommit` callback is called. On the other side, `databaseDidRollback` confirms their invalidation:
 
@@ -189,7 +183,7 @@ class PlayerObserver: TransactionObserver {
 }
 ```
 
-### Support for SQLite Pre-Update Hooks
+## Support for SQLite Pre-Update Hooks
 
 When SQLite is built with the `SQLITE_ENABLE_PREUPDATE_HOOK` option, `TransactionObserver` gets an extra callback which lets you observe individual column values in the rows modified by a transaction:
 
@@ -235,6 +229,45 @@ This extra API can be activated in two ways:
 
 2. Use a [custom SQLite build](http://github.com/groue/GRDB.swift/blob/master/Documentation/CustomSQLiteBuilds.md) and activate the `SQLITE_ENABLE_PREUPDATE_HOOK` compilation option.
 
+## Dealing with Undetected Changes
+
+Some changes and transactions are not automatically notified to transaction observers:
+
+- Read-only transactions.
+- Changes and transactions performed by external database connections.
+- Changes performed by SQLite statements that are not both compiled and executed through GRDB APIs.
+- Changes to the database schema, changes to internal system tables such as `sqlite_master`.
+- Changes to [`WITHOUT ROWID`](https://www.sqlite.org/withoutrowid.html) tables.
+- The deletion of duplicate rows triggered by [`ON CONFLICT REPLACE`](https://www.sqlite.org/lang_conflict.html) clauses (this last exception might change in a future release of SQLite).
+
+Undetected changes are notified to the ``databaseDidChange()-7olv7`` callback when you perform an explicit call to the ``Database/notifyChanges(in:)`` `Database` method.
+
+Event filtering described in <doc:GRDB/TransactionObserver#Filtering-Database-Events> still applies: the `databaseDidChange()` callback is not called for changes that are not observed. 
+
+For example:
+
+```swift
+try dbQueue.write { db in
+    // Notify observers that some changes were performed in the database
+    try db.notifyChanges(in: .fullDatabase)
+
+    // Notify observers that some changes were performed in the player table
+    try db.notifyChanges(in: Player.all())
+
+    // Equivalent alternative
+    try db.notifyChanges(in: Table("player"))
+}
+```
+
+To notify a change in the database schema, notify a change to the `sqlite_master` table:
+
+```swift
+try dbQueue.write { db in
+    // Notify all observers of the sqlite_master table
+    try db.notifyChanges(in: Table("sqlite_master"))
+}
+```
+
 ## Topics
 
 ### Filtering Database Changes
@@ -244,6 +277,7 @@ This extra API can be activated in two ways:
 
 ### Handling Database Changes
 
+- ``databaseDidChange()-7olv7``
 - ``databaseDidChange(with:)``
 - ``stopObservingDatabaseChangesUntilNextTransaction()``
 - ``DatabaseEvent``

GRDB/Documentation.docc/Extension/ValueObservation.md

@@ -8,7 +8,8 @@
 
 The only changes that are not notified are:
 
-- Changes performed by external database connections. See <doc:DatabaseSharing#How-to-perform-cross-process-database-observation> for more information.
+- Changes performed by external database connections.
+- Changes performed by SQLite statements that are not compiled and executed by GRDB.
 - Changes to the database schema, changes to internal system tables such as `sqlite_master`.
 - Changes to [`WITHOUT ROWID`](https://www.sqlite.org/withoutrowid.html) tables.
 

GRDB/ValueObservation/Observers/ValueConcurrentObserver.swift

@@ -710,6 +710,13 @@ extension ValueConcurrentObserver: TransactionObserver {
         }
     }
 
+    func databaseDidChange() {
+        // Database was modified!
+        observationState.isModified = true
+        // We can stop observing the current transaction
+        stopObservingDatabaseChangesUntilNextTransaction()
+    }
+
     func databaseDidChange(with event: DatabaseEvent) {
         if let region = observationState.region, region.isModified(by: event) {
             // Database was modified!