chore: unify desktop and mobile databases (#764)

* chore: unify desktop and mobile databases

- migrate `package:sqflite_flutter` to `sqlcipher_flutter_libs`
- use FFI for all SQLite operations
- use `SQfLiteEncryptionHelper` for database encryption
- enforce encryption for new SQLite datbase implementation
- migrate existing SQLite databases
 - encrypt unencrypted ones
 - migrate database locations to unified approach
- drop dependency on sqlite

Signed-off-by: The one with the braid <[email protected]>

* chore: add sqlcipher to macos CI

Signed-off-by: The one with the braid <[email protected]>

---------

Signed-off-by: The one with the braid <[email protected]>

.github/workflows/integrate.yaml

@@ -67,7 +67,7 @@ jobs:
           flutter-version: ${{ env.FLUTTER_VERSION }}
           cache: true
       - name: Install dependencies
-        run: sudo apt-get update && sudo apt-get install curl clang cmake ninja-build pkg-config libgtk-3-dev libblkid-dev liblzma-dev libjsoncpp-dev cmake-data libsecret-1-dev libsecret-1-0 librhash0 -y
+        run: sudo apt-get update && sudo apt-get install curl clang cmake ninja-build pkg-config libgtk-3-dev libblkid-dev liblzma-dev libjsoncpp-dev cmake-data libsecret-1-dev libsecret-1-0 librhash0 libssl-dev -y
       - run: flutter pub get
       - run: flutter build linux --target-platform linux-x64
 
@@ -84,5 +84,6 @@ jobs:
         uses: maxim-lobanov/[email protected]
         with:
           xcode-version: latest
+      - run: brew install sqlcipher
       - run: flutter pub get
       - run: flutter build ios --no-codesign

.github/workflows/release.yaml

@@ -113,7 +113,7 @@ jobs:
           flutter-version: ${{ env.FLUTTER_VERSION }}
           cache: true
       - name: Install dependencies
-        run: sudo apt-get update && sudo apt-get install curl clang cmake ninja-build pkg-config libgtk-3-dev libblkid-dev liblzma-dev libjsoncpp-dev cmake-data libsecret-1-dev libsecret-1-0 librhash0 -y
+        run: sudo apt-get update && sudo apt-get install curl clang cmake ninja-build pkg-config libgtk-3-dev libblkid-dev liblzma-dev libjsoncpp-dev cmake-data libsecret-1-dev libsecret-1-0 librhash0 libssl-dev -y
       - run: flutter pub get
       - run: flutter build linux --release --target-platform linux-x64
       - name: Create archive

ios/Podfile

@@ -38,6 +38,12 @@ post_install do |installer|
   installer.pods_project.targets.each do |target|
     flutter_additional_ios_build_settings(target)
     target.build_configurations.each do |config|
+      # ensure all dependencies are using SQLCipher instead of SQLite
+      xcconfig_path = config.base_configuration_reference.real_path
+      xcconfig = File.read(xcconfig_path)
+      new_xcconfig = xcconfig.sub(' -l"sqlite3"', '')
+      File.open(xcconfig_path, "w") { |file| file << new_xcconfig }
+
       config.build_settings['ENABLE_BITCODE'] = 'NO'
 
       # see https://github.com/flutter-webrtc/flutter-webrtc/issues/1054

lib/utils/client_manager.dart

@@ -18,7 +18,7 @@ import 'package:fluffychat/utils/custom_image_resizer.dart';
 import 'package:fluffychat/utils/init_with_restore.dart';
 import 'package:fluffychat/utils/matrix_sdk_extensions/flutter_hive_collections_database.dart';
 import 'package:fluffychat/utils/platform_infos.dart';
-import 'matrix_sdk_extensions/flutter_matrix_sdk_database_builder.dart';
+import 'matrix_sdk_extensions/flutter_matrix_dart_sdk_database/builder.dart';
 
 abstract class ClientManager {
   static const String clientNamespace = 'im.fluffychat.store.clients';

lib/utils/matrix_sdk_extensions/flutter_matrix_dart_sdk_database/builder.dart

@@ -0,0 +1,125 @@
+import 'dart:io';
+
+import 'package:flutter/foundation.dart';
+
+import 'package:flutter_gen/gen_l10n/l10n.dart';
+import 'package:matrix/matrix.dart';
+import 'package:path/path.dart';
+import 'package:path_provider/path_provider.dart';
+import 'package:sqflite_common_ffi/sqflite_ffi.dart';
+import 'package:universal_html/html.dart' as html;
+
+import 'package:fluffychat/config/app_config.dart';
+import 'package:fluffychat/utils/client_manager.dart';
+import 'package:fluffychat/utils/matrix_sdk_extensions/flutter_hive_collections_database.dart';
+import 'package:fluffychat/utils/platform_infos.dart';
+import 'cipher.dart';
+
+import 'sqlcipher_stub.dart'
+    if (dart.library.io) 'package:sqlcipher_flutter_libs/sqlcipher_flutter_libs.dart';
+
+Future<DatabaseApi> flutterMatrixSdkDatabaseBuilder(Client client) async {
+  MatrixSdkDatabase? database;
+  try {
+    database = await _constructDatabase(client);
+    await database.open();
+    return database;
+  } catch (e) {
+    // Try to delete database so that it can created again on next init:
+    database?.delete().catchError(
+          (e, s) => Logs().w(
+            'Unable to delete database, after failed construction',
+            e,
+            s,
+          ),
+        );
+
+    // Send error notification:
+    final l10n = lookupL10n(PlatformDispatcher.instance.locale);
+    ClientManager.sendInitNotification(
+      l10n.initAppError,
+      l10n.databaseBuildErrorBody(
+        AppConfig.newIssueUrl.toString(),
+        e.toString(),
+      ),
+    );
+
+    return FlutterHiveCollectionsDatabase.databaseBuilder(client);
+  }
+}
+
+Future<MatrixSdkDatabase> _constructDatabase(Client client) async {
+  if (kIsWeb) {
+    html.window.navigator.storage?.persist();
+    return MatrixSdkDatabase(client.clientName);
+  }
+
+  final cipher = await getDatabaseCipher();
+
+  final fileStoragePath = PlatformInfos.isIOS || PlatformInfos.isMacOS
+      ? await getLibraryDirectory()
+      : await getApplicationSupportDirectory();
+
+  final path = join(fileStoragePath.path, '${client.clientName}.sqlite');
+
+  // fix dlopen for old Android
+  await applyWorkaroundToOpenSqlCipherOnOldAndroidVersions();
+  // import the SQLite / SQLCipher shared objects / dynamic libraries
+  final factory =
+      createDatabaseFactoryFfi(ffiInit: SQfLiteEncryptionHelper.ffiInit);
+
+  // migrate from potential previous SQLite database path to current one
+  await _migrateLegacyLocation(path, client.clientName);
+
+  // required for [getDatabasesPath]
+  databaseFactory = factory;
+
+  // in case we got a cipher, we use the encryption helper
+  // to manage SQLite encryption
+  final helper = SQfLiteEncryptionHelper(
+    factory: factory,
+    path: path,
+    cipher: cipher,
+  );
+
+  // check whether the DB is already encrypted and otherwise do so
+  await helper.ensureDatabaseFileEncrypted();
+
+  final database = await factory.openDatabase(
+    path,
+    options: OpenDatabaseOptions(
+      version: 1,
+      // most important : apply encryption when opening the DB
+      onConfigure: helper.applyPragmaKey,
+    ),
+  );
+
+  return MatrixSdkDatabase(
+    client.clientName,
+    database: database,
+    maxFileSize: 1024 * 1024 * 10,
+    fileStoragePath: fileStoragePath,
+    deleteFilesAfterDuration: const Duration(days: 30),
+  );
+}
+
+Future<void> _migrateLegacyLocation(
+  String sqlFilePath,
+  String clientName,
+) async {
+  final oldPath = PlatformInfos.isDesktop
+      ? (await getApplicationSupportDirectory()).path
+      : await getDatabasesPath();
+
+  final oldFilePath = join(oldPath, clientName);
+  if (oldFilePath == sqlFilePath) return;
+
+  final maybeOldFile = File(oldFilePath);
+  if (await maybeOldFile.exists()) {
+    Logs().i(
+      'Migrate legacy location for database from "$oldFilePath" to "$sqlFilePath"',
+    );
+    await maybeOldFile.copy(sqlFilePath);
+    await maybeOldFile.delete();
+  }
+}

lib/utils/matrix_sdk_extensions/flutter_matrix_dart_sdk_database/cipher.dart

@@ -0,0 +1,48 @@
+import 'dart:convert';
+import 'dart:math';
+
+import 'package:flutter/services.dart';
+
+import 'package:flutter_secure_storage/flutter_secure_storage.dart';
+import 'package:matrix/matrix.dart';
+
+const _passwordStorageKey = 'database_password';
+
+Future<String> getDatabaseCipher() async {
+  String? password;
+
+  try {
+    const secureStorage = FlutterSecureStorage();
+    final containsEncryptionKey =
+        await secureStorage.read(key: _passwordStorageKey) != null;
+    if (!containsEncryptionKey) {
+      final rng = Random.secure();
+      final list = Uint8List(32);
+      list.setAll(0, Iterable.generate(list.length, (i) => rng.nextInt(256)));
+      final newPassword = base64UrlEncode(list);
+      await secureStorage.write(
+        key: _passwordStorageKey,
+        value: newPassword,
+      );
+    }
+    // workaround for if we just wrote to the key and it still doesn't exist
+    password = await secureStorage.read(key: _passwordStorageKey);
+    if (password == null) throw MissingPluginException();
+  } on MissingPluginException catch (_) {
+    const FlutterSecureStorage()
+        .delete(key: _passwordStorageKey)
+        .catchError((_) {});
+    Logs().i('Database encryption is not supported on this platform');
+  } catch (e, s) {
+    const FlutterSecureStorage()
+        .delete(key: _passwordStorageKey)
+        .catchError((_) {});
+    Logs().w('Unable to init database encryption', e, s);
+  }
+
+  // with the new database, we should no longer allow unencrypted storage
+  // secure_storage now supports all platforms we support
+  assert(password != null);
+
+  return password!;
+}

lib/utils/matrix_sdk_extensions/flutter_matrix_dart_sdk_database/sqlcipher_stub.dart

@@ -0,0 +1 @@
+Future<void> applyWorkaroundToOpenSqlCipherOnOldAndroidVersions() async {}

linux/flutter/generated_plugin_registrant.cc

@@ -14,6 +14,7 @@
 #include <flutter_webrtc/flutter_web_r_t_c_plugin.h>
 #include <pasteboard/pasteboard_plugin.h>
 #include <record_linux/record_linux_plugin.h>
+#include <sqlcipher_flutter_libs/sqlite3_flutter_libs_plugin.h>
 #include <url_launcher_linux/url_launcher_plugin.h>
 #include <window_to_front/window_to_front_plugin.h>
 
@@ -42,6 +43,9 @@ void fl_register_plugins(FlPluginRegistry* registry) {
   g_autoptr(FlPluginRegistrar) record_linux_registrar =
       fl_plugin_registry_get_registrar_for_plugin(registry, "RecordLinuxPlugin");
   record_linux_plugin_register_with_registrar(record_linux_registrar);
+  g_autoptr(FlPluginRegistrar) sqlcipher_flutter_libs_registrar =
+      fl_plugin_registry_get_registrar_for_plugin(registry, "Sqlite3FlutterLibsPlugin");
+  sqlite3_flutter_libs_plugin_register_with_registrar(sqlcipher_flutter_libs_registrar);
   g_autoptr(FlPluginRegistrar) url_launcher_linux_registrar =
       fl_plugin_registry_get_registrar_for_plugin(registry, "UrlLauncherPlugin");
   url_launcher_plugin_register_with_registrar(url_launcher_linux_registrar);

linux/flutter/generated_plugins.cmake

@@ -11,6 +11,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
   flutter_webrtc
   pasteboard
   record_linux
+  sqlcipher_flutter_libs
   url_launcher_linux
   window_to_front
 )

macos/Flutter/GeneratedPluginRegistrant.swift

@@ -28,7 +28,7 @@ import record_macos
 import share_plus
 import shared_preferences_foundation
 import sqflite
-import sqflite_sqlcipher
+import sqlcipher_flutter_libs
 import url_launcher_macos
 import video_compress
 import video_player_avfoundation
@@ -59,7 +59,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) {
   SharePlusMacosPlugin.register(with: registry.registrar(forPlugin: "SharePlusMacosPlugin"))
   SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin"))
   SqflitePlugin.register(with: registry.registrar(forPlugin: "SqflitePlugin"))
-  SqfliteSqlCipherPlugin.register(with: registry.registrar(forPlugin: "SqfliteSqlCipherPlugin"))
+  Sqlite3FlutterLibsPlugin.register(with: registry.registrar(forPlugin: "Sqlite3FlutterLibsPlugin"))
   UrlLauncherPlugin.register(with: registry.registrar(forPlugin: "UrlLauncherPlugin"))
   VideoCompressPlugin.register(with: registry.registrar(forPlugin: "VideoCompressPlugin"))
   FVPVideoPlayerPlugin.register(with: registry.registrar(forPlugin: "FVPVideoPlayerPlugin"))