diff --git a/lib/src/database/indexeddb_box.dart b/lib/src/database/indexeddb_box.dart index cf6561fb8..dd1c74606 100644 --- a/lib/src/database/indexeddb_box.dart +++ b/lib/src/database/indexeddb_box.dart @@ -50,6 +50,11 @@ class BoxCollection { if (cache.isEmpty) return; final txn = _db.transaction(boxNames, readOnly ? 'readonly' : 'readwrite'); for (final fun in cache) { + // The IDB methods return a Future in Dart but must not be awaited in + // order to have an actual transaction. They must only be performed and + // then the transaction object must call `txn.completed;` which then + // returns the actual future. + // https://developer.mozilla.org/en-US/docs/Web/API/IDBTransaction unawaited(fun(txn)); } await txn.completed; diff --git a/lib/src/database/matrix_sdk_database.dart b/lib/src/database/matrix_sdk_database.dart index ffb9d4ee3..c16f73164 100644 --- a/lib/src/database/matrix_sdk_database.dart +++ b/lib/src/database/matrix_sdk_database.dart @@ -45,8 +45,14 @@ class MatrixSdkDatabase extends DatabaseApi { late Box _toDeviceQueueBox; /// Key is a tuple as TupleKey(roomId, type) where stateKey can be - /// an empty string. - late Box _roomStateBox; + /// an empty string. Must contain only states of type + /// client.importantRoomStates. + late Box _preloadRoomStateBox; + + /// Key is a tuple as TupleKey(roomId, type) where stateKey can be + /// an empty string. Must NOT contain states of a type from + /// client.importantRoomStates. + late Box _nonPreloadRoomStateBox; /// Key is a tuple as TupleKey(roomId, userId) late Box _roomMembersBox; @@ -86,43 +92,49 @@ class MatrixSdkDatabase extends DatabaseApi { final Directory? fileStoragePath; final Duration? deleteFilesAfterDuration; - String get _clientBoxName => 'box_client'; + static const String _clientBoxName = 'box_client'; + + static const String _accountDataBoxName = 'box_account_data'; - String get _accountDataBoxName => 'box_account_data'; + static const String _roomsBoxName = 'box_rooms'; - String get _roomsBoxName => 'box_rooms'; + static const String _toDeviceQueueBoxName = 'box_to_device_queue'; - String get _toDeviceQueueBoxName => 'box_to_device_queue'; + static const String _preloadRoomStateBoxName = 'box_preload_room_states'; - String get _roomStateBoxName => 'box_room_states'; + static const String _nonPreloadRoomStateBoxName = + 'box_non_preload_room_states'; - String get _roomMembersBoxName => 'box_room_members'; + static const String _roomMembersBoxName = 'box_room_members'; - String get _roomAccountDataBoxName => 'box_room_account_data'; + static const String _roomAccountDataBoxName = 'box_room_account_data'; - String get _inboundGroupSessionsBoxName => 'box_inbound_group_session'; + static const String _inboundGroupSessionsBoxName = + 'box_inbound_group_session'; - String get _outboundGroupSessionsBoxName => 'box_outbound_group_session'; + static const String _outboundGroupSessionsBoxName = + 'box_outbound_group_session'; - String get _olmSessionsBoxName => 'box_olm_session'; + static const String _olmSessionsBoxName = 'box_olm_session'; - String get _userDeviceKeysBoxName => 'box_user_device_keys'; + static const String _userDeviceKeysBoxName = 'box_user_device_keys'; - String get _userDeviceKeysOutdatedBoxName => 'box_user_device_keys_outdated'; + static const String _userDeviceKeysOutdatedBoxName = + 'box_user_device_keys_outdated'; - String get _userCrossSigningKeysBoxName => 'box_cross_signing_keys'; + static const String _userCrossSigningKeysBoxName = 'box_cross_signing_keys'; - String get _ssssCacheBoxName => 'box_ssss_cache'; + static const String _ssssCacheBoxName = 'box_ssss_cache'; - String get _presencesBoxName => 'box_presences'; + static const String _presencesBoxName = 'box_presences'; - String get _timelineFragmentsBoxName => 'box_timeline_fragments'; + static const String _timelineFragmentsBoxName = 'box_timeline_fragments'; - String get _eventsBoxName => 'box_events'; + static const String _eventsBoxName = 'box_events'; - String get _seenDeviceIdsBoxName => 'box_seen_device_ids'; + static const String _seenDeviceIdsBoxName = 'box_seen_device_ids'; - String get _seenDeviceKeysBoxName => 'box_seen_device_keys'; + static const String _seenDeviceKeysBoxName = 'box_seen_device_keys'; Database? database; @@ -148,7 +160,8 @@ class MatrixSdkDatabase extends DatabaseApi { _accountDataBoxName, _roomsBoxName, _toDeviceQueueBoxName, - _roomStateBoxName, + _preloadRoomStateBoxName, + _nonPreloadRoomStateBoxName, _roomMembersBoxName, _roomAccountDataBoxName, _inboundGroupSessionsBoxName, @@ -176,8 +189,11 @@ class MatrixSdkDatabase extends DatabaseApi { _roomsBox = _collection.openBox( _roomsBoxName, ); - _roomStateBox = _collection.openBox( - _roomStateBoxName, + _preloadRoomStateBox = _collection.openBox( + _preloadRoomStateBoxName, + ); + _nonPreloadRoomStateBox = _collection.openBox( + _nonPreloadRoomStateBoxName, ); _roomMembersBox = _collection.openBox( _roomMembersBoxName, @@ -249,7 +265,8 @@ class MatrixSdkDatabase extends DatabaseApi { Future clearCache() => transaction(() async { await _roomsBox.clear(); await _accountDataBox.clear(); - await _roomStateBox.clear(); + await _preloadRoomStateBox.clear(); + await _nonPreloadRoomStateBox.clear(); await _roomMembersBox.clear(); await _eventsBox.clear(); await _timelineFragmentsBox.clear(); @@ -291,34 +308,41 @@ class MatrixSdkDatabase extends DatabaseApi { } @override - Future forgetRoom(String roomId) => transaction(() async { - await _timelineFragmentsBox.delete(TupleKey(roomId, '').toString()); - final eventsBoxKeys = await _eventsBox.getAllKeys(); - for (final key in eventsBoxKeys) { - final multiKey = TupleKey.fromString(key); - if (multiKey.parts.first != roomId) continue; - await _eventsBox.delete(key); - } - final roomStateBoxKeys = await _roomStateBox.getAllKeys(); - for (final key in roomStateBoxKeys) { - final multiKey = TupleKey.fromString(key); - if (multiKey.parts.first != roomId) continue; - await _roomStateBox.delete(key); - } - final roomMembersBoxKeys = await _roomMembersBox.getAllKeys(); - for (final key in roomMembersBoxKeys) { - final multiKey = TupleKey.fromString(key); - if (multiKey.parts.first != roomId) continue; - await _roomMembersBox.delete(key); - } - final roomAccountDataBoxKeys = await _roomAccountDataBox.getAllKeys(); - for (final key in roomAccountDataBoxKeys) { - final multiKey = TupleKey.fromString(key); - if (multiKey.parts.first != roomId) continue; - await _roomAccountDataBox.delete(key); - } - await _roomsBox.delete(roomId); - }); + Future forgetRoom(String roomId) async { + await _timelineFragmentsBox.delete(TupleKey(roomId, '').toString()); + final eventsBoxKeys = await _eventsBox.getAllKeys(); + for (final key in eventsBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _eventsBox.delete(key); + } + final preloadRoomStateBoxKeys = await _preloadRoomStateBox.getAllKeys(); + for (final key in preloadRoomStateBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _preloadRoomStateBox.delete(key); + } + final nonPreloadRoomStateBoxKeys = + await _nonPreloadRoomStateBox.getAllKeys(); + for (final key in nonPreloadRoomStateBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _nonPreloadRoomStateBox.delete(key); + } + final roomMembersBoxKeys = await _roomMembersBox.getAllKeys(); + for (final key in roomMembersBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomMembersBox.delete(key); + } + final roomAccountDataBoxKeys = await _roomAccountDataBox.getAllKeys(); + for (final key in roomAccountDataBoxKeys) { + final multiKey = TupleKey.fromString(key); + if (multiKey.parts.first != roomId) continue; + await _roomAccountDataBox.delete(key); + } + await _roomsBox.delete(roomId); + } @override Future> getAccountData() => @@ -508,7 +532,7 @@ class MatrixSdkDatabase extends DatabaseApi { final dbKeys = client.importantStateEvents .map((state) => TupleKey(roomId, state).toString()) .toList(); - final rawStates = await _roomStateBox.getAll(dbKeys); + final rawStates = await _preloadRoomStateBox.getAll(dbKeys); for (final rawState in rawStates) { if (rawState == null || rawState[''] == null) continue; room.setState(Event.fromJson(copyMap(rawState['']), room)); @@ -525,35 +549,29 @@ class MatrixSdkDatabase extends DatabaseApi { final rawRooms = await _roomsBox.getAllValues(); - final getRoomStateRequests = >{}; - for (final raw in rawRooms.values) { // Get the room final room = Room.fromJson(copyMap(raw), client); - // Get the "important" room states. All other states will be loaded once - // `getUnimportantRoomStates()` is called. - final dbKeys = client.importantStateEvents - .map((state) => TupleKey(room.id, state).toString()) - .toList(); - getRoomStateRequests[room.id] = _roomStateBox.getAll(dbKeys); // Add to the list and continue. rooms[room.id] = room; } - for (final room in rooms.values) { - // Add states to the room - final statesList = await getRoomStateRequests[room.id]; - if (statesList != null) { - for (final states in statesList) { - if (states == null) continue; - final stateEvents = states.values - .map((raw) => Event.fromJson(copyMap(raw), room)) - .toList(); - for (final state in stateEvents) { - room.setState(state); - } - } + final roomStatesDataRaws = await _preloadRoomStateBox.getAllValues(); + for (final entry in roomStatesDataRaws.entries) { + final keys = TupleKey.fromString(entry.key); + final roomId = keys.parts.first; + final room = rooms[roomId]; + if (room == null) { + Logs().w('Found event in store for unknown room', entry.value); + continue; + } + final states = entry.value; + final stateEvents = states.values + .map((raw) => Event.fromJson(copyMap(raw), room)) + .toList(); + for (final state in stateEvents) { + room.setState(state); } } @@ -601,14 +619,14 @@ class MatrixSdkDatabase extends DatabaseApi { @override Future> getUnimportantRoomEventStatesForRoom( List events, Room room) async { - final keys = (await _roomStateBox.getAllKeys()).where((key) { + final keys = (await _nonPreloadRoomStateBox.getAllKeys()).where((key) { final tuple = TupleKey.fromString(key); return tuple.parts.first == room.id && !events.contains(tuple.parts[1]); }); final unimportantEvents = []; for (final key in keys) { - final states = await _roomStateBox.get(key); + final states = await _nonPreloadRoomStateBox.get(key); if (states == null) continue; unimportantEvents.addAll( states.values.map((raw) => Event.fromJson(copyMap(raw), room))); @@ -928,10 +946,17 @@ class MatrixSdkDatabase extends DatabaseApi { event.toJson()); if (tmpRoom.lastEvent?.eventId == event.eventId) { - await _roomStateBox.put( - TupleKey(eventUpdate.roomID, event.type).toString(), - {'': event.toJson()}, - ); + if (client.importantStateEvents.contains(event.type)) { + await _preloadRoomStateBox.put( + TupleKey(eventUpdate.roomID, event.type).toString(), + {'': event.toJson()}, + ); + } else { + await _nonPreloadRoomStateBox.put( + TupleKey(eventUpdate.roomID, event.type).toString(), + {'': event.toJson()}, + ); + } } } } @@ -1047,11 +1072,15 @@ class MatrixSdkDatabase extends DatabaseApi { ).toString(), eventUpdate.content); } else { + final type = eventUpdate.content['type'] as String; + final roomStateBox = client.importantStateEvents.contains(type) + ? _preloadRoomStateBox + : _nonPreloadRoomStateBox; final key = TupleKey( eventUpdate.roomID, - eventUpdate.content['type'], + type, ).toString(); - final stateMap = copyMap(await _roomStateBox.get(key) ?? {}); + final stateMap = copyMap(await roomStateBox.get(key) ?? {}); // store state events and new messages, that either are not an edit or an edit of the lastest message // An edit is an event, that has an edit relation to the latest event. In some cases for the second edit, we need to compare if both have an edit relation to the same event instead. if (eventUpdate.content @@ -1059,7 +1088,7 @@ class MatrixSdkDatabase extends DatabaseApi { ?.tryGetMap('m.relates_to') == null) { stateMap[stateKey] = eventUpdate.content; - await _roomStateBox.put(key, stateMap); + await roomStateBox.put(key, stateMap); } else { final editedEventRelationshipEventId = eventUpdate.content .tryGetMap('content') @@ -1088,7 +1117,7 @@ class MatrixSdkDatabase extends DatabaseApi { ?.relationshipEventId) // edit of latest (edited event) event ) { stateMap[stateKey] = eventUpdate.content; - await _roomStateBox.put(key, stateMap); + await roomStateBox.put(key, stateMap); } } } @@ -1411,7 +1440,8 @@ class MatrixSdkDatabase extends DatabaseApi { _clientBoxName: await _clientBox.getAllValues(), _accountDataBoxName: await _accountDataBox.getAllValues(), _roomsBoxName: await _roomsBox.getAllValues(), - _roomStateBoxName: await _roomStateBox.getAllValues(), + _preloadRoomStateBoxName: await _preloadRoomStateBox.getAllValues(), + _nonPreloadRoomStateBoxName: await _nonPreloadRoomStateBox.getAllValues(), _roomMembersBoxName: await _roomMembersBox.getAllValues(), _toDeviceQueueBoxName: await _toDeviceQueueBox.getAllValues(), _roomAccountDataBoxName: await _roomAccountDataBox.getAllValues(), @@ -1452,8 +1482,13 @@ class MatrixSdkDatabase extends DatabaseApi { for (final key in json[_roomsBoxName]!.keys) { await _roomsBox.put(key, json[_roomsBoxName]![key]); } - for (final key in json[_roomStateBoxName]!.keys) { - await _roomStateBox.put(key, json[_roomStateBoxName]![key]); + for (final key in json[_preloadRoomStateBoxName]!.keys) { + await _preloadRoomStateBox.put( + key, json[_preloadRoomStateBoxName]![key]); + } + for (final key in json[_nonPreloadRoomStateBoxName]!.keys) { + await _nonPreloadRoomStateBox.put( + key, json[_nonPreloadRoomStateBoxName]![key]); } for (final key in json[_roomMembersBoxName]!.keys) { await _roomMembersBox.put(key, json[_roomMembersBoxName]![key]); diff --git a/lib/src/database/sqflite_box.dart b/lib/src/database/sqflite_box.dart index 9b4a6e914..31d5edc83 100644 --- a/lib/src/database/sqflite_box.dart +++ b/lib/src/database/sqflite_box.dart @@ -1,7 +1,10 @@ +import 'dart:async'; import 'dart:convert'; import 'package:sqflite_common/sqlite_api.dart'; +import 'package:matrix/matrix.dart'; + /// Key-Value store abstraction over Sqflite so that the sdk database can use /// a single interface for all platforms. API is inspired by Hive. class BoxCollection { @@ -40,19 +43,65 @@ class BoxCollection { Batch? _activeBatch; + Completer? _transactionLock; + final _transactionZones = {}; + Future transaction( Future Function() action, { List? boxNames, bool readOnly = false, }) async { - boxNames ??= this.boxNames.toList(); - _activeBatch = _db.batch(); - await action(); - final batch = _activeBatch; - _activeBatch = null; - if (batch == null) return; - await batch.commit(noResult: true); - return; + // we want transactions to lock, however NOT if transactoins are run inside of each other. + // to be able to do this, we use dart zones (https://dart.dev/articles/archive/zones). + // _transactionZones holds a set of all zones which are currently running a transaction. + // _transactionLock holds the lock. + + // first we try to determine if we are inside of a transaction currently + var isInTransaction = false; + Zone? zone = Zone.current; + // for that we keep on iterating to the parent zone until there is either no zone anymore + // or we have found a zone inside of _transactionZones. + while (zone != null) { + if (_transactionZones.contains(zone)) { + isInTransaction = true; + break; + } + zone = zone.parent; + } + // if we are inside a transaction....just run the action + if (isInTransaction) { + return await action(); + } + // if we are *not* in a transaction, time to wait for the lock! + while (_transactionLock != null) { + await _transactionLock!.future; + } + // claim the lock + final lock = Completer(); + _transactionLock = lock; + try { + // run the action inside of a new zone + return await runZoned(() async { + try { + // don't forget to add the new zone to _transactionZones! + _transactionZones.add(Zone.current); + + final batch = _db.batch(); + _activeBatch = batch; + await action(); + _activeBatch = null; + await batch.commit(noResult: true); + return; + } finally { + // aaaand remove the zone from _transactionZones again + _transactionZones.remove(Zone.current); + } + }); + } finally { + // aaaand finally release the lock + _transactionLock = null; + lock.complete(); + } } Future clear() => transaction( @@ -144,12 +193,12 @@ class Box { final result = await executor.query(name); return Map.fromEntries( - result.where((row) => row['v'] != null).map( - (row) => MapEntry( - row['k'] as String, - _fromString(row['v']) as V, - ), - ), + result.map( + (row) => MapEntry( + row['k'] as String, + _fromString(row['v']) as V, + ), + ), ); } diff --git a/test/database_api_test.dart b/test/database_api_test.dart index f85d97013..6e3c9999e 100644 --- a/test/database_api_test.dart +++ b/test/database_api_test.dart @@ -27,10 +27,10 @@ import 'package:matrix/matrix.dart'; import 'fake_database.dart'; void main() { - group('HiveCollections Database Test', () { + group('Matrix SDK Database Test', () { late DatabaseApi database; late int toDeviceQueueIndex; - test('Open', () async { + test('Setup', () async { database = await getMatrixSdkDatabase(null); }); test('transaction', () async { diff --git a/test/fake_database.dart b/test/fake_database.dart index 59d966834..0f8f7188c 100644 --- a/test/fake_database.dart +++ b/test/fake_database.dart @@ -43,8 +43,10 @@ Future getHiveCollectionsDatabase(Client? c) async { // ignore: deprecated_member_use_from_same_package Future getMatrixSdkDatabase(Client? c) async { - final fileSystem = MemoryFileSystem(); - final path = '${fileSystem.path}/${Random().nextDouble()}'; + final fileSystem = await LocalFileSystem() + .systemTempDirectory + .createTemp('dart-sdk-tests-database'); + final path = '${fileSystem.path}/${c.hashCode}'; final database = await databaseFactoryFfi.openDatabase(path); final db = MatrixSdkDatabase('unit_test.${c?.hashCode}', database: database); await db.open();