Skip to content

Commit

Permalink
feat: Split up preload and nonpreload room state boxes
Browse files Browse the repository at this point in the history
  • Loading branch information
@krille-chan
krille-chan committed Dec 5, 2023
1 parent 4c480df commit e63d51c
Show file tree
Hide file tree
Showing 5 changed files with 200 additions and 117 deletions.
21 changes: 10 additions & 11 deletions lib/src/database/indexeddb_box.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ class BoxCollection {
if (cache.isEmpty) return;
final txn = _db.transaction(boxNames, readOnly ? 'readonly' : 'readwrite');
for (final fun in cache) {
// The IDB methods return a Future in Dart but must not be awaited in
// order to have an actual transaction. They must only be performed and
// then the transaction object must call `txn.completed;` which then
// returns the actual future.
// https://developer.mozilla.org/en-US/docs/Web/API/IDBTransaction
unawaited(fun(txn));
}
await txn.completed;
Expand All @@ -62,7 +67,10 @@ class BoxCollection {
}
}

Future<void> close() async => _db.close();
Future<void> close() async {
assert(_txnCache == null, 'Database closed while in transaction!');
return _db.close();
}
}

class Box<V> {
Expand All @@ -85,15 +93,6 @@ class Box<V> {
return keys;
}

Future<V?> getWhere(String indexName, String pattern,
[Transaction? txn]) async {
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
final index = store.index(indexName);
final value = await index.get(pattern) as V?;
return value;
}

Future<Map<String, V>> getAllValues([Transaction? txn]) async {
txn ??= boxCollection._db.transaction(name, 'readonly');
final store = txn.objectStore(name);
Expand All @@ -114,7 +113,7 @@ class Box<V> {
}

Future<List<V?>> getAll(List<String> keys, [Transaction? txn]) async {
if (!keys.any((key) => !_cache.containsKey(key))) {
if (keys.every((key) => _cache.containsKey(key))) {
return keys.map((key) => _cache[key]).toList();
}
txn ??= boxCollection._db.transaction(name, 'readonly');
Expand Down
211 changes: 123 additions & 88 deletions lib/src/database/matrix_sdk_database.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,14 @@ class MatrixSdkDatabase extends DatabaseApi {
late Box<Map> _toDeviceQueueBox;

/// Key is a tuple as TupleKey(roomId, type) where stateKey can be
/// an empty string.
late Box<Map> _roomStateBox;
/// an empty string. Must contain only states of type
/// client.importantRoomStates.
late Box<Map> _preloadRoomStateBox;

/// Key is a tuple as TupleKey(roomId, type) where stateKey can be
/// an empty string. Must NOT contain states of a type from
/// client.importantRoomStates.
late Box<Map> _nonPreloadRoomStateBox;

/// Key is a tuple as TupleKey(roomId, userId)
late Box<Map> _roomMembersBox;
Expand Down Expand Up @@ -86,43 +92,49 @@ class MatrixSdkDatabase extends DatabaseApi {
final Directory? fileStoragePath;
final Duration? deleteFilesAfterDuration;

String get _clientBoxName => 'box_client';
static const String _clientBoxName = 'box_client';

static const String _accountDataBoxName = 'box_account_data';

String get _accountDataBoxName => 'box_account_data';
static const String _roomsBoxName = 'box_rooms';

String get _roomsBoxName => 'box_rooms';
static const String _toDeviceQueueBoxName = 'box_to_device_queue';

String get _toDeviceQueueBoxName => 'box_to_device_queue';
static const String _preloadRoomStateBoxName = 'box_preload_room_states';

String get _roomStateBoxName => 'box_room_states';
static const String _nonPreloadRoomStateBoxName =
'box_non_preload_room_states';

String get _roomMembersBoxName => 'box_room_members';
static const String _roomMembersBoxName = 'box_room_members';

String get _roomAccountDataBoxName => 'box_room_account_data';
static const String _roomAccountDataBoxName = 'box_room_account_data';

String get _inboundGroupSessionsBoxName => 'box_inbound_group_session';
static const String _inboundGroupSessionsBoxName =
'box_inbound_group_session';

String get _outboundGroupSessionsBoxName => 'box_outbound_group_session';
static const String _outboundGroupSessionsBoxName =
'box_outbound_group_session';

String get _olmSessionsBoxName => 'box_olm_session';
static const String _olmSessionsBoxName = 'box_olm_session';

String get _userDeviceKeysBoxName => 'box_user_device_keys';
static const String _userDeviceKeysBoxName = 'box_user_device_keys';

String get _userDeviceKeysOutdatedBoxName => 'box_user_device_keys_outdated';
static const String _userDeviceKeysOutdatedBoxName =
'box_user_device_keys_outdated';

String get _userCrossSigningKeysBoxName => 'box_cross_signing_keys';
static const String _userCrossSigningKeysBoxName = 'box_cross_signing_keys';

String get _ssssCacheBoxName => 'box_ssss_cache';
static const String _ssssCacheBoxName = 'box_ssss_cache';

String get _presencesBoxName => 'box_presences';
static const String _presencesBoxName = 'box_presences';

String get _timelineFragmentsBoxName => 'box_timeline_fragments';
static const String _timelineFragmentsBoxName = 'box_timeline_fragments';

String get _eventsBoxName => 'box_events';
static const String _eventsBoxName = 'box_events';

String get _seenDeviceIdsBoxName => 'box_seen_device_ids';
static const String _seenDeviceIdsBoxName = 'box_seen_device_ids';

String get _seenDeviceKeysBoxName => 'box_seen_device_keys';
static const String _seenDeviceKeysBoxName = 'box_seen_device_keys';

Database? database;

Expand All @@ -148,7 +160,8 @@ class MatrixSdkDatabase extends DatabaseApi {
_accountDataBoxName,
_roomsBoxName,
_toDeviceQueueBoxName,
_roomStateBoxName,
_preloadRoomStateBoxName,
_nonPreloadRoomStateBoxName,
_roomMembersBoxName,
_roomAccountDataBoxName,
_inboundGroupSessionsBoxName,
Expand Down Expand Up @@ -176,8 +189,11 @@ class MatrixSdkDatabase extends DatabaseApi {
_roomsBox = _collection.openBox<Map>(
_roomsBoxName,
);
_roomStateBox = _collection.openBox(
_roomStateBoxName,
_preloadRoomStateBox = _collection.openBox(
_preloadRoomStateBoxName,
);
_nonPreloadRoomStateBox = _collection.openBox(
_nonPreloadRoomStateBoxName,
);
_roomMembersBox = _collection.openBox(
_roomMembersBoxName,
Expand Down Expand Up @@ -249,7 +265,8 @@ class MatrixSdkDatabase extends DatabaseApi {
Future<void> clearCache() => transaction(() async {
await _roomsBox.clear();
await _accountDataBox.clear();
await _roomStateBox.clear();
await _preloadRoomStateBox.clear();
await _nonPreloadRoomStateBox.clear();
await _roomMembersBox.clear();
await _eventsBox.clear();
await _timelineFragmentsBox.clear();
Expand Down Expand Up @@ -291,34 +308,41 @@ class MatrixSdkDatabase extends DatabaseApi {
}

@override
Future<void> forgetRoom(String roomId) => transaction(() async {
await _timelineFragmentsBox.delete(TupleKey(roomId, '').toString());
final eventsBoxKeys = await _eventsBox.getAllKeys();
for (final key in eventsBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _eventsBox.delete(key);
}
final roomStateBoxKeys = await _roomStateBox.getAllKeys();
for (final key in roomStateBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _roomStateBox.delete(key);
}
final roomMembersBoxKeys = await _roomMembersBox.getAllKeys();
for (final key in roomMembersBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _roomMembersBox.delete(key);
}
final roomAccountDataBoxKeys = await _roomAccountDataBox.getAllKeys();
for (final key in roomAccountDataBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _roomAccountDataBox.delete(key);
}
await _roomsBox.delete(roomId);
});
Future<void> forgetRoom(String roomId) async {
await _timelineFragmentsBox.delete(TupleKey(roomId, '').toString());
final eventsBoxKeys = await _eventsBox.getAllKeys();
for (final key in eventsBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _eventsBox.delete(key);
}
final preloadRoomStateBoxKeys = await _preloadRoomStateBox.getAllKeys();
for (final key in preloadRoomStateBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _preloadRoomStateBox.delete(key);
}
final nonPreloadRoomStateBoxKeys =
await _nonPreloadRoomStateBox.getAllKeys();
for (final key in nonPreloadRoomStateBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _nonPreloadRoomStateBox.delete(key);
}
final roomMembersBoxKeys = await _roomMembersBox.getAllKeys();
for (final key in roomMembersBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _roomMembersBox.delete(key);
}
final roomAccountDataBoxKeys = await _roomAccountDataBox.getAllKeys();
for (final key in roomAccountDataBoxKeys) {
final multiKey = TupleKey.fromString(key);
if (multiKey.parts.first != roomId) continue;
await _roomAccountDataBox.delete(key);
}
await _roomsBox.delete(roomId);
}

@override
Future<Map<String, BasicEvent>> getAccountData() =>
Expand Down Expand Up @@ -508,7 +532,7 @@ class MatrixSdkDatabase extends DatabaseApi {
final dbKeys = client.importantStateEvents
.map((state) => TupleKey(roomId, state).toString())
.toList();
final rawStates = await _roomStateBox.getAll(dbKeys);
final rawStates = await _preloadRoomStateBox.getAll(dbKeys);
for (final rawState in rawStates) {
if (rawState == null || rawState[''] == null) continue;
room.setState(Event.fromJson(copyMap(rawState['']), room));
Expand All @@ -525,35 +549,29 @@ class MatrixSdkDatabase extends DatabaseApi {

final rawRooms = await _roomsBox.getAllValues();

final getRoomStateRequests = <String, Future<List>>{};

for (final raw in rawRooms.values) {
// Get the room
final room = Room.fromJson(copyMap(raw), client);
// Get the "important" room states. All other states will be loaded once
// `getUnimportantRoomStates()` is called.
final dbKeys = client.importantStateEvents
.map((state) => TupleKey(room.id, state).toString())
.toList();
getRoomStateRequests[room.id] = _roomStateBox.getAll(dbKeys);

// Add to the list and continue.
rooms[room.id] = room;
}

for (final room in rooms.values) {
// Add states to the room
final statesList = await getRoomStateRequests[room.id];
if (statesList != null) {
for (final states in statesList) {
if (states == null) continue;
final stateEvents = states.values
.map((raw) => Event.fromJson(copyMap(raw), room))
.toList();
for (final state in stateEvents) {
room.setState(state);
}
}
final roomStatesDataRaws = await _preloadRoomStateBox.getAllValues();
for (final entry in roomStatesDataRaws.entries) {
final keys = TupleKey.fromString(entry.key);
final roomId = keys.parts.first;
final room = rooms[roomId];
if (room == null) {
Logs().w('Found event in store for unknown room', entry.value);
continue;
}
final states = entry.value;
final stateEvents = states.values
.map((raw) => Event.fromJson(copyMap(raw), room))
.toList();
for (final state in stateEvents) {
room.setState(state);
}
}

Expand Down Expand Up @@ -601,14 +619,14 @@ class MatrixSdkDatabase extends DatabaseApi {
@override
Future<List<Event>> getUnimportantRoomEventStatesForRoom(
List<String> events, Room room) async {
final keys = (await _roomStateBox.getAllKeys()).where((key) {
final keys = (await _nonPreloadRoomStateBox.getAllKeys()).where((key) {
final tuple = TupleKey.fromString(key);
return tuple.parts.first == room.id && !events.contains(tuple.parts[1]);
});

final unimportantEvents = <Event>[];
for (final key in keys) {
final states = await _roomStateBox.get(key);
final states = await _nonPreloadRoomStateBox.get(key);
if (states == null) continue;
unimportantEvents.addAll(
states.values.map((raw) => Event.fromJson(copyMap(raw), room)));
Expand Down Expand Up @@ -928,10 +946,17 @@ class MatrixSdkDatabase extends DatabaseApi {
event.toJson());

if (tmpRoom.lastEvent?.eventId == event.eventId) {
await _roomStateBox.put(
TupleKey(eventUpdate.roomID, event.type).toString(),
{'': event.toJson()},
);
if (client.importantStateEvents.contains(event.type)) {
await _preloadRoomStateBox.put(
TupleKey(eventUpdate.roomID, event.type).toString(),
{'': event.toJson()},
);
} else {
await _nonPreloadRoomStateBox.put(
TupleKey(eventUpdate.roomID, event.type).toString(),
{'': event.toJson()},
);
}
}
}
}
Expand Down Expand Up @@ -1047,19 +1072,23 @@ class MatrixSdkDatabase extends DatabaseApi {
).toString(),
eventUpdate.content);
} else {
final type = eventUpdate.content['type'] as String;
final roomStateBox = client.importantStateEvents.contains(type)
? _preloadRoomStateBox
: _nonPreloadRoomStateBox;
final key = TupleKey(
eventUpdate.roomID,
eventUpdate.content['type'],
type,
).toString();
final stateMap = copyMap(await _roomStateBox.get(key) ?? {});
final stateMap = copyMap(await roomStateBox.get(key) ?? {});
// store state events and new messages, that either are not an edit or an edit of the lastest message
// An edit is an event, that has an edit relation to the latest event. In some cases for the second edit, we need to compare if both have an edit relation to the same event instead.
if (eventUpdate.content
.tryGetMap<String, dynamic>('content')
?.tryGetMap<String, dynamic>('m.relates_to') ==
null) {
stateMap[stateKey] = eventUpdate.content;
await _roomStateBox.put(key, stateMap);
await roomStateBox.put(key, stateMap);
} else {
final editedEventRelationshipEventId = eventUpdate.content
.tryGetMap<String, dynamic>('content')
Expand Down Expand Up @@ -1088,7 +1117,7 @@ class MatrixSdkDatabase extends DatabaseApi {
?.relationshipEventId) // edit of latest (edited event) event
) {
stateMap[stateKey] = eventUpdate.content;
await _roomStateBox.put(key, stateMap);
await roomStateBox.put(key, stateMap);
}
}
}
Expand Down Expand Up @@ -1411,7 +1440,8 @@ class MatrixSdkDatabase extends DatabaseApi {
_clientBoxName: await _clientBox.getAllValues(),
_accountDataBoxName: await _accountDataBox.getAllValues(),
_roomsBoxName: await _roomsBox.getAllValues(),
_roomStateBoxName: await _roomStateBox.getAllValues(),
_preloadRoomStateBoxName: await _preloadRoomStateBox.getAllValues(),
_nonPreloadRoomStateBoxName: await _nonPreloadRoomStateBox.getAllValues(),
_roomMembersBoxName: await _roomMembersBox.getAllValues(),
_toDeviceQueueBoxName: await _toDeviceQueueBox.getAllValues(),
_roomAccountDataBoxName: await _roomAccountDataBox.getAllValues(),
Expand Down Expand Up @@ -1452,8 +1482,13 @@ class MatrixSdkDatabase extends DatabaseApi {
for (final key in json[_roomsBoxName]!.keys) {
await _roomsBox.put(key, json[_roomsBoxName]![key]);
}
for (final key in json[_roomStateBoxName]!.keys) {
await _roomStateBox.put(key, json[_roomStateBoxName]![key]);
for (final key in json[_preloadRoomStateBoxName]!.keys) {
await _preloadRoomStateBox.put(
key, json[_preloadRoomStateBoxName]![key]);
}
for (final key in json[_nonPreloadRoomStateBoxName]!.keys) {
await _nonPreloadRoomStateBox.put(
key, json[_nonPreloadRoomStateBoxName]![key]);
}
for (final key in json[_roomMembersBoxName]!.keys) {
await _roomMembersBox.put(key, json[_roomMembersBoxName]![key]);
Expand Down
Loading

0 comments on commit e63d51c

Please sign in to comment.