From 143d6cb6b471671cc4fa4eebfdaa42dad6655bf9 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Wed, 12 Feb 2025 09:11:12 -0500 Subject: [PATCH 01/14] Update auth --- 3rd-party/auth | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rd-party/auth b/3rd-party/auth index 56cf85438..579ce1083 160000 --- a/3rd-party/auth +++ b/3rd-party/auth @@ -1 +1 @@ -Subproject commit 56cf85438990004975f34bf121bdef22d4b81068 +Subproject commit 579ce1083313034264967fd842f3fd19e7f49ee5 From 40118d788428a17732cae901b2a4831db7d6931e Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Tue, 11 Feb 2025 22:33:43 -0500 Subject: [PATCH 02/14] Add stream encryption to crypto service --- .../auth/services/crypto/crypto.service.ts | 83 +++++++++++++++++-- .../src/nest/auth/services/crypto/types.ts | 10 ++- 2 files changed, 84 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/nest/auth/services/crypto/crypto.service.ts b/packages/backend/src/nest/auth/services/crypto/crypto.service.ts index a385f93c9..2c1549355 100644 --- a/packages/backend/src/nest/auth/services/crypto/crypto.service.ts +++ b/packages/backend/src/nest/auth/services/crypto/crypto.service.ts @@ -12,9 +12,18 @@ import { } from './types' import { ChainServiceBase } from '../chainServiceBase' import { SigChain } from '../../sigchain' -import { asymmetric, Base58, Keyset, LocalUserContext, Member, SignedEnvelope } from '@localfirst/auth' +import { + asymmetric, + Base58, + Keyset, + LocalUserContext, + Member, + SignedEnvelope, + EncryptStreamTeamPayload, +} from '@localfirst/auth' import { DEFAULT_SEARCH_OPTIONS, MemberSearchOptions } from '../members/types' import { createLogger } from '../../../common/logger' +import { KeyMetadata } from '3rd-party/auth/packages/crdx/dist' const logger = createLogger('auth:cryptoService') @@ -69,7 +78,7 @@ class CryptoService extends ChainServiceBase { const envelope = this.sigChain.team!.encrypt(message, scope.name) return { - contents: bs58.default.encode(envelope.contents) as Base58, + contents: envelope.contents, scope: { ...scope, generation: envelope.recipient.generation, @@ -87,7 +96,7 @@ class CryptoService extends ChainServiceBase { const senderKey = context.user.keys.encryption.secretKey const generation = recipientKeys[0].generation - const encryptedContents = asymmetric.encrypt({ + const encryptedContents = asymmetric.encryptBytes({ secret: message, senderSecretKey: senderKey, recipientPublicKey: recipientKey, @@ -146,7 +155,7 @@ class CryptoService extends ChainServiceBase { } return this.sigChain.team!.decrypt({ - contents: bs58.default.decode(encrypted.contents), + contents: encrypted.contents, recipient: { ...encrypted.scope, // you don't need a name on the scope when encrypting but you need one for decrypting because of how LFA searches for keys in lockboxes @@ -164,12 +173,76 @@ class CryptoService extends ChainServiceBase { const recipientKey = context.user.keys.encryption.secretKey const senderKey = senderKeys[0].encryption - return asymmetric.decrypt({ + return asymmetric.decryptBytes({ cipher: encrypted.contents, senderPublicKey: senderKey, recipientSecretKey: recipientKey, }) as T } + + public encryptStream(stream: AsyncIterable, scope: EncryptionScope): EncryptStreamTeamPayload { + let payload: EncryptStreamTeamPayload + switch (scope.type) { + // Symmetrical Encryption Types + case EncryptionScopeType.CHANNEL: + case EncryptionScopeType.ROLE: + case EncryptionScopeType.TEAM: + payload = this.symEncryptStream(stream, scope) + break + // Asymmetrical Encryption Types + case EncryptionScopeType.USER: + throw new Error(`Stream encryption for scope type ${scope.type} is not currently supported!`) + // Unknown Type + default: + throw new Error(`Unknown encryption type ${scope.type} provided!`) + } + + return payload + } + + private symEncryptStream(stream: AsyncIterable, scope: EncryptionScope): EncryptStreamTeamPayload { + if (scope.type != EncryptionScopeType.TEAM && scope.name == null) { + throw new Error(`Must provide a scope name when encryption scope is set to ${scope.type}`) + } + + return this.sigChain.team!.encryptStream(stream, scope.name) + } + + public decryptStream( + encryptedStream: AsyncIterable, + header: Uint8Array, + scope: KeyMetadata + ): AsyncGenerator { + let decryptedStream: AsyncGenerator + switch (scope.type) { + // Symmetrical Encryption Types + case EncryptionScopeType.CHANNEL: + case EncryptionScopeType.ROLE: + case EncryptionScopeType.TEAM: + decryptedStream = this.symDecryptStream(encryptedStream, header, scope) + break + // Asymmetrical Encryption Types + case EncryptionScopeType.USER: + throw new Error(`Stream encryption for scope type ${scope.type} is not currently supported!`) + // Unknown Type + default: + throw new Error(`Unknown encryption scope type ${scope.type}`) + } + + return decryptedStream + } + + private symDecryptStream( + encryptedStream: AsyncIterable, + header: Uint8Array, + scope: KeyMetadata + ): AsyncGenerator { + if (scope.type !== EncryptionScopeType.TEAM && scope.name == null) { + throw new Error(`Must provide a scope name when encryption scope is set to ${scope.type}`) + } + + return this.sigChain.team!.decryptStream(encryptedStream, header, scope) + } } export { CryptoService } diff --git a/packages/backend/src/nest/auth/services/crypto/types.ts b/packages/backend/src/nest/auth/services/crypto/types.ts index 7c9c0bc49..854f143c4 100644 --- a/packages/backend/src/nest/auth/services/crypto/types.ts +++ b/packages/backend/src/nest/auth/services/crypto/types.ts @@ -12,11 +12,13 @@ export type EncryptionScope = { name?: string } +export type EncryptionScopeDetail = EncryptionScope & { + generation: number +} + export type EncryptedPayload = { - contents: Base58 - scope: EncryptionScope & { - generation: number - } + contents: Uint8Array + scope: EncryptionScopeDetail } export type EncryptedAndSignedPayload = { From a579dc11a91e677d1ecbc6571f61b24ae21e30aa Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Tue, 11 Feb 2025 22:34:09 -0500 Subject: [PATCH 03/14] Add oneToOne chunker --- packages/backend/package-lock.json | 900 +++++++++++------- packages/backend/package.json | 1 + .../unixfs-utils/oneToOneChunker.ts | 36 + 3 files changed, 607 insertions(+), 330 deletions(-) create mode 100644 packages/backend/src/nest/ipfs-file-manager/unixfs-utils/oneToOneChunker.ts diff --git a/packages/backend/package-lock.json b/packages/backend/package-lock.json index 9dc894693..67bafe399 100644 --- a/packages/backend/package-lock.json +++ b/packages/backend/package-lock.json @@ -67,6 +67,7 @@ "https-proxy-agent": "^7.0.5", "image-size": "^1.0.1", "interface-datastore": "8.3.1", + "ipfs-unixfs-importer": "15.3.1", "it-drain": "^3.0.7", "it-first": "^3.0.6", "it-length-prefixed": "^9.1.0", @@ -1251,11 +1252,6 @@ "uint8arrays": "^5.0.2" } }, - "node_modules/@helia/unixfs/node_modules/@assemblyscript/loader": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.9.4.tgz", - "integrity": "sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA==" - }, "node_modules/@helia/unixfs/node_modules/@helia/interface": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@helia/interface/-/interface-5.1.0.tgz", @@ -1386,16 +1382,6 @@ } ] }, - "node_modules/@helia/unixfs/node_modules/bl": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", - "integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==", - "dependencies": { - "buffer": "^6.0.3", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "node_modules/@helia/unixfs/node_modules/braces": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", @@ -1524,11 +1510,6 @@ } ] }, - "node_modules/@helia/unixfs/node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, "node_modules/@helia/unixfs/node_modules/interface-blockstore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/interface-blockstore/-/interface-blockstore-5.3.1.tgz", @@ -1575,28 +1556,6 @@ "progress-events": "^1.0.1" } }, - "node_modules/@helia/unixfs/node_modules/ipfs-unixfs-importer": { - "version": "15.3.1", - "resolved": "https://registry.npmjs.org/ipfs-unixfs-importer/-/ipfs-unixfs-importer-15.3.1.tgz", - "integrity": "sha512-wHCTBqNsZXLJZ9/GSr7Msb3FDXD5yXF20Y9sKyUbbqNjbvaXs3n3h1+NM/5+WrgESHfwRcJIlJtaOKafL8Ymdg==", - "dependencies": { - "@ipld/dag-pb": "^4.1.2", - "@multiformats/murmur3": "^2.1.8", - "hamt-sharding": "^3.0.6", - "interface-blockstore": "^5.3.0", - "interface-store": "^6.0.0", - "ipfs-unixfs": "^11.0.0", - "it-all": "^3.0.6", - "it-batch": "^3.0.6", - "it-first": "^3.0.6", - "it-parallel-batch": "^3.0.6", - "multiformats": "^13.2.3", - "progress-events": "^1.0.1", - "rabin-wasm": "^0.1.5", - "uint8arraylist": "^2.4.8", - "uint8arrays": "^5.1.0" - } - }, "node_modules/@helia/unixfs/node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -1637,11 +1596,6 @@ "resolved": "https://registry.npmjs.org/it-all/-/it-all-3.0.6.tgz", "integrity": "sha512-HXZWbxCgQZJfrv5rXvaVeaayXED8nTKx9tj9fpBhmcUJcedVZshMMMqTj0RG2+scGypb9Ut1zd1ifbf3lA8L+Q==" }, - "node_modules/@helia/unixfs/node_modules/it-batch": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/it-batch/-/it-batch-3.0.6.tgz", - "integrity": "sha512-pQAAlSvJ4aV6xM/6LRvkPdKSKXxS4my2fGzNUxJyAQ8ccFdxPmK1bUuF5OoeUDkcdrbs8jtsmc4DypCMrGY6sg==" - }, "node_modules/@helia/unixfs/node_modules/it-filter": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/it-filter/-/it-filter-3.1.1.tgz", @@ -1679,14 +1633,6 @@ "p-defer": "^4.0.1" } }, - "node_modules/@helia/unixfs/node_modules/it-parallel-batch": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/it-parallel-batch/-/it-parallel-batch-3.0.6.tgz", - "integrity": "sha512-3wgiQGvMMHy65OXScrtrtmY+bJSF7P6St1AP+BU+SK83fEr8NNk/MrmJKrtB1+MahYX2a8I+pOGKDj8qVtuV0Q==", - "dependencies": { - "it-batch": "^3.0.0" - } - }, "node_modules/@helia/unixfs/node_modules/it-peekable": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/it-peekable/-/it-peekable-3.0.5.tgz", @@ -1723,14 +1669,6 @@ "node": ">=8.6" } }, - "node_modules/@helia/unixfs/node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/@helia/unixfs/node_modules/ms": { "version": "3.0.0-canary.1", "resolved": "https://registry.npmjs.org/ms/-/ms-3.0.0-canary.1.tgz", @@ -1747,25 +1685,6 @@ "node": ">=8.0.0" } }, - "node_modules/@helia/unixfs/node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/@helia/unixfs/node_modules/p-queue": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz", @@ -1832,35 +1751,6 @@ } ] }, - "node_modules/@helia/unixfs/node_modules/rabin-wasm": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/rabin-wasm/-/rabin-wasm-0.1.5.tgz", - "integrity": "sha512-uWgQTo7pim1Rnj5TuWcCewRDTf0PEFTSlaUjWP4eY9EbLV9em08v89oCz/WO+wRxpYuO36XEHp4wgYQnAgOHzA==", - "dependencies": { - "@assemblyscript/loader": "^0.9.4", - "bl": "^5.0.0", - "debug": "^4.3.1", - "minimist": "^1.2.5", - "node-fetch": "^2.6.1", - "readable-stream": "^3.6.0" - }, - "bin": { - "rabin-wasm": "cli/bin.js" - } - }, - "node_modules/@helia/unixfs/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/@helia/unixfs/node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -1883,38 +1773,11 @@ "queue-microtask": "^1.2.2" } }, - "node_modules/@helia/unixfs/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/@helia/unixfs/node_modules/sparse-array": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/sparse-array/-/sparse-array-1.3.2.tgz", "integrity": "sha512-ZT711fePGn3+kQyLuv1fpd3rNSkNF8vd5Kv2D+qnOANeyKs3fx6bUMGWRPvgTTcYV64QMqZKZwcuaQSP3AZ0tg==" }, - "node_modules/@helia/unixfs/node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/@helia/unixfs/node_modules/supports-color": { "version": "9.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-9.4.0.tgz", @@ -1937,11 +1800,6 @@ "node": ">=8.0" } }, - "node_modules/@helia/unixfs/node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "node_modules/@helia/unixfs/node_modules/uint8-varint": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/uint8-varint/-/uint8-varint-2.0.4.tgz", @@ -1951,11 +1809,6 @@ "uint8arrays": "^5.0.0" } }, - "node_modules/@helia/unixfs/node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" - }, "node_modules/@helia/unixfs/node_modules/weald": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/weald/-/weald-1.0.4.tgz", @@ -1965,20 +1818,6 @@ "supports-color": "^9.4.0" } }, - "node_modules/@helia/unixfs/node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/@helia/unixfs/node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/@ipld/dag-cbor": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.2.2.tgz", @@ -13841,60 +13680,365 @@ "resolved": "https://registry.npmjs.org/interface-store/-/interface-store-6.0.2.tgz", "integrity": "sha512-KSFCXtBlNoG0hzwNa0RmhHtrdhzexp+S+UY2s0rWTBJyfdEIgn6i6Zl9otVqrcFYbYrneBT7hbmHQ8gE0C3umA==" }, - "node_modules/it-drain": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/it-drain/-/it-drain-3.0.7.tgz", - "integrity": "sha512-vy6S1JKjjHSIFHgBpLpD1zhkCRl3z1zYWUxE14+kAYf+BL9ssWSFImJfhl361IIcwr0ofw8etzg11VqqB+ntUA==" + "node_modules/ipfs-unixfs-importer": { + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/ipfs-unixfs-importer/-/ipfs-unixfs-importer-15.3.1.tgz", + "integrity": "sha512-wHCTBqNsZXLJZ9/GSr7Msb3FDXD5yXF20Y9sKyUbbqNjbvaXs3n3h1+NM/5+WrgESHfwRcJIlJtaOKafL8Ymdg==", + "dependencies": { + "@ipld/dag-pb": "^4.1.2", + "@multiformats/murmur3": "^2.1.8", + "hamt-sharding": "^3.0.6", + "interface-blockstore": "^5.3.0", + "interface-store": "^6.0.0", + "ipfs-unixfs": "^11.0.0", + "it-all": "^3.0.6", + "it-batch": "^3.0.6", + "it-first": "^3.0.6", + "it-parallel-batch": "^3.0.6", + "multiformats": "^13.2.3", + "progress-events": "^1.0.1", + "rabin-wasm": "^0.1.5", + "uint8arraylist": "^2.4.8", + "uint8arrays": "^5.1.0" + } }, - "node_modules/it-first": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/it-first/-/it-first-3.0.6.tgz", - "integrity": "sha512-ExIewyK9kXKNAplg2GMeWfgjUcfC1FnUXz/RPfAvIXby+w7U4b3//5Lic0NV03gXT8O/isj5Nmp6KiY0d45pIQ==" + "node_modules/ipfs-unixfs-importer/node_modules/@assemblyscript/loader": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.9.4.tgz", + "integrity": "sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA==" }, - "node_modules/it-length-prefixed": { - "version": "9.1.1", - "resolved": "https://registry.npmjs.org/it-length-prefixed/-/it-length-prefixed-9.1.1.tgz", - "integrity": "sha512-O88nBweT6M9ozsmok68/auKH7ik/slNM4pYbM9lrfy2z5QnpokW5SlrepHZDKtN71llhG2sZvd6uY4SAl+lAQg==", + "node_modules/ipfs-unixfs-importer/node_modules/@multiformats/murmur3": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/@multiformats/murmur3/-/murmur3-2.1.8.tgz", + "integrity": "sha512-6vId1C46ra3R1sbJUOFCZnsUIveR9oF20yhPmAFxPm0JfrX3/ZRCgP3YDrBzlGoEppOXnA9czHeYc0T9mB6hbA==", "dependencies": { - "it-reader": "^6.0.1", - "it-stream-types": "^2.0.1", - "uint8-varint": "^2.0.1", - "uint8arraylist": "^2.0.0", - "uint8arrays": "^5.0.1" + "multiformats": "^13.0.0", + "murmurhash3js-revisited": "^3.0.0" }, "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" } }, - "node_modules/it-length-prefixed/node_modules/it-reader": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/it-reader/-/it-reader-6.0.4.tgz", - "integrity": "sha512-XCWifEcNFFjjBHtor4Sfaj8rcpt+FkY0L6WdhD578SCDhV4VUm7fCkF3dv5a+fTcfQqvN9BsxBTvWbYO6iCjTg==", + "node_modules/ipfs-unixfs-importer/node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/ipfs-unixfs-importer/node_modules/bl": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", + "integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==", "dependencies": { - "it-stream-types": "^2.0.1", - "uint8arraylist": "^2.0.0" - }, - "engines": { - "node": ">=16.0.0", - "npm": ">=7.0.0" + "buffer": "^6.0.3", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" } }, - "node_modules/it-length-prefixed/node_modules/uint8-varint": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/uint8-varint/-/uint8-varint-2.0.4.tgz", - "integrity": "sha512-FwpTa7ZGA/f/EssWAb5/YV6pHgVF1fViKdW8cWaEarjB8t7NyofSWBdOTyFPaGuUG4gx3v1O3PQ8etsiOs3lcw==", + "node_modules/ipfs-unixfs-importer/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], "dependencies": { - "uint8arraylist": "^2.0.0", - "uint8arrays": "^5.0.0" + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" } }, - "node_modules/it-pipe": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/it-pipe/-/it-pipe-3.0.1.tgz", - "integrity": "sha512-sIoNrQl1qSRg2seYSBH/3QxWhJFn9PKYvOf/bHdtCBF0bnghey44VyASsWzn5dAx0DCDDABq1hZIuzKmtBZmKA==", - "dependencies": { - "it-merge": "^3.0.0", + "node_modules/ipfs-unixfs-importer/node_modules/hamt-sharding": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/hamt-sharding/-/hamt-sharding-3.0.6.tgz", + "integrity": "sha512-nZeamxfymIWLpVcAN0CRrb7uVq3hCOGj9IcL6NMA6VVCVWqj+h9Jo/SmaWuS92AEDf1thmHsM5D5c70hM3j2Tg==", + "dependencies": { + "sparse-array": "^1.3.1", + "uint8arrays": "^5.0.1" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/ipfs-unixfs-importer/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ipfs-unixfs-importer/node_modules/interface-blockstore": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/interface-blockstore/-/interface-blockstore-5.3.1.tgz", + "integrity": "sha512-nhgrQnz6yUQEqxTFLhlOBurQOy5lWlwCpgFmZ3GTObTVTQS9RZjK/JTozY6ty9uz2lZs7VFJSqwjWAltorJ4Vw==", + "dependencies": { + "interface-store": "^6.0.0", + "multiformats": "^13.2.3" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/interface-store": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/interface-store/-/interface-store-6.0.2.tgz", + "integrity": "sha512-KSFCXtBlNoG0hzwNa0RmhHtrdhzexp+S+UY2s0rWTBJyfdEIgn6i6Zl9otVqrcFYbYrneBT7hbmHQ8gE0C3umA==" + }, + "node_modules/ipfs-unixfs-importer/node_modules/ipfs-unixfs": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/ipfs-unixfs/-/ipfs-unixfs-11.2.0.tgz", + "integrity": "sha512-J8FN1qM5nfrDo8sQKQwfj0+brTg1uBfZK2vY9hxci33lcl3BFrsELS9+1+4q/8tO1ASKfxZO8W3Pi2O4sVX2Lg==", + "dependencies": { + "protons-runtime": "^5.5.0", + "uint8arraylist": "^2.4.8" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/it-all": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/it-all/-/it-all-3.0.6.tgz", + "integrity": "sha512-HXZWbxCgQZJfrv5rXvaVeaayXED8nTKx9tj9fpBhmcUJcedVZshMMMqTj0RG2+scGypb9Ut1zd1ifbf3lA8L+Q==" + }, + "node_modules/ipfs-unixfs-importer/node_modules/it-batch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/it-batch/-/it-batch-3.0.6.tgz", + "integrity": "sha512-pQAAlSvJ4aV6xM/6LRvkPdKSKXxS4my2fGzNUxJyAQ8ccFdxPmK1bUuF5OoeUDkcdrbs8jtsmc4DypCMrGY6sg==" + }, + "node_modules/ipfs-unixfs-importer/node_modules/it-parallel-batch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/it-parallel-batch/-/it-parallel-batch-3.0.6.tgz", + "integrity": "sha512-3wgiQGvMMHy65OXScrtrtmY+bJSF7P6St1AP+BU+SK83fEr8NNk/MrmJKrtB1+MahYX2a8I+pOGKDj8qVtuV0Q==", + "dependencies": { + "it-batch": "^3.0.0" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/murmurhash3js-revisited": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/murmurhash3js-revisited/-/murmurhash3js-revisited-3.0.0.tgz", + "integrity": "sha512-/sF3ee6zvScXMb1XFJ8gDsSnY+X8PbOyjIuBhtgis10W2Jx4ZjIhikUCIF9c4gpJxVnQIsPAFrSwTCuAjicP6g==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/protons-runtime": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/protons-runtime/-/protons-runtime-5.5.0.tgz", + "integrity": "sha512-EsALjF9QsrEk6gbCx3lmfHxVN0ah7nG3cY7GySD4xf4g8cr7g543zB88Foh897Sr1RQJ9yDCUsoT1i1H/cVUFA==", + "dependencies": { + "uint8-varint": "^2.0.2", + "uint8arraylist": "^2.4.3", + "uint8arrays": "^5.0.1" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/rabin-wasm": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/rabin-wasm/-/rabin-wasm-0.1.5.tgz", + "integrity": "sha512-uWgQTo7pim1Rnj5TuWcCewRDTf0PEFTSlaUjWP4eY9EbLV9em08v89oCz/WO+wRxpYuO36XEHp4wgYQnAgOHzA==", + "dependencies": { + "@assemblyscript/loader": "^0.9.4", + "bl": "^5.0.0", + "debug": "^4.3.1", + "minimist": "^1.2.5", + "node-fetch": "^2.6.1", + "readable-stream": "^3.6.0" + }, + "bin": { + "rabin-wasm": "cli/bin.js" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/ipfs-unixfs-importer/node_modules/sparse-array": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/sparse-array/-/sparse-array-1.3.2.tgz", + "integrity": "sha512-ZT711fePGn3+kQyLuv1fpd3rNSkNF8vd5Kv2D+qnOANeyKs3fx6bUMGWRPvgTTcYV64QMqZKZwcuaQSP3AZ0tg==" + }, + "node_modules/ipfs-unixfs-importer/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/ipfs-unixfs-importer/node_modules/uint8-varint": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/uint8-varint/-/uint8-varint-2.0.4.tgz", + "integrity": "sha512-FwpTa7ZGA/f/EssWAb5/YV6pHgVF1fViKdW8cWaEarjB8t7NyofSWBdOTyFPaGuUG4gx3v1O3PQ8etsiOs3lcw==", + "dependencies": { + "uint8arraylist": "^2.0.0", + "uint8arrays": "^5.0.0" + } + }, + "node_modules/ipfs-unixfs-importer/node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "node_modules/ipfs-unixfs-importer/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/ipfs-unixfs-importer/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/it-drain": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/it-drain/-/it-drain-3.0.7.tgz", + "integrity": "sha512-vy6S1JKjjHSIFHgBpLpD1zhkCRl3z1zYWUxE14+kAYf+BL9ssWSFImJfhl361IIcwr0ofw8etzg11VqqB+ntUA==" + }, + "node_modules/it-first": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/it-first/-/it-first-3.0.6.tgz", + "integrity": "sha512-ExIewyK9kXKNAplg2GMeWfgjUcfC1FnUXz/RPfAvIXby+w7U4b3//5Lic0NV03gXT8O/isj5Nmp6KiY0d45pIQ==" + }, + "node_modules/it-length-prefixed": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/it-length-prefixed/-/it-length-prefixed-9.1.1.tgz", + "integrity": "sha512-O88nBweT6M9ozsmok68/auKH7ik/slNM4pYbM9lrfy2z5QnpokW5SlrepHZDKtN71llhG2sZvd6uY4SAl+lAQg==", + "dependencies": { + "it-reader": "^6.0.1", + "it-stream-types": "^2.0.1", + "uint8-varint": "^2.0.1", + "uint8arraylist": "^2.0.0", + "uint8arrays": "^5.0.1" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/it-length-prefixed/node_modules/it-reader": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/it-reader/-/it-reader-6.0.4.tgz", + "integrity": "sha512-XCWifEcNFFjjBHtor4Sfaj8rcpt+FkY0L6WdhD578SCDhV4VUm7fCkF3dv5a+fTcfQqvN9BsxBTvWbYO6iCjTg==", + "dependencies": { + "it-stream-types": "^2.0.1", + "uint8arraylist": "^2.0.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/it-length-prefixed/node_modules/uint8-varint": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/uint8-varint/-/uint8-varint-2.0.4.tgz", + "integrity": "sha512-FwpTa7ZGA/f/EssWAb5/YV6pHgVF1fViKdW8cWaEarjB8t7NyofSWBdOTyFPaGuUG4gx3v1O3PQ8etsiOs3lcw==", + "dependencies": { + "uint8arraylist": "^2.0.0", + "uint8arrays": "^5.0.0" + } + }, + "node_modules/it-pipe": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/it-pipe/-/it-pipe-3.0.1.tgz", + "integrity": "sha512-sIoNrQl1qSRg2seYSBH/3QxWhJFn9PKYvOf/bHdtCBF0bnghey44VyASsWzn5dAx0DCDDABq1hZIuzKmtBZmKA==", + "dependencies": { + "it-merge": "^3.0.0", "it-pushable": "^3.1.2", "it-stream-types": "^2.0.1" }, @@ -22679,11 +22823,6 @@ "uint8arrays": "^5.0.2" }, "dependencies": { - "@assemblyscript/loader": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.9.4.tgz", - "integrity": "sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA==" - }, "@helia/interface": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@helia/interface/-/interface-5.1.0.tgz", @@ -22783,16 +22922,6 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, - "bl": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", - "integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==", - "requires": { - "buffer": "^6.0.3", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "braces": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", @@ -22875,11 +23004,6 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, "interface-blockstore": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/interface-blockstore/-/interface-blockstore-5.3.1.tgz", @@ -22926,28 +23050,6 @@ "progress-events": "^1.0.1" } }, - "ipfs-unixfs-importer": { - "version": "15.3.1", - "resolved": "https://registry.npmjs.org/ipfs-unixfs-importer/-/ipfs-unixfs-importer-15.3.1.tgz", - "integrity": "sha512-wHCTBqNsZXLJZ9/GSr7Msb3FDXD5yXF20Y9sKyUbbqNjbvaXs3n3h1+NM/5+WrgESHfwRcJIlJtaOKafL8Ymdg==", - "requires": { - "@ipld/dag-pb": "^4.1.2", - "@multiformats/murmur3": "^2.1.8", - "hamt-sharding": "^3.0.6", - "interface-blockstore": "^5.3.0", - "interface-store": "^6.0.0", - "ipfs-unixfs": "^11.0.0", - "it-all": "^3.0.6", - "it-batch": "^3.0.6", - "it-first": "^3.0.6", - "it-parallel-batch": "^3.0.6", - "multiformats": "^13.2.3", - "progress-events": "^1.0.1", - "rabin-wasm": "^0.1.5", - "uint8arraylist": "^2.4.8", - "uint8arrays": "^5.1.0" - } - }, "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -22976,11 +23078,6 @@ "resolved": "https://registry.npmjs.org/it-all/-/it-all-3.0.6.tgz", "integrity": "sha512-HXZWbxCgQZJfrv5rXvaVeaayXED8nTKx9tj9fpBhmcUJcedVZshMMMqTj0RG2+scGypb9Ut1zd1ifbf3lA8L+Q==" }, - "it-batch": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/it-batch/-/it-batch-3.0.6.tgz", - "integrity": "sha512-pQAAlSvJ4aV6xM/6LRvkPdKSKXxS4my2fGzNUxJyAQ8ccFdxPmK1bUuF5OoeUDkcdrbs8jtsmc4DypCMrGY6sg==" - }, "it-filter": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/it-filter/-/it-filter-3.1.1.tgz", @@ -23018,14 +23115,6 @@ "p-defer": "^4.0.1" } }, - "it-parallel-batch": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/it-parallel-batch/-/it-parallel-batch-3.0.6.tgz", - "integrity": "sha512-3wgiQGvMMHy65OXScrtrtmY+bJSF7P6St1AP+BU+SK83fEr8NNk/MrmJKrtB1+MahYX2a8I+pOGKDj8qVtuV0Q==", - "requires": { - "it-batch": "^3.0.0" - } - }, "it-peekable": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/it-peekable/-/it-peekable-3.0.5.tgz", @@ -23053,11 +23142,6 @@ "picomatch": "^2.3.1" } }, - "minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" - }, "ms": { "version": "3.0.0-canary.1", "resolved": "https://registry.npmjs.org/ms/-/ms-3.0.0-canary.1.tgz", @@ -23068,14 +23152,6 @@ "resolved": "https://registry.npmjs.org/murmurhash3js-revisited/-/murmurhash3js-revisited-3.0.0.tgz", "integrity": "sha512-/sF3ee6zvScXMb1XFJ8gDsSnY+X8PbOyjIuBhtgis10W2Jx4ZjIhikUCIF9c4gpJxVnQIsPAFrSwTCuAjicP6g==" }, - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, "p-queue": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz", @@ -23110,55 +23186,19 @@ "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" }, - "rabin-wasm": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/rabin-wasm/-/rabin-wasm-0.1.5.tgz", - "integrity": "sha512-uWgQTo7pim1Rnj5TuWcCewRDTf0PEFTSlaUjWP4eY9EbLV9em08v89oCz/WO+wRxpYuO36XEHp4wgYQnAgOHzA==", - "requires": { - "@assemblyscript/loader": "^0.9.4", - "bl": "^5.0.0", - "debug": "^4.3.1", - "minimist": "^1.2.5", - "node-fetch": "^2.6.1", - "readable-stream": "^3.6.0" - } - }, - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, "run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", "requires": { "queue-microtask": "^1.2.2" - } - }, - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + } }, "sparse-array": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/sparse-array/-/sparse-array-1.3.2.tgz", "integrity": "sha512-ZT711fePGn3+kQyLuv1fpd3rNSkNF8vd5Kv2D+qnOANeyKs3fx6bUMGWRPvgTTcYV64QMqZKZwcuaQSP3AZ0tg==" }, - "string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "requires": { - "safe-buffer": "~5.2.0" - } - }, "supports-color": { "version": "9.4.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-9.4.0.tgz", @@ -23172,11 +23212,6 @@ "is-number": "^7.0.0" } }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "uint8-varint": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/uint8-varint/-/uint8-varint-2.0.4.tgz", @@ -23186,11 +23221,6 @@ "uint8arrays": "^5.0.0" } }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" - }, "weald": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/weald/-/weald-1.0.4.tgz", @@ -23199,20 +23229,6 @@ "ms": "^3.0.0-canary.1", "supports-color": "^9.4.0" } - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } } } }, @@ -32213,6 +32229,230 @@ } } }, + "ipfs-unixfs-importer": { + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/ipfs-unixfs-importer/-/ipfs-unixfs-importer-15.3.1.tgz", + "integrity": "sha512-wHCTBqNsZXLJZ9/GSr7Msb3FDXD5yXF20Y9sKyUbbqNjbvaXs3n3h1+NM/5+WrgESHfwRcJIlJtaOKafL8Ymdg==", + "requires": { + "@ipld/dag-pb": "^4.1.2", + "@multiformats/murmur3": "^2.1.8", + "hamt-sharding": "^3.0.6", + "interface-blockstore": "^5.3.0", + "interface-store": "^6.0.0", + "ipfs-unixfs": "^11.0.0", + "it-all": "^3.0.6", + "it-batch": "^3.0.6", + "it-first": "^3.0.6", + "it-parallel-batch": "^3.0.6", + "multiformats": "^13.2.3", + "progress-events": "^1.0.1", + "rabin-wasm": "^0.1.5", + "uint8arraylist": "^2.4.8", + "uint8arrays": "^5.1.0" + }, + "dependencies": { + "@assemblyscript/loader": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/@assemblyscript/loader/-/loader-0.9.4.tgz", + "integrity": "sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA==" + }, + "@multiformats/murmur3": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/@multiformats/murmur3/-/murmur3-2.1.8.tgz", + "integrity": "sha512-6vId1C46ra3R1sbJUOFCZnsUIveR9oF20yhPmAFxPm0JfrX3/ZRCgP3YDrBzlGoEppOXnA9czHeYc0T9mB6hbA==", + "requires": { + "multiformats": "^13.0.0", + "murmurhash3js-revisited": "^3.0.0" + } + }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, + "bl": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", + "integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==", + "requires": { + "buffer": "^6.0.3", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "hamt-sharding": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/hamt-sharding/-/hamt-sharding-3.0.6.tgz", + "integrity": "sha512-nZeamxfymIWLpVcAN0CRrb7uVq3hCOGj9IcL6NMA6VVCVWqj+h9Jo/SmaWuS92AEDf1thmHsM5D5c70hM3j2Tg==", + "requires": { + "sparse-array": "^1.3.1", + "uint8arrays": "^5.0.1" + } + }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "interface-blockstore": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/interface-blockstore/-/interface-blockstore-5.3.1.tgz", + "integrity": "sha512-nhgrQnz6yUQEqxTFLhlOBurQOy5lWlwCpgFmZ3GTObTVTQS9RZjK/JTozY6ty9uz2lZs7VFJSqwjWAltorJ4Vw==", + "requires": { + "interface-store": "^6.0.0", + "multiformats": "^13.2.3" + } + }, + "interface-store": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/interface-store/-/interface-store-6.0.2.tgz", + "integrity": "sha512-KSFCXtBlNoG0hzwNa0RmhHtrdhzexp+S+UY2s0rWTBJyfdEIgn6i6Zl9otVqrcFYbYrneBT7hbmHQ8gE0C3umA==" + }, + "ipfs-unixfs": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/ipfs-unixfs/-/ipfs-unixfs-11.2.0.tgz", + "integrity": "sha512-J8FN1qM5nfrDo8sQKQwfj0+brTg1uBfZK2vY9hxci33lcl3BFrsELS9+1+4q/8tO1ASKfxZO8W3Pi2O4sVX2Lg==", + "requires": { + "protons-runtime": "^5.5.0", + "uint8arraylist": "^2.4.8" + } + }, + "it-all": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/it-all/-/it-all-3.0.6.tgz", + "integrity": "sha512-HXZWbxCgQZJfrv5rXvaVeaayXED8nTKx9tj9fpBhmcUJcedVZshMMMqTj0RG2+scGypb9Ut1zd1ifbf3lA8L+Q==" + }, + "it-batch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/it-batch/-/it-batch-3.0.6.tgz", + "integrity": "sha512-pQAAlSvJ4aV6xM/6LRvkPdKSKXxS4my2fGzNUxJyAQ8ccFdxPmK1bUuF5OoeUDkcdrbs8jtsmc4DypCMrGY6sg==" + }, + "it-parallel-batch": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/it-parallel-batch/-/it-parallel-batch-3.0.6.tgz", + "integrity": "sha512-3wgiQGvMMHy65OXScrtrtmY+bJSF7P6St1AP+BU+SK83fEr8NNk/MrmJKrtB1+MahYX2a8I+pOGKDj8qVtuV0Q==", + "requires": { + "it-batch": "^3.0.0" + } + }, + "minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" + }, + "murmurhash3js-revisited": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/murmurhash3js-revisited/-/murmurhash3js-revisited-3.0.0.tgz", + "integrity": "sha512-/sF3ee6zvScXMb1XFJ8gDsSnY+X8PbOyjIuBhtgis10W2Jx4ZjIhikUCIF9c4gpJxVnQIsPAFrSwTCuAjicP6g==" + }, + "node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "protons-runtime": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/protons-runtime/-/protons-runtime-5.5.0.tgz", + "integrity": "sha512-EsALjF9QsrEk6gbCx3lmfHxVN0ah7nG3cY7GySD4xf4g8cr7g543zB88Foh897Sr1RQJ9yDCUsoT1i1H/cVUFA==", + "requires": { + "uint8-varint": "^2.0.2", + "uint8arraylist": "^2.4.3", + "uint8arrays": "^5.0.1" + } + }, + "rabin-wasm": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/rabin-wasm/-/rabin-wasm-0.1.5.tgz", + "integrity": "sha512-uWgQTo7pim1Rnj5TuWcCewRDTf0PEFTSlaUjWP4eY9EbLV9em08v89oCz/WO+wRxpYuO36XEHp4wgYQnAgOHzA==", + "requires": { + "@assemblyscript/loader": "^0.9.4", + "bl": "^5.0.0", + "debug": "^4.3.1", + "minimist": "^1.2.5", + "node-fetch": "^2.6.1", + "readable-stream": "^3.6.0" + } + }, + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "sparse-array": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/sparse-array/-/sparse-array-1.3.2.tgz", + "integrity": "sha512-ZT711fePGn3+kQyLuv1fpd3rNSkNF8vd5Kv2D+qnOANeyKs3fx6bUMGWRPvgTTcYV64QMqZKZwcuaQSP3AZ0tg==" + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "uint8-varint": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/uint8-varint/-/uint8-varint-2.0.4.tgz", + "integrity": "sha512-FwpTa7ZGA/f/EssWAb5/YV6pHgVF1fViKdW8cWaEarjB8t7NyofSWBdOTyFPaGuUG4gx3v1O3PQ8etsiOs3lcw==", + "requires": { + "uint8arraylist": "^2.0.0", + "uint8arrays": "^5.0.0" + } + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } + }, "it-drain": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/it-drain/-/it-drain-3.0.7.tgz", diff --git a/packages/backend/package.json b/packages/backend/package.json index c792b3f71..cb6c27bb4 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -152,6 +152,7 @@ "http-server": "^0.12.3", "https-proxy-agent": "^7.0.5", "image-size": "^1.0.1", + "ipfs-unixfs-importer": "15.3.1", "interface-datastore": "8.3.1", "it-length-prefixed": "^9.1.0", "it-pushable": "^3.2.3", diff --git a/packages/backend/src/nest/ipfs-file-manager/unixfs-utils/oneToOneChunker.ts b/packages/backend/src/nest/ipfs-file-manager/unixfs-utils/oneToOneChunker.ts new file mode 100644 index 000000000..fc42a9424 --- /dev/null +++ b/packages/backend/src/nest/ipfs-file-manager/unixfs-utils/oneToOneChunker.ts @@ -0,0 +1,36 @@ +/** + * This is an implementation of the UnixFS Chunker interface that writes chunks of a stream/iterable + * to blocks one-to-one. + * + * NOTE: In UnixFS the chunker implementation used determines how bytes from a stream are written to the blockstore + * with the default implementation writing blocks of a fixed size. This results in UnixFS chunks having more than one + * byte chunk from the stream grouped together like so: + * + * Byte chunk 1: + * <0 1 2 3 4 5> + * + * Byte chunk 2: + * <6 7 8 9 10 11> + * + * Byte chunk 3: + * <12 13 14 15 16 17> + * + * UnixFS Chunk 1: + * <0 1 2 3 4 5 6 7 8> + * + * UnixFS Chunk 2: + * <9 10 11 12 13 14 15 16 17> + * + * In the case of file encryption we want to keep distinct encrypted chunks of the byte stream separate from other encrypted + * chunks so they can be consumed by the decrypt stream. + */ + +import type { Chunker } from 'ipfs-unixfs-importer/chunker' + +export const oneToOne = (): Chunker => { + return async function* oneToOneChunker(source) { + for await (const buffer of source) { + yield buffer + } + } +} From 1e3fa179843ec6c9663087181ba712aaac63bd94 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Tue, 11 Feb 2025 23:08:16 -0500 Subject: [PATCH 04/14] Add file encryption --- .../ipfs-file-manager.const.ts | 7 +- .../ipfs-file-manager.module.ts | 3 +- .../ipfs-file-manager.service.ts | 111 ++++++++++++++---- .../backend/src/nest/validation/validators.ts | 12 ++ packages/types/src/files.ts | 23 +++- 5 files changed, 125 insertions(+), 31 deletions(-) diff --git a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.const.ts b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.const.ts index 55182f665..9ebd814ef 100644 --- a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.const.ts +++ b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.const.ts @@ -5,6 +5,7 @@ export const UPDATE_STATUS_INTERVAL_MS = 1_000 // 1 second // Not sure if this is safe enough, nodes with CID data usually contain at most around 270 hashes. export const MAX_EVENT_LISTENERS = 600 -// 1048576 is the number of bytes in a block uploaded via unixfs -// Reference: packages/backend/node_modules/@helia/unixfs/src/commands/add.ts -export const DEFAULT_CAT_BLOCK_CHUNK_SIZE = 1048576 * 10 +// The default chunk size written by unixfs is 262144 bytes when using the fixedSize chunker +// Reference: https://github.com/ipfs/js-ipfs-unixfs/blob/bf060cda444221225675663e2a760ef562437963/packages/ipfs-unixfs-importer/src/chunker/fixed-size.ts#L8 +export const UNIXFS_CHUNK_SIZE = 524288 +export const UNIXFS_CAT_CHUNK_SIZE = UNIXFS_CHUNK_SIZE * 25 // This determines how much we read when downloading blocks from peers diff --git a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.module.ts b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.module.ts index 43b68d8e3..0ed247829 100644 --- a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.module.ts +++ b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.module.ts @@ -1,9 +1,10 @@ import { Module } from '@nestjs/common' import { IpfsFileManagerService } from './ipfs-file-manager.service' import { IpfsModule } from '../ipfs/ipfs.module' +import { SigChainModule } from '../auth/sigchain.service.module' @Module({ - imports: [IpfsModule], + imports: [IpfsModule, SigChainModule], providers: [IpfsFileManagerService], exports: [IpfsFileManagerService], }) diff --git a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts index 61a41c484..0c59a1b64 100644 --- a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts +++ b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts @@ -1,6 +1,6 @@ import { Inject, Injectable } from '@nestjs/common' import { EventEmitter, setMaxListeners } from 'events' -import fs from 'fs' +import fs, { WriteStream } from 'fs' import path from 'path' import crypto from 'crypto' import { AddPinEvents, GetBlockProgressEvents, type Helia } from 'helia' @@ -8,7 +8,14 @@ import { AddEvents, CatOptions, GetEvents, StatOptions, unixfs, UnixFSStats, typ import { promisify } from 'util' import sizeOf from 'image-size' import { CID } from 'multiformats/cid' +import { DateTime } from 'luxon' +import { CustomProgressEvent } from 'progress-events' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' + +import { QuietLogger } from '@quiet/logger' import { DownloadProgress, DownloadState, DownloadStatus, FileMetadata, imagesExtensions } from '@quiet/types' + import { QUIET_DIR } from '../const' import { BlockStat, @@ -23,10 +30,11 @@ import { } from './ipfs-file-manager.types' import { StorageEvents, UnixFSEvents } from '../storage/storage.types' import { - DEFAULT_CAT_BLOCK_CHUNK_SIZE, MAX_EVENT_LISTENERS, TRANSFER_SPEED_SPAN, TRANSFER_SPEED_SPAN_MS, + UNIXFS_CAT_CHUNK_SIZE, + UNIXFS_CHUNK_SIZE, UPDATE_STATUS_INTERVAL_MS, } from './ipfs-file-manager.const' import { sleep } from '../common/sleep' @@ -34,10 +42,12 @@ const sizeOfPromisified = promisify(sizeOf) const { createPaths, compare } = await import('../common/utils') import { createLogger } from '../common/logger' import { IpfsService } from '../ipfs/ipfs.service' -import { CustomProgressEvent } from 'progress-events' -import { DateTime } from 'luxon' -import { QuietLogger } from '@quiet/logger' import { abortableAsyncIterable } from '../common/utils' +import { SigChainService } from '../auth/sigchain.service' +import { EncryptionScopeType } from '../auth/services/crypto/types' +import { SigChain } from '../auth/sigchain' +import { RoleName } from '../auth/services/roles/roles' +import { oneToOne } from './unixfs-utils/oneToOneChunker' @Injectable() export class IpfsFileManagerService extends EventEmitter { @@ -55,6 +65,7 @@ export class IpfsFileManagerService extends EventEmitter { private readonly logger = createLogger(IpfsFileManagerService.name) constructor( @Inject(QUIET_DIR) public readonly quietDir: string, + private readonly sigChainService: SigChainService, private readonly ipfsService: IpfsService ) { super() @@ -185,11 +196,17 @@ export class IpfsFileManagerService extends EventEmitter { public async uploadFile(metadata: FileMetadata) { const _logger = createLogger(`${IpfsFileManagerService.name}:upload`) + const sigChain = this.sigChainService.getActiveChain() + if (sigChain == null) { + throw new Error(`Can't upload file because there was no active sigchain`) + } + let width: number | undefined let height: number | undefined if (!metadata.path) { throw new Error(`File metadata (cid ${metadata.cid}) does not contain path`) } + if (imagesExtensions.includes(metadata.ext)) { let imageSize: { width: number | undefined; height: number | undefined } | undefined // ISizeCalculationResult try { @@ -212,31 +229,46 @@ export class IpfsFileManagerService extends EventEmitter { const filename = `${uuid}_${metadata.name}${metadata.ext}` // Save copy to separate directory + _logger.info(`Copying ${filename} to uploads directory`) const filePath = this.copyFile(metadata.path, filename) + _logger.time(`Writing ${filename} to ipfs`) const handleUploadProgressEvents = (event: AddEvents): void => { _logger.info(`Upload progress`, event) } - const stream = fs.createReadStream(filePath, { highWaterMark: 64 * 1024 * 10 }) + const stream = fs.createReadStream(filePath, { highWaterMark: UNIXFS_CHUNK_SIZE }) const uploadedFileStreamIterable = { // eslint-disable-next-line prettier/prettier, generator-star-spacing async *[Symbol.asyncIterator]() { for await (const data of stream) { + _logger.info(`Streaming ${(data as Buffer).byteLength} bytes from ${filename}`) yield data } }, } - const fileCid = await this.ufs.addByteStream(uploadedFileStreamIterable, { + const { header, recipient, encryptStream } = sigChain.crypto.encryptStream(uploadedFileStreamIterable, { + type: EncryptionScopeType.ROLE, + name: RoleName.MEMBER, + }) + + const fileCid = await this.ufs.addByteStream(encryptStream, { wrapWithDirectory: true, onProgress: handleUploadProgressEvents, + leafType: 'raw', + rawLeaves: true, + reduceSingleLeafToSelf: true, + // NOTE: this is what makes file encryption possible because it ensures that the encrypted chunks generated by the encrypt stream method + // in LFA are written as individual blocks vs multiple chunks per block with partial chunks written to a given block + chunker: oneToOne(), }) _logger.timeEnd(`Writing ${filename} to ipfs`) this.emit(StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: metadata.cid }) + const fileMetadata: FileMetadata = { ...metadata, tmpPath: undefined, @@ -245,6 +277,10 @@ export class IpfsFileManagerService extends EventEmitter { size: Number((await this.ufs.stat(fileCid)).fileSize), width, height, + enc: { + header: uint8ArrayToString(header, 'base64url'), + recipient, + }, } this.emit(StorageEvents.FILE_UPLOADED, fileMetadata) @@ -366,12 +402,11 @@ export class IpfsFileManagerService extends EventEmitter { // handler for events where we have found the block on the network and are adding it to our local blockstore const handleDownloadBlock = async (event: CustomProgressEvent) => { - const { bytesRead, totalBytes } = event.detail _logger.info(`Block found and downloaded to local blockstore`, event.detail) const blockStat = { fetchTimeMs: DateTime.utc().toMillis(), - byteLength: Number(totalBytes) - Number(bytesRead), + byteLength: UNIXFS_CHUNK_SIZE, } blocksStats.push(blockStat) downloadedBlocks += 1 @@ -516,7 +551,7 @@ export class IpfsFileManagerService extends EventEmitter { return DownloadState.Canceled } - const finishedWriting = await this.writeBlocksToFilesystem(fileCid, writeStream, { + const finishedWriting = await this.writeBlocksToFilesystem(fileCid, fileMetadata, writeStream, { logger: _logger, signal: controller.signal, catOptions: baseCatOptions, @@ -723,11 +758,11 @@ export class IpfsFileManagerService extends EventEmitter { const catOptions: CatOptions = { ...options.catOptions, offset: downloadedSize, - length: DEFAULT_CAT_BLOCK_CHUNK_SIZE, + length: UNIXFS_CAT_CHUNK_SIZE, } options.logger.info( - `Getting blocks totalling ${DEFAULT_CAT_BLOCK_CHUNK_SIZE} bytes with offset ${downloadedSize} (total bytes: ${totalSize})` + `Getting blocks totalling ${UNIXFS_CAT_CHUNK_SIZE} bytes with offset ${downloadedSize} (total bytes: ${totalSize})` ) try { @@ -765,7 +800,7 @@ export class IpfsFileManagerService extends EventEmitter { await sleep(500) continue } - offset += DEFAULT_CAT_BLOCK_CHUNK_SIZE + offset += UNIXFS_CAT_CHUNK_SIZE } return true @@ -773,6 +808,7 @@ export class IpfsFileManagerService extends EventEmitter { private async writeBlocksToFilesystem( cid: CID, + fileMetadata: FileMetadata, writeStream: fs.WriteStream, options: GetBlocksOptions ): Promise { @@ -782,6 +818,12 @@ export class IpfsFileManagerService extends EventEmitter { return false } + const sigChain = this.sigChainService.getActiveChain() + if (sigChain == null) { + options.logger.error(`Cancelling download because initial stat check threw an error`) + return false + } + try { const entries = await this.getBlocks( cid, @@ -802,19 +844,7 @@ export class IpfsFileManagerService extends EventEmitter { return false } - for await (const entry of entries) { - options.logger.info(`Writing block with size (in bytes)`, entry.byteLength) - - await new Promise((resolve, reject) => { - writeStream.write(entry, err => { - if (err) { - this.logger.error(`${cid.toString()} writing to file error`, err) - reject(err) - } - }) - resolve() - }) - } + await this._decryptBlockAndWriteToFile(cid, fileMetadata, entries, sigChain, writeStream, options.logger) } catch (e) { if (options.signal?.aborted) { options.logger.warn(`Cancelling download while writing block data to filesystem`, e) @@ -827,6 +857,35 @@ export class IpfsFileManagerService extends EventEmitter { return true } + private async _decryptBlockAndWriteToFile( + cid: CID, + fileMetadata: FileMetadata, + catStream: AsyncIterable, + sigChain: SigChain, + writeStream: WriteStream, + _logger: QuietLogger + ) { + _logger.info(`Decrypting and writing blocks to file`) + if (fileMetadata.enc == null) { + throw new Error(`Must include encryption metadata`) + } + + const { header, recipient } = fileMetadata.enc + const decryptStream = sigChain.crypto.decryptStream(catStream, uint8ArrayFromString(header, 'base64url'), recipient) + for await (const decryptedEntry of decryptStream) { + _logger.info(`Writing block with size (in bytes)`, decryptedEntry.byteLength) + await new Promise((resolve, reject) => { + writeStream.write(decryptedEntry, err => { + if (err) { + this.logger.error(`${cid.toString()} writing to file error`, err) + reject(err) + } + }) + resolve() + }) + } + } + private async validateDownload( cid: CID, metadataSize: number | undefined, diff --git a/packages/backend/src/nest/validation/validators.ts b/packages/backend/src/nest/validation/validators.ts index b9c26d3a7..a89b7fb6c 100644 --- a/packages/backend/src/nest/validation/validators.ts +++ b/packages/backend/src/nest/validation/validators.ts @@ -20,6 +20,18 @@ const messageMediaSchema = joi.object({ channelId: joi.string(), channelAddress: joi.string(), }), + enc: joi + .object({ + header: joi.binary().required(), + recipient: joi + .object({ + generation: joi.number().required(), + type: joi.string().required(), + name: joi.string().allow(null), + }) + .required(), + }) + .allow(null), }) const messageSchema = joi.object({ diff --git a/packages/types/src/files.ts b/packages/types/src/files.ts index 3e3d83bd8..00d435450 100644 --- a/packages/types/src/files.ts +++ b/packages/types/src/files.ts @@ -17,6 +17,14 @@ export interface FileMetadata extends FileContent { size?: number width?: number height?: number + enc?: { + header: string + recipient: { + generation: number + type: string + name: string + } + } } export interface UploadFilePayload { @@ -83,4 +91,17 @@ export enum DownloadState { Malicious = 'malicious', } -export const imagesExtensions = ['.gif', '.png', '.jpg', '.jpeg'] +export const imagesExtensions = [ + '.gif', + '.png', + '.apng', + '.jpg', + '.jpeg', + '.svg', + '.avif', + '.webp', + '.bmp', + '.ico', + '.tif', + '.tiff', +] From 1bda86de4d41b206d99a6e36b39892232c0691b4 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Wed, 12 Feb 2025 09:04:53 -0500 Subject: [PATCH 05/14] Fix (most) backend tests --- .../ipfs-file-manager/big-files.long.spec.ts | 8 +- .../ipfs-file-manager.service.spec.ts | 127 +++++++++++++----- .../storage/channels/channels.service.spec.ts | 28 ++-- .../messages/messages.service.spec.ts | 3 +- 4 files changed, 119 insertions(+), 47 deletions(-) diff --git a/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts b/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts index 947101e95..0017eb704 100644 --- a/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts +++ b/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts @@ -16,6 +16,8 @@ import { IpfsFileManagerModule } from './ipfs-file-manager.module' import { IpfsFileManagerService } from './ipfs-file-manager.service' import fs from 'fs' import { createLogger } from '../common/logger' +import { SigChainService } from '../auth/sigchain.service' +import { SigChainModule } from '../auth/sigchain.service.module' const logger = createLogger('bigFiles:test') const BIG_FILE_SIZE = 2147483000 @@ -25,6 +27,7 @@ describe('IpfsFileManagerService', () => { let ipfsFileManagerService: IpfsFileManagerService let ipfsService: IpfsService let libp2pService: Libp2pService + let sigChainService: SigChainService let tmpDir: DirResult let filePath: string @@ -35,9 +38,12 @@ describe('IpfsFileManagerService', () => { // Generate 2.1GB file await createArbitraryFile(filePath, BIG_FILE_SIZE) module = await Test.createTestingModule({ - imports: [TestModule, IpfsFileManagerModule, IpfsModule, SocketModule, Libp2pModule], + imports: [TestModule, IpfsFileManagerModule, IpfsModule, SocketModule, Libp2pModule, SigChainModule], }).compile() + sigChainService = await module.resolve(SigChainService) + await sigChainService.createChain('community', 'username', true) + ipfsFileManagerService = await module.resolve(IpfsFileManagerService) libp2pService = await module.resolve(Libp2pService) diff --git a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.spec.ts b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.spec.ts index 37593f1a4..59119f806 100644 --- a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.spec.ts +++ b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.spec.ts @@ -21,6 +21,10 @@ import { IpfsFilesManagerEvents } from './ipfs-file-manager.types' import { sleep } from '../common/sleep' import { LocalDbModule } from '../local-db/local-db.module' import { LocalDbService } from '../local-db/local-db.service' +import { SigChainModule } from '../auth/sigchain.service.module' +import { SigChainService } from '../auth/sigchain.service' +import { EncryptionScopeType } from '../auth/services/crypto/types' +import { RoleName } from '../auth/services/roles/roles' const filename = fileURLToPath(import.meta.url) const dirname = path.dirname(filename) @@ -31,6 +35,7 @@ describe('IpfsFileManagerService', () => { let localDbService: LocalDbService let ipfsService: IpfsService let libp2pService: Libp2pService + let sigChainService: SigChainService let tmpDir: DirResult let filePath: string @@ -41,9 +46,20 @@ describe('IpfsFileManagerService', () => { filePath = path.join(dirname, '/testUtils/500kB-file.txt') module = await Test.createTestingModule({ - imports: [TestModule, IpfsFileManagerModule, IpfsModule, SocketModule, Libp2pModule, LocalDbModule], + imports: [ + TestModule, + IpfsFileManagerModule, + IpfsModule, + SocketModule, + Libp2pModule, + LocalDbModule, + SigChainModule, + ], }).compile() + sigChainService = await module.resolve(SigChainService) + await sigChainService.createChain('community', 'username', true) + ipfsFileManagerService = await module.resolve(IpfsFileManagerService) localDbService = await module.resolve(LocalDbService) @@ -115,19 +131,27 @@ describe('IpfsFileManagerService', () => { 2, StorageEvents.FILE_UPLOADED, expect.objectContaining({ - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), ext: '.png', height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15847, + size: 15886, width: 824, + enc: { + header: expect.any(String), + recipient: { + generation: 0, + type: EncryptionScopeType.ROLE, + name: RoleName.MEMBER, + }, + }, }) ) }) await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.DOWNLOAD_PROGRESS, { - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), downloadProgress: undefined, downloadState: 'hosted', mid: 'id', @@ -159,19 +183,27 @@ describe('IpfsFileManagerService', () => { 2, StorageEvents.FILE_UPLOADED, expect.objectContaining({ - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), ext: '.pdf', height: undefined, message: { channelId: 'channelId', id: 'id' }, name: 'test-file', - size: 761797, + size: 761860, width: undefined, + enc: { + header: expect.any(String), + recipient: { + generation: 0, + type: EncryptionScopeType.ROLE, + name: RoleName.MEMBER, + }, + }, }) ) }) await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.DOWNLOAD_PROGRESS, { - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), downloadProgress: undefined, downloadState: 'hosted', mid: 'id', @@ -182,12 +214,12 @@ describe('IpfsFileManagerService', () => { 4, StorageEvents.MESSAGE_MEDIA_UPDATED, expect.objectContaining({ - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), ext: '.pdf', height: undefined, message: { channelId: 'channelId', id: 'id' }, name: 'test-file', - size: 761797, + size: 761860, width: undefined, }) ) @@ -229,20 +261,28 @@ describe('IpfsFileManagerService', () => { 2, StorageEvents.FILE_UPLOADED, expect.objectContaining({ - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), ext: '.png', height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15847, + size: 15886, width: 824, tmpPath: undefined, + enc: { + header: expect.any(String), + recipient: { + generation: 0, + type: EncryptionScopeType.ROLE, + name: RoleName.MEMBER, + }, + }, }) ) }) await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.DOWNLOAD_PROGRESS, { - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), downloadProgress: undefined, downloadState: 'hosted', mid: 'id', @@ -298,19 +338,27 @@ describe('IpfsFileManagerService', () => { 2, StorageEvents.FILE_UPLOADED, expect.objectContaining({ - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), ext: '.pdf', height: undefined, message: { channelId: 'channelId', id: 'id' }, name: 'test-file', - size: 761797, + size: 761860, width: undefined, + enc: { + header: expect.any(String), + recipient: { + generation: 0, + type: EncryptionScopeType.ROLE, + name: RoleName.MEMBER, + }, + }, }) ) }) await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.DOWNLOAD_PROGRESS, { - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), downloadProgress: undefined, downloadState: 'hosted', mid: 'id', @@ -321,12 +369,12 @@ describe('IpfsFileManagerService', () => { 4, StorageEvents.MESSAGE_MEDIA_UPDATED, expect.objectContaining({ - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), ext: '.pdf', height: undefined, message: { channelId: 'channelId', id: 'id' }, name: 'test-file', - size: 761797, + size: 761860, width: undefined, }) ) @@ -350,7 +398,7 @@ describe('IpfsFileManagerService', () => { await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.DOWNLOAD_PROGRESS, { - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), downloadProgress: undefined, downloadState: 'malicious', mid: 'id', @@ -375,8 +423,6 @@ describe('IpfsFileManagerService', () => { }, } - const imageCid = 'bafkreigemnq7fljgbxdqjhq5nhj5pprt4qkvyl7vcymbnucc5azkxms4v4' - await ipfsFileManagerService.uploadFile(metadata) await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(1, StorageEvents.REMOVE_DOWNLOAD_STATUS, { cid: 'uploading_id' }) @@ -387,20 +433,28 @@ describe('IpfsFileManagerService', () => { 2, StorageEvents.FILE_UPLOADED, expect.objectContaining({ - cid: imageCid, + cid: expect.stringContaining('bafy'), ext: '.png', height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15847, + size: 15886, width: 824, + enc: { + header: expect.any(String), + recipient: { + generation: 0, + type: EncryptionScopeType.ROLE, + name: RoleName.MEMBER, + }, + }, }) ) }, 10_000) await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(3, StorageEvents.DOWNLOAD_PROGRESS, { - cid: imageCid, + cid: expect.stringContaining('bafy'), downloadProgress: undefined, downloadState: 'hosted', mid: 'id', @@ -412,12 +466,12 @@ describe('IpfsFileManagerService', () => { 4, StorageEvents.MESSAGE_MEDIA_UPDATED, expect.objectContaining({ - cid: imageCid, + cid: expect.stringContaining('bafy'), ext: '.png', height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15847, + size: 15886, width: 824, path: expect.stringContaining('_test-image.png'), }) @@ -434,17 +488,26 @@ describe('IpfsFileManagerService', () => { expect(eventSpy).toHaveBeenNthCalledWith(5, IpfsFilesManagerEvents.DOWNLOAD_FILE, uploadMetadata) }, 10_000) + await waitForExpect(() => { + expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.DOWNLOAD_PROGRESS, { + cid: expect.stringContaining('bafy'), + downloadProgress: { downloaded: 15886, size: 15886, transferSpeed: 0 }, + downloadState: 'downloading', + mid: 'id', + }) + }, 20_000) + await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith( - 6, + 7, StorageEvents.MESSAGE_MEDIA_UPDATED, expect.objectContaining({ - cid: expect.stringContaining('bafk'), + cid: expect.stringContaining('bafy'), ext: '.png', height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15847, + size: 15886, width: 824, path: expect.stringContaining('.png'), }) @@ -452,15 +515,15 @@ describe('IpfsFileManagerService', () => { }, 20_000) await waitForExpect(() => { - expect(eventSpy).toHaveBeenNthCalledWith(7, StorageEvents.DOWNLOAD_PROGRESS, { - cid: expect.stringContaining('bafk'), - downloadProgress: { downloaded: 15847, size: 15847, transferSpeed: 0 }, + expect(eventSpy).toHaveBeenNthCalledWith(8, StorageEvents.DOWNLOAD_PROGRESS, { + cid: expect.stringContaining('bafy'), + downloadProgress: { downloaded: 15886, size: 15886, transferSpeed: 0 }, downloadState: 'completed', mid: 'id', }) }, 20_000) - expect(eventSpy).toBeCalledTimes(7) + expect(eventSpy).toBeCalledTimes(8) }) // this case causes other tests to fail diff --git a/packages/backend/src/nest/storage/channels/channels.service.spec.ts b/packages/backend/src/nest/storage/channels/channels.service.spec.ts index cfc1b6d05..b521121c1 100644 --- a/packages/backend/src/nest/storage/channels/channels.service.spec.ts +++ b/packages/backend/src/nest/storage/channels/channels.service.spec.ts @@ -179,19 +179,21 @@ describe('ChannelsService', () => { expect(eventSpy).toHaveBeenCalled() const savedMessages = await channelsService.getMessages(channelio.id) expect(savedMessages?.messages.length).toBe(1) - expect(savedMessages?.messages[0]).toEqual({ - ...messageCopy, - verified: true, - encSignature: expect.objectContaining({ - author: { - generation: 0, - type: 'USER', - name: sigChainService.getActiveChain().localUserContext.user.userId, - }, - contents: expect.any(String), - signature: expect.any(String), - }), - }) + expect(savedMessages?.messages[0]).toEqual( + expect.objectContaining({ + ...messageCopy, + verified: true, + encSignature: expect.objectContaining({ + author: { + generation: 0, + type: 'USER', + name: sigChainService.getActiveChain().localUserContext.user.userId, + }, + contents: expect.any(Uint8Array), + signature: expect.any(String), + }), + }) + ) }) // TODO: figure out a good way to spoof the signature diff --git a/packages/backend/src/nest/storage/channels/messages/messages.service.spec.ts b/packages/backend/src/nest/storage/channels/messages/messages.service.spec.ts index 5d9dd31bd..694787d45 100644 --- a/packages/backend/src/nest/storage/channels/messages/messages.service.spec.ts +++ b/packages/backend/src/nest/storage/channels/messages/messages.service.spec.ts @@ -11,6 +11,7 @@ import { import { ChannelMessage, Community, Identity, PublicChannel, TestMessage } from '@quiet/types' import { isBase58 } from 'class-validator' import { FactoryGirl } from 'factory-girl' +import { isUint8Array } from 'util/types' import { EncryptionScopeType } from '../../../auth/services/crypto/types' import { RoleName } from '../../../auth/services/roles/roles' import { SigChainService } from '../../../auth/sigchain.service' @@ -104,7 +105,7 @@ describe('MessagesService', () => { }), }) ) - expect(isBase58(encryptedMessage.message.contents)).toBeTruthy() + expect(isUint8Array(encryptedMessage.message.contents)).toBeTruthy() }) }) From fbf86ff766e05bd4429403b50939ee6678e2a020 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Wed, 12 Feb 2025 09:19:14 -0500 Subject: [PATCH 06/14] Update .gitignore --- packages/backend/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/.gitignore b/packages/backend/.gitignore index 34ec76216..ffe428b2e 100644 --- a/packages/backend/.gitignore +++ b/packages/backend/.gitignore @@ -5,4 +5,4 @@ /testingFixtures/certificates/files2 /testingFixtures/certificates/files3 /src/nest/storage/testUtils/large-file* -/src/ipfs-file-manager/testUtils/large-file* \ No newline at end of file +/src/nest/ipfs-file-manager/testUtils/large-file* \ No newline at end of file From e4d382a1570e6bce439fcaa21a37b2183eeb83b1 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Tue, 11 Feb 2025 21:56:56 -0500 Subject: [PATCH 07/14] Log at start of timer, too --- packages/logger/src/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/logger/src/index.ts b/packages/logger/src/index.ts index 90e9bc9d8..388361fbe 100644 --- a/packages/logger/src/index.ts +++ b/packages/logger/src/index.ts @@ -194,6 +194,9 @@ export class QuietLogger { return } + const formattedLogStrings = this.formatLog(LogLevel.TIMER, name, `- timer started`) + this.printLog(LogLevel.LOG, ...formattedLogStrings) + const startMs = DateTime.utc().toMillis() this.timers.set(name, startMs) } From 63711f21e5cd8cf8fc711be65d3727e708c3cefe Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Tue, 11 Feb 2025 21:58:14 -0500 Subject: [PATCH 08/14] Tweak logs --- .../src/sagas/messages/sendMessage/sendMessage.saga.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/state-manager/src/sagas/messages/sendMessage/sendMessage.saga.ts b/packages/state-manager/src/sagas/messages/sendMessage/sendMessage.saga.ts index 0177b06fc..16dd0e4a5 100644 --- a/packages/state-manager/src/sagas/messages/sendMessage/sendMessage.saga.ts +++ b/packages/state-manager/src/sagas/messages/sendMessage/sendMessage.saga.ts @@ -82,7 +82,7 @@ export function* sendMessageSaga( const isUploadingFileMessage = action.payload.media?.cid?.includes('uploading') if (isUploadingFileMessage) { - logger.info(`Failed to send message ${id} - file upload is in progress`) + logger.info(`Waiting to send message ${id} - file upload is in progress`) return // Do not broadcast message until file is uploaded } @@ -98,7 +98,7 @@ export function* sendMessageSaga( logger.info(`Channel ${channelId} subscribed`) break } - logger.error(`Failed to send message ${id} - channel not subscribed. Waiting...`) + logger.error(`Waiting to send message ${id} - channel not subscribed`) yield* take(publicChannelsActions.setChannelSubscribed) } From e8ee5ec92831fdd127745022009539560db65f60 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 14 Feb 2025 11:37:46 -0500 Subject: [PATCH 09/14] Update branch on submodule --- .gitmodules | 1 + 3rd-party/js-libp2p-noise | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 82da16ad8..331a09bfe 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,7 @@ [submodule "3rd-party/auth"] path = 3rd-party/auth url = https://github.com/TryQuiet/auth.git + branch = feat/2744-stream-encryption [submodule "3rd-party/js-libp2p-noise"] path = 3rd-party/js-libp2p-noise url = https://github.com/TryQuiet/js-libp2p-noise.git diff --git a/3rd-party/js-libp2p-noise b/3rd-party/js-libp2p-noise index ae3800261..a0ba021a1 160000 --- a/3rd-party/js-libp2p-noise +++ b/3rd-party/js-libp2p-noise @@ -1 +1 @@ -Subproject commit ae3800261cb57603b2f61be7b64194da7d26839e +Subproject commit a0ba021a1c743e71c0ef4ac9b99487ae96d1050f From 975ef33fdd2928a3c2df5bd4045f76d4c043eb18 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 14 Feb 2025 11:40:05 -0500 Subject: [PATCH 10/14] Update action.yml --- .github/actions/setup-env/action.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 37a10855d..23bfcc0c1 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -63,9 +63,6 @@ runs: - name: "Build submodules" run: | - git submodule update --init --recursive - git submodule update --recursive --remote - npm run build:auth npm run build:noise shell: bash From c461c18dab9eb594588361eae8606d6b49b34d98 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 14 Feb 2025 12:05:34 -0500 Subject: [PATCH 11/14] Update ipfs-file-manager.service.spec.ts --- .../ipfs-file-manager.service.spec.ts | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.spec.ts b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.spec.ts index 59119f806..3a19a0e69 100644 --- a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.spec.ts +++ b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.spec.ts @@ -136,7 +136,7 @@ describe('IpfsFileManagerService', () => { height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15886, + size: 15881, width: 824, enc: { header: expect.any(String), @@ -188,7 +188,7 @@ describe('IpfsFileManagerService', () => { height: undefined, message: { channelId: 'channelId', id: 'id' }, name: 'test-file', - size: 761860, + size: 761848, width: undefined, enc: { header: expect.any(String), @@ -219,7 +219,7 @@ describe('IpfsFileManagerService', () => { height: undefined, message: { channelId: 'channelId', id: 'id' }, name: 'test-file', - size: 761860, + size: 761848, width: undefined, }) ) @@ -266,7 +266,7 @@ describe('IpfsFileManagerService', () => { height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15886, + size: 15881, width: 824, tmpPath: undefined, enc: { @@ -343,7 +343,7 @@ describe('IpfsFileManagerService', () => { height: undefined, message: { channelId: 'channelId', id: 'id' }, name: 'test-file', - size: 761860, + size: 761848, width: undefined, enc: { header: expect.any(String), @@ -374,8 +374,16 @@ describe('IpfsFileManagerService', () => { height: undefined, message: { channelId: 'channelId', id: 'id' }, name: 'test-file', - size: 761860, + size: 761848, width: undefined, + enc: { + header: expect.any(String), + recipient: { + generation: 0, + type: EncryptionScopeType.ROLE, + name: RoleName.MEMBER, + }, + }, }) ) }) @@ -438,7 +446,7 @@ describe('IpfsFileManagerService', () => { height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15886, + size: 15881, width: 824, enc: { header: expect.any(String), @@ -471,7 +479,7 @@ describe('IpfsFileManagerService', () => { height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15886, + size: 15881, width: 824, path: expect.stringContaining('_test-image.png'), }) @@ -491,7 +499,7 @@ describe('IpfsFileManagerService', () => { await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(6, StorageEvents.DOWNLOAD_PROGRESS, { cid: expect.stringContaining('bafy'), - downloadProgress: { downloaded: 15886, size: 15886, transferSpeed: 0 }, + downloadProgress: { downloaded: 15881, size: 15881, transferSpeed: 0 }, downloadState: 'downloading', mid: 'id', }) @@ -507,9 +515,17 @@ describe('IpfsFileManagerService', () => { height: 44, message: { channelId: 'channelId', id: 'id' }, name: 'test-image', - size: 15886, + size: 15881, width: 824, path: expect.stringContaining('.png'), + enc: { + header: expect.any(String), + recipient: { + generation: 0, + type: EncryptionScopeType.ROLE, + name: RoleName.MEMBER, + }, + }, }) ) }, 20_000) @@ -517,7 +533,7 @@ describe('IpfsFileManagerService', () => { await waitForExpect(() => { expect(eventSpy).toHaveBeenNthCalledWith(8, StorageEvents.DOWNLOAD_PROGRESS, { cid: expect.stringContaining('bafy'), - downloadProgress: { downloaded: 15886, size: 15886, transferSpeed: 0 }, + downloadProgress: { downloaded: 15881, size: 15881, transferSpeed: 0 }, downloadState: 'completed', mid: 'id', }) From 397c8cb361bccc6d3cec9c4f6246e1dca31d4f2b Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Fri, 14 Feb 2025 14:07:07 -0500 Subject: [PATCH 12/14] Fix test --- .../src/nest/ipfs-file-manager/big-files.long.spec.ts | 6 +++--- .../src/nest/ipfs-file-manager/ipfs-file-manager.service.ts | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts b/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts index 0017eb704..3c7120a02 100644 --- a/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts +++ b/packages/backend/src/nest/ipfs-file-manager/big-files.long.spec.ts @@ -20,7 +20,7 @@ import { SigChainService } from '../auth/sigchain.service' import { SigChainModule } from '../auth/sigchain.service.module' const logger = createLogger('bigFiles:test') -const BIG_FILE_SIZE = 2147483000 +const BIG_FILE_SIZE = 2097152000 describe('IpfsFileManagerService', () => { let module: TestingModule @@ -34,7 +34,7 @@ describe('IpfsFileManagerService', () => { beforeAll(async () => { tmpDir = createTmpDir() - filePath = new URL('./testUtils/large-file.txt', import.meta.url).pathname + filePath = new URL('./testUtils/large-file.bin', import.meta.url).pathname // Generate 2.1GB file await createArbitraryFile(filePath, BIG_FILE_SIZE) module = await Test.createTestingModule({ @@ -75,7 +75,7 @@ describe('IpfsFileManagerService', () => { const metadata: FileMetadata = { path: filePath, name: 'test-large-file', - ext: '.txt', + ext: '.bin', cid: 'uploading_id', message: { id: 'id', diff --git a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts index 0c59a1b64..bfa31cdc7 100644 --- a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts +++ b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts @@ -257,7 +257,6 @@ export class IpfsFileManagerService extends EventEmitter { const fileCid = await this.ufs.addByteStream(encryptStream, { wrapWithDirectory: true, onProgress: handleUploadProgressEvents, - leafType: 'raw', rawLeaves: true, reduceSingleLeafToSelf: true, // NOTE: this is what makes file encryption possible because it ensures that the encrypted chunks generated by the encrypt stream method From e37ad21b28bc5d81fe3acd9f436f6619f64c4752 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Mon, 17 Feb 2025 11:53:25 -0500 Subject: [PATCH 13/14] Point auth at main --- .gitmodules | 3 ++- 3rd-party/auth | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 331a09bfe..c15df40f7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,8 @@ [submodule "3rd-party/auth"] path = 3rd-party/auth url = https://github.com/TryQuiet/auth.git - branch = feat/2744-stream-encryption + branch = main [submodule "3rd-party/js-libp2p-noise"] path = 3rd-party/js-libp2p-noise url = https://github.com/TryQuiet/js-libp2p-noise.git + branch = master diff --git a/3rd-party/auth b/3rd-party/auth index 579ce1083..5fe7806c2 160000 --- a/3rd-party/auth +++ b/3rd-party/auth @@ -1 +1 @@ -Subproject commit 579ce1083313034264967fd842f3fd19e7f49ee5 +Subproject commit 5fe7806c2950b6b0688ffc292db52c700f3ce0e6 From 2f139472ffed5bb39778ee9ae48e05d0693786a6 Mon Sep 17 00:00:00 2001 From: Isla Koenigsknecht Date: Mon, 17 Feb 2025 14:32:19 -0500 Subject: [PATCH 14/14] Less spammy logs --- .../nest/ipfs-file-manager/ipfs-file-manager.service.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts index bfa31cdc7..cf573baf1 100644 --- a/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts +++ b/packages/backend/src/nest/ipfs-file-manager/ipfs-file-manager.service.ts @@ -235,7 +235,7 @@ export class IpfsFileManagerService extends EventEmitter { _logger.time(`Writing ${filename} to ipfs`) const handleUploadProgressEvents = (event: AddEvents): void => { - _logger.info(`Upload progress`, event) + _logger.trace(`Upload progress`, event) } const stream = fs.createReadStream(filePath, { highWaterMark: UNIXFS_CHUNK_SIZE }) @@ -243,7 +243,7 @@ export class IpfsFileManagerService extends EventEmitter { // eslint-disable-next-line prettier/prettier, generator-star-spacing async *[Symbol.asyncIterator]() { for await (const data of stream) { - _logger.info(`Streaming ${(data as Buffer).byteLength} bytes from ${filename}`) + _logger.trace(`Streaming ${(data as Buffer).byteLength} bytes from ${filename}`) yield data } }, @@ -760,7 +760,7 @@ export class IpfsFileManagerService extends EventEmitter { length: UNIXFS_CAT_CHUNK_SIZE, } - options.logger.info( + options.logger.trace( `Getting blocks totalling ${UNIXFS_CAT_CHUNK_SIZE} bytes with offset ${downloadedSize} (total bytes: ${totalSize})` ) @@ -872,7 +872,7 @@ export class IpfsFileManagerService extends EventEmitter { const { header, recipient } = fileMetadata.enc const decryptStream = sigChain.crypto.decryptStream(catStream, uint8ArrayFromString(header, 'base64url'), recipient) for await (const decryptedEntry of decryptStream) { - _logger.info(`Writing block with size (in bytes)`, decryptedEntry.byteLength) + _logger.trace(`Writing block with size (in bytes)`, decryptedEntry.byteLength) await new Promise((resolve, reject) => { writeStream.write(decryptedEntry, err => { if (err) {