diff --git a/.mocharc-integration.json b/.mocharc-integration.json index 5515dfd9..5d80018e 100644 --- a/.mocharc-integration.json +++ b/.mocharc-integration.json @@ -8,6 +8,6 @@ "exit": true, "recursive": true, "extension": ["ts", "js"], - "watch-files": ["src/**/*.js", "test/**/*.js"], + "watch-files": ["src/**/*.{js,ts}", "test/**/*.{js,ts}"], "ignore": ["node_modules"] } diff --git a/.mocharc.json b/.mocharc.json index a7723b3c..b628528f 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -5,9 +5,9 @@ "package": "./package.json", "reporter": "spec", "ui": "bdd", - "recursive": true, "exit": true, + "recursive": true, "extension": ["ts", "js"], "watch-files": ["src/**/*.{js,ts}", "test/**/*.{js,ts}"], - "ignore": ["node_modules", "test/integration/**/*"] + "ignore": ["node_modules", "test/integration/**/*.{js,ts}"] } diff --git a/package-lock.json b/package-lock.json index fa918476..22f17b65 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "async": "^2.6.1", "aws-sdk": "^2.395.0", "axios": "^1.6.7", + "bigint-buffer": "^1.1.5", "body-parser": "^1.18.3", "bufferutil": "4.0.8", "cache-manager": "^6.3.1", @@ -58,6 +59,7 @@ "helmet": "^3.21.1", "http-proxy-middleware": "^2.0.1", "impresso-jscommons": "https://github.com/impresso/impresso-jscommons/tarball/v1.4.3", + "json-bigint": "^1.0.0", "json2csv": "^4.3.3", "jsonpath-plus": "^10.0.1", "jsonschema": "^1.4.1", @@ -101,6 +103,7 @@ "@types/cache-manager": "^2.10.3", "@types/generic-pool": "^3.1.9", "@types/ioredis": "^4.28.5", + "@types/json-bigint": "^1.0.4", "@types/mocha": "^10.0.6", "@types/node": "^22.5.5", "@types/node-fetch": "^2.5.6", @@ -117,7 +120,7 @@ "prettier": "3.2.5", "sinon": "^19.0.2", "tsx": "^4.19.2", - "typescript": "5.6.3", + "typescript": "5.7.3", "typescript-cp": "0.1.9" }, "engines": { @@ -2540,6 +2543,13 @@ "@types/node": "*" } }, + "node_modules/@types/json-bigint": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/json-bigint/-/json-bigint-1.0.4.tgz", + "integrity": "sha512-ydHooXLbOmxBbubnA7Eh+RpBzuaIiQjh8WGJYQB50JFGFrdxW7JzVlyEV7fAXw0T2sqJ1ysTneJbiyNLqZRAag==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -2801,10 +2811,11 @@ } }, "node_modules/acorn": { - "version": "8.8.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", - "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", + "version": "8.14.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz", + "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==", "dev": true, + "license": "MIT", "bin": { "acorn": "bin/acorn" }, @@ -3291,6 +3302,28 @@ "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz", "integrity": "sha1-mrVie5PmBiH/fNrF2pczAn3x0Ms=" }, + "node_modules/bigint-buffer": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/bigint-buffer/-/bigint-buffer-1.1.5.tgz", + "integrity": "sha512-trfYco6AoZ+rKhKnxA0hgX0HAbVP/s808/EuDSe2JDzUnCp/xAsli35Orvk67UrTEcwuxZqYZDmfA2RXJgxVvA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bindings": "^1.3.0" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -3300,6 +3333,15 @@ "node": ">=8" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bitsyntax": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bitsyntax/-/bitsyntax-0.1.0.tgz", @@ -5924,6 +5966,12 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/fill-range": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", @@ -6023,10 +6071,11 @@ } }, "node_modules/flatted": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.2.tgz", - "integrity": "sha512-JaTY/wtrcSyvXJl4IMFHPKyFur1sE9AUqc0QnhOaJ0CxHtAoIV8pYDzeEfAaNEtGkOfq4gr3LBFmdXW5mOQFnA==", - "dev": true + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.2.tgz", + "integrity": "sha512-AiwGJM8YcNOaobumgtng+6NHuOqC3A7MixFeDafM3X9cIUM+xUXoS5Vfgf+OihAYe20fxqNM9yPBXJzRtZ/4eA==", + "dev": true, + "license": "ISC" }, "node_modules/fn.name": { "version": "1.1.0", @@ -7505,6 +7554,15 @@ "node": ">= 10.16.0" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", @@ -11470,9 +11528,9 @@ } }, "node_modules/typescript": { - "version": "5.6.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", - "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", + "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", diff --git a/package.json b/package.json index 9f81660c..87762400 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "copy-files": "tscp", "test": "mocha", "test-watch": "mocha --watch 'test/**/*.test.{js,ts}'", - "integration-test": "NODE_ENV=test mocha --config ./.mocharc-integration.json 'test/integration/**/*.test.js'", + "integration-test": "mocha --inspect --config ./.mocharc-integration.json", "lintfix": "eslint src/. --config .eslintrc.js --fix", "lint": "eslint src/. --config .eslintrc.js", "lint-api-spec": "spectral lint http://localhost:3030/swagger.json ", @@ -55,6 +55,7 @@ "async": "^2.6.1", "aws-sdk": "^2.395.0", "axios": "^1.6.7", + "bigint-buffer": "^1.1.5", "body-parser": "^1.18.3", "bufferutil": "4.0.8", "cache-manager": "^6.3.1", @@ -91,6 +92,7 @@ "helmet": "^3.21.1", "http-proxy-middleware": "^2.0.1", "impresso-jscommons": "https://github.com/impresso/impresso-jscommons/tarball/v1.4.3", + "json-bigint": "^1.0.0", "json2csv": "^4.3.3", "jsonpath-plus": "^10.0.1", "jsonschema": "^1.4.1", @@ -134,6 +136,7 @@ "@types/cache-manager": "^2.10.3", "@types/generic-pool": "^3.1.9", "@types/ioredis": "^4.28.5", + "@types/json-bigint": "^1.0.4", "@types/mocha": "^10.0.6", "@types/node": "^22.5.5", "@types/node-fetch": "^2.5.6", @@ -150,7 +153,7 @@ "prettier": "3.2.5", "sinon": "^19.0.2", "tsx": "^4.19.2", - "typescript": "5.6.3", + "typescript": "5.7.3", "typescript-cp": "0.1.9" } } diff --git a/src/app.ts b/src/app.ts index ca61c232..bd2520b7 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,17 +1,21 @@ import express, { Application, static as staticMiddleware } from '@feathersjs/express' import { feathers } from '@feathersjs/feathers' -import bodyParser from 'body-parser' +import compress from 'compression' +import path from 'path' import appHooksFactory from './app.hooks' import authentication from './authentication' import cache from './cache' import celery, { init as initCelery } from './celery' import channels from './channels' -import configuration from './configuration' +import configuration, { Configuration } from './configuration' +import { init as simpleSolrClient } from './internalServices/simpleSolr' import { startupJobs } from './jobs' +import middleware from './middleware' import errorHandling from './middleware/errorHandling' import openApiValidator, { init as initOpenApiValidator } from './middleware/openApiValidator' import swagger from './middleware/swagger' import transport from './middleware/transport' +import multer from './multer' import redis, { init as initRedis } from './redis' import sequelize from './sequelize' import services from './services' @@ -19,17 +23,13 @@ import rateLimiter from './services/internal/rateLimiter/redis' import media from './services/media' import proxy from './services/proxy' import schemas from './services/schemas' -import { ImpressoApplication } from './types' -import { init as simpleSolrClient } from './internalServices/simpleSolr' -import path from 'path' -import compress from 'compression' -import middleware from './middleware' -import multer from './multer' +import { AppServices, ImpressoApplication } from './types' +import { customJsonMiddleware } from './util/express' const helmet = require('helmet') const cookieParser = require('cookie-parser') -const app: ImpressoApplication & Application = express(feathers()) +const app: ImpressoApplication & Application = express(feathers()) // Load app configuration app.configure(configuration) @@ -45,8 +45,7 @@ app.configure(simpleSolrClient) app.use(helmet()) app.use(compress()) app.use(cookieParser()) -// needed to access body in non-feathers middlewares, like openapi validator -app.use(bodyParser.json({ limit: '50mb' })) +app.use(customJsonMiddleware()) // JSON body parser / serializer // configure local multer service. app.configure(multer) diff --git a/src/authentication.ts b/src/authentication.ts index e9cfb103..22d6eb6c 100644 --- a/src/authentication.ts +++ b/src/authentication.ts @@ -15,10 +15,18 @@ import User from './models/users.model' import { docs } from './services/authentication/authentication.schema' import { ImpressoApplication } from './types' import { BufferUserPlanGuest } from './models/user-bitmap.model' +import { bigIntToBuffer, bufferToBigInt } from './util/bigint' const debug = initDebug('impresso/authentication') -type AuthPayload = Omit & { userId: string; bitmap: number } +/** + * Using base64 for the bitmap to keep the size + * of the JWT token as small as possible. + */ +type AuthPayload = Omit & { + userId: string + bitmap: string // bigint as a base64 string +} class CustomisedAuthenticationService extends AuthenticationService { async getPayload(authResult: AuthenticationResult, params: AuthenticationParams) { @@ -31,7 +39,9 @@ class CustomisedAuthenticationService extends AuthenticationService { payload.groups = user.groups.map(d => d.name) } payload.isStaff = user.isStaff - payload.bitmap = Number(user.bitmap != null ? BigInt(user.bitmap) : BufferUserPlanGuest) + payload.bitmap = bigIntToBuffer(user.bitmap != null ? BigInt(user.bitmap) : BufferUserPlanGuest).toString( + 'base64' + ) } return payload } @@ -64,9 +74,6 @@ export interface SlimUser { uid: string id: number isStaff: boolean - /** - * Bitmap as number Number(BigInt) - */ bitmap: bigint groups: string[] } @@ -99,7 +106,7 @@ class NoDBJWTStrategy extends JWTStrategy { const slimUser: SlimUser = { uid: payload.userId, id: parseInt(payload.sub), - bitmap: payload.bitmap != null ? BigInt(payload.bitmap) : BufferUserPlanGuest, + bitmap: payload.bitmap != null ? bufferToBigInt(Buffer.from(payload.bitmap, 'base64')) : BufferUserPlanGuest, isStaff: payload.isStaff ?? false, groups: payload.groups ?? [], } diff --git a/src/hooks/redaction.ts b/src/hooks/redaction.ts index 20731a15..d0d3c0bd 100644 --- a/src/hooks/redaction.ts +++ b/src/hooks/redaction.ts @@ -100,31 +100,14 @@ export const bitmapsAlign = ( * and we use the user's bitmap. * If the context is not authenticated, we use the bitmap of a guest user. */ - const userBitmap: bigint | undefined = user?.bitmap ?? BufferUserPlanGuest + const userBitmap = user?.bitmap ?? BufferUserPlanGuest - const contentBitmap: bigint | undefined = - contentBitmapExtractor != null && redactable != null ? contentBitmapExtractor(redactable) : undefined - - if ( - userBitmap == null || - contentBitmap == null || - !(typeof userBitmap == 'bigint') || - !(typeof contentBitmap == 'bigint') - ) - return false + const contentBitmap = + contentBitmapExtractor != null && redactable != null ? contentBitmapExtractor(redactable) : BigInt(0) return (contentBitmap & userBitmap) != BigInt(0) } -/** - * Default condition we should currently use: - * - running as Public API - * - AND user is not in the NoRedaction group - */ -export const defaultCondition: RedactCondition = (context, redactable) => { - return inPublicApi(context, redactable) && !bitmapsAlign(context, redactable) -} - export type { RedactionPolicy } const authBitmapExtractor = (redactable: Redactable, kind: keyof AuthorizationBitmapsDTO) => { @@ -136,7 +119,10 @@ const authBitmapExtractor = (redactable: Redactable, kind: keyof AuthorizationBi } /** - * condition that grants user access to content (transcript, title). + * Condition that instructs redactor to redact parts of the content: + * Redact if: + * - the request is made through the public API + * - user bitmap does not align with the content item bitmap */ export const publicApiTranscriptRedactionCondition: RedactCondition = (context, redactable) => { return ( @@ -146,12 +132,19 @@ export const publicApiTranscriptRedactionCondition: RedactCondition = (context, const webappAuthBitmapExtractor = (redactable: Redactable, kind: keyof AuthorizationBitmapsDTO) => { const actualKey = `bitmap${kind.charAt(0).toUpperCase() + kind.slice(1)}` - return redactable[actualKey] ?? BigInt(0) + const value = redactable[actualKey] + return value != null ? BigInt(value) : BigInt(0) } +/** + * Condition that instructs redactor to redact parts of the content: + * Redact if: + * - the request is made through the app (not the public API) + * - user bitmap does not align with the content item bitmap + */ export const webAppTranscriptRedactionCondition: RedactCondition = (context, redactable) => { return ( !inPublicApi(context, redactable) && - !bitmapsAlign(context, redactable, x => webappAuthBitmapExtractor(x, 'getTranscript')) + !bitmapsAlign(context, redactable, x => webappAuthBitmapExtractor(x, 'explore')) ) } diff --git a/src/hooks/redis.js b/src/hooks/redis.js index cc7b1f3a..f26f4aee 100644 --- a/src/hooks/redis.js +++ b/src/hooks/redis.js @@ -1,5 +1,5 @@ /* eslint-disable consistent-return */ -import { customJSONReplacer } from '../util/jsonEncoder' +import { safeStringifyJson } from '../util/jsonCodec' const debug = require('debug')('impresso/hooks:redis') const feathers = require('@feathersjs/feathers') const { generateHash } = require('../crypto') @@ -141,6 +141,6 @@ export const saveResultsInCache = () => async context => { } if (!context.params.isCached || context.app.get('cache').override) { // do not override cached contents. See returnCachedContents - await client.set(context.params.cacheKey, JSON.stringify(context.result, customJSONReplacer)) + await client.set(context.params.cacheKey, safeStringifyJson(context.result)) } } diff --git a/src/internalServices/simpleSolr.ts b/src/internalServices/simpleSolr.ts index 496cd5d4..10cc1793 100644 --- a/src/internalServices/simpleSolr.ts +++ b/src/internalServices/simpleSolr.ts @@ -16,6 +16,7 @@ import { ensureServiceIsFeathersCompatible } from '../util/feathers' import { serialize } from '../util/serialize' import { defaultCachingStrategy } from '../util/solr/cacheControl' import { removeNullAndUndefined } from '../util/fn' +import { safeParseJson, safeStringifyJson } from '../util/jsonCodec' const DefaultSuggesterDictonary = 'm_suggester_infix' @@ -81,7 +82,7 @@ export interface ErrorContainer { } export type Bucket = { - val?: string | number + val?: string | number | BigInt count?: number } & { // subfacets @@ -278,11 +279,11 @@ class DefaultSimpleSolrClient implements SimpleSolrClient { ...buildAuthHeader(auth), 'Content-Type': 'application/json', }), - body: JSON.stringify(removeNullAndUndefined(request.body)), + body: safeStringifyJson(removeNullAndUndefined(request.body)), } const responseBody = await this.fetch(pool, url, init) - return JSON.parse(responseBody) + return safeParseJson(responseBody) } } @@ -313,7 +314,7 @@ class CachedDefaultSimpleSolrClient extends DefaultSimpleSolrClient { const response = await super.fetch(pool, url, init) - const action = this.cachingStrategy?.(url, init.body as string, JSON.stringify(response)) ?? 'cache' + const action = this.cachingStrategy?.(url, init.body as string, safeStringifyJson(response)) ?? 'cache' if (action === 'cache') { await this.cache.set(cacheKey, response) @@ -334,7 +335,7 @@ export const init = (app: ImpressoApplication) => { ? new CachedDefaultSimpleSolrClient(solrConfiguration, cache, defaultCachingStrategy) : new DefaultSimpleSolrClient(solrConfiguration) - console.log('is cache enabled', isCacheEnabled) + logger.info(`Using SOLR client: ${client.constructor.name}`) app.use('simpleSolrClient', ensureServiceIsFeathersCompatible(client), { methods: [], }) diff --git a/src/middleware/transport.ts b/src/middleware/transport.ts index e0f21d80..c571b672 100644 --- a/src/middleware/transport.ts +++ b/src/middleware/transport.ts @@ -1,5 +1,5 @@ import type { Application as ExpressApplication } from '@feathersjs/express' -import { json, rest, urlencoded } from '@feathersjs/express' +import { rest, urlencoded } from '@feathersjs/express' import { Decoder } from 'socket.io-parser' import cors from 'cors' import { ImpressoApplication } from '../types' @@ -7,15 +7,14 @@ import { ImpressoApplication } from '../types' import socketio from '@feathersjs/socketio' import { logger } from '../logger' -import { CustomEncoder } from '../util/jsonEncoder' +import { CustomEncoder, CustomDecoder } from '../util/jsonCodec' +import { customJsonMiddleware } from '../util/express' export default (app: ImpressoApplication & ExpressApplication) => { const isPublicApi = app.get('isPublicApi') if (isPublicApi) { logger.info('Public API - enabling REST transport') - // Turn on JSON parser for REST services - app.use(json()) // Turn on URL-encoded parser for REST services app.use(urlencoded({ extended: true })) diff --git a/src/models/generated/common.d.ts b/src/models/generated/common.d.ts index 037e009c..c6bc1c2a 100644 --- a/src/models/generated/common.d.ts +++ b/src/models/generated/common.d.ts @@ -218,6 +218,13 @@ export interface FeaturesConfig { enabled: boolean; [k: string]: unknown; }; + adminEndpoints?: { + /** + * Enable admin endpoints (see services/index) + */ + enabled: boolean; + [k: string]: unknown; + }; [k: string]: unknown; } export interface PaginateConfig { diff --git a/src/models/generated/schemasPublic.d.ts b/src/models/generated/schemasPublic.d.ts index d9ba4500..16874a44 100644 --- a/src/models/generated/schemasPublic.d.ts +++ b/src/models/generated/schemasPublic.d.ts @@ -366,6 +366,14 @@ export interface EntityMention { } +/** + * Freeform schema - a schema that allows any property to be added to the object. + */ +export interface Freeform { + [k: string]: unknown; +} + + /** * A media source is what a content item belongs to. This can be a newspaper, a TV or a radio station, etc. */ diff --git a/src/models/user-bitmap.model.ts b/src/models/user-bitmap.model.ts index c3e67637..cdb8cc8e 100644 --- a/src/models/user-bitmap.model.ts +++ b/src/models/user-bitmap.model.ts @@ -1,11 +1,11 @@ import { DataTypes, ModelDefined, Sequelize } from 'sequelize' import SubscriptionDataset, { type SubscriptionDatasetAttributes } from './subscription-datasets.model' -import Group from './groups.model' +import { bigIntToBuffer, bufferToBigInt } from '../util/bigint' export interface UserBitmapAttributes { id: number user_id: number - bitmap: number | Buffer + bitmap: bigint dateAcceptedTerms: Date | null subscriptionDatasets?: SubscriptionDatasetAttributes[] } @@ -21,14 +21,14 @@ export const BufferUserPlanResearcher = BigInt(0b1011) export default class UserBitmap { id: number user_id: number - bitmap: number | Buffer + bitmap: bigint dateAcceptedTerms: Date | null subscriptionDatasets?: SubscriptionDatasetAttributes[] constructor({ id = 0, user_id = 0, - bitmap = Number(BufferUserPlanGuest), + bitmap = BufferUserPlanGuest, dateAcceptedTerms = null, subscriptionDatasets = [], }: UserBitmapAttributes) { @@ -73,13 +73,15 @@ export default class UserBitmap { field: 'user_id', }, bitmap: { - // models.BinaryField + // Storing the bitmap as a BLOB type: DataTypes.BLOB, allowNull: true, - defaultValue: Buffer.from([Number(BufferUserPlanGuest)]), + defaultValue: bigIntToBuffer(BufferUserPlanGuest), get() { - const value = this.getDataValue('bitmap') as Buffer - return value.readUInt8(0) + return bufferToBigInt(this.getDataValue('bitmap') as any as Buffer) + }, + set(value: bigint) { + this.setDataValue('bitmap', bigIntToBuffer(value) as any as bigint) }, }, dateAcceptedTerms: { diff --git a/src/models/users.model.ts b/src/models/users.model.ts index 03c410ab..0c6c51a7 100644 --- a/src/models/users.model.ts +++ b/src/models/users.model.ts @@ -23,7 +23,7 @@ export interface UserAttributes { profile: Profile groups: Group[] userBitmap?: UserBitmap - bitmap?: number + bitmap?: bigint toJSON: (params?: { obfuscate?: boolean; groups?: Group[] }) => UserAttributes } @@ -44,7 +44,7 @@ export default class User { creationDate: Date | string profile: Profile groups: Group[] - bitmap?: number + bitmap?: bigint constructor({ id = 0, @@ -85,7 +85,7 @@ export default class User { } this.creationDate = creationDate instanceof Date ? creationDate : new Date(creationDate) this.groups = groups - this.bitmap = bitmap ?? 1 + this.bitmap = bitmap ?? BufferUserPlanGuest } static getMe({ user, profile }: { user: User; profile: Profile }) { @@ -270,7 +270,7 @@ export default class User { return new User({ ...this.get(), groups, - bitmap: userBitmap?.bitmap ?? Number(BufferUserPlanGuest), + bitmap: userBitmap?.bitmap ?? BufferUserPlanGuest, }) } diff --git a/src/schema/common/config.json b/src/schema/common/config.json index bab43ef1..149975bb 100644 --- a/src/schema/common/config.json +++ b/src/schema/common/config.json @@ -190,6 +190,13 @@ "enabled": { "type": "boolean", "description": "Enable text reuse features" } }, "required": ["enabled"] + }, + "adminEndpoints": { + "type": "object", + "properties": { + "enabled": { "type": "boolean", "description": "Enable admin endpoints (see services/index)" } + }, + "required": ["enabled"] } } }, diff --git a/src/schema/schemasPublic/Freeform.json b/src/schema/schemasPublic/Freeform.json new file mode 100644 index 00000000..fdb5e6f9 --- /dev/null +++ b/src/schema/schemasPublic/Freeform.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Freeform", + "description": "Freeform schema - a schema that allows any property to be added to the object.", + "type": "object", + "additionalProperties": true, + "properties": {} +} diff --git a/src/sequelize.js b/src/sequelize.js index 438e78b9..dc1c3b2d 100644 --- a/src/sequelize.js +++ b/src/sequelize.js @@ -1,3 +1,4 @@ +import { logger } from './logger' const debug = require('debug')('impresso/sequelize') const verbose = require('debug')('verbose:impresso/sequelize') @@ -34,7 +35,7 @@ const getSequelizeClient = config => }, }) -module.exports = function (app) { +export default function (app) { const config = app.get('sequelize') const sequelize = getSequelizeClient(config) debug(`Sequelize ${config.dialect} database name: ${config.database} ..`) @@ -43,7 +44,9 @@ module.exports = function (app) { sequelize .authenticate() .then(() => { - debug(`Sequelize is ready! ${config.dialect} database name: ${config.database}`) + logger.info( + `DB connection has been established successfully to a "${config.dialect}" database: ${config.database} on ${config.host}:${config.port}` + ) }) .catch(err => { debug(`Unable to connect to the ${config.dialect}: ${config.database}: ${err}`) @@ -52,4 +55,4 @@ module.exports = function (app) { app.set('sequelizeClient', sequelize) } -module.exports.client = getSequelizeClient +export const client = getSequelizeClient diff --git a/src/services/admin/admin.class.ts b/src/services/admin/admin.class.ts new file mode 100644 index 00000000..6f0a15bb --- /dev/null +++ b/src/services/admin/admin.class.ts @@ -0,0 +1,25 @@ +import { ClientService, Params } from '@feathersjs/feathers' +import { ImpressoApplication } from '../../types' +import { + ContentItemPermissionsDetails, + getContentItemsPermissionsDetails, +} from '../../useCases/getContentItemsPermissionsDetails' +import { getUserAccountsWithAvailablePermissions, UserAccount } from '../../useCases/getUsersPermissionsDetails' + +interface FindResponse { + permissionsDetails: ContentItemPermissionsDetails + userAccounts: UserAccount[] +} +interface FindParams {} + +type IService = Pick, 'find'> + +export class Service implements IService { + constructor(private readonly app: ImpressoApplication) {} + + async find(params?: Params): Promise { + const permissionsDetails = await getContentItemsPermissionsDetails(this.app.service('simpleSolrClient')) + const userAccounts = await getUserAccountsWithAvailablePermissions(this.app.get('sequelizeClient')!) + return { permissionsDetails, userAccounts } satisfies FindResponse + } +} diff --git a/src/services/admin/admin.schema.ts b/src/services/admin/admin.schema.ts new file mode 100644 index 00000000..43130d21 --- /dev/null +++ b/src/services/admin/admin.schema.ts @@ -0,0 +1,20 @@ +import { ServiceSwaggerOptions } from 'feathers-swagger' +import { getStandardResponses } from '../../util/openapi' + +export const getDocs = (): ServiceSwaggerOptions => ({ + description: 'Admin information', + securities: ['find'], + operations: { + find: { + operationId: 'admin', + description: 'Get admin information', + parameters: [], + responses: getStandardResponses({ + method: 'find', + schema: 'Freeform', + isPublic: true, + standardPagination: false, + }), + }, + }, +}) diff --git a/src/services/admin/admin.service.ts b/src/services/admin/admin.service.ts new file mode 100644 index 00000000..4140495e --- /dev/null +++ b/src/services/admin/admin.service.ts @@ -0,0 +1,12 @@ +import { ServiceOptions } from '@feathersjs/feathers' +import { createSwaggerServiceOptions } from 'feathers-swagger' +import { ImpressoApplication } from '../../types' +import { Service } from './admin.class' +import { getDocs } from './admin.schema' + +export default (app: ImpressoApplication) => { + app.use('/admin', new Service(app), { + events: [], + docs: createSwaggerServiceOptions({ schemas: {}, docs: getDocs() }), + } as ServiceOptions) +} diff --git a/src/services/articles/articles.hooks.js b/src/services/articles/articles.hooks.js index bbc9bb89..ab5b0eed 100644 --- a/src/services/articles/articles.hooks.js +++ b/src/services/articles/articles.hooks.js @@ -27,10 +27,12 @@ const { resolveTopics, resolveUserAddons } = require('../../hooks/resolvers/arti const { obfuscate } = require('../../hooks/access-rights') const { SolrMappings } = require('../../data/constants') -const contentItemRedactionPolicy = loadYamlFile(`${__dirname}/resources/contentItemRedactionPolicy.yml`) -const contentItemRedactionPolicyWebApp = loadYamlFile(`${__dirname}/resources/contentItemRedactionPolicyWebApp.yml`) +export const contentItemRedactionPolicy = loadYamlFile(`${__dirname}/resources/contentItemRedactionPolicy.yml`) +export const contentItemRedactionPolicyWebApp = loadYamlFile( + `${__dirname}/resources/contentItemRedactionPolicyWebApp.yml` +) -module.exports = { +export default { around: { all: [authenticate({ allowUnauthenticated: true }), rateLimit()], }, diff --git a/src/services/articles/articles.service.ts b/src/services/articles/articles.service.ts index d8fe9f60..20d2bb42 100644 --- a/src/services/articles/articles.service.ts +++ b/src/services/articles/articles.service.ts @@ -3,9 +3,7 @@ import { createSwaggerServiceOptions } from 'feathers-swagger' import { ImpressoApplication } from '../../types' import { docs } from './articles.schema' import createService from './articles.class' - -// Initializes the `articles` service on path `/articles` -const hooks = require('./articles.hooks') +import hooks from './articles.hooks' export default function (app: ImpressoApplication) { const paginate = app.get('paginate') diff --git a/src/services/index.ts b/src/services/index.ts index 9cb0adff..d898fcf2 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -21,6 +21,8 @@ const publicApiServices = [ 'media-sources', ] +const adminServices = ['admin'] + const internalApiServices = [ 'issues', 'suggestions', @@ -71,7 +73,14 @@ const internalApiServices = [ export default (app: ImpressoApplication) => { const isPublicApi = app.get('isPublicApi') - const services = isPublicApi ? publicApiServices : publicApiServices.concat(internalApiServices) + const features = app.get('features') + const adminEndpointsEnabled = features?.adminEndpoints?.enabled + + const services = [ + ...publicApiServices, + ...(!isPublicApi ? internalApiServices : []), + ...(isPublicApi && adminEndpointsEnabled ? adminServices : []), + ] logger.info(`Loading services: ${services.join(', ')}`) diff --git a/src/services/sequelize.service.js b/src/services/sequelize.service.js index 24e5ef4d..8d8ed980 100644 --- a/src/services/sequelize.service.js +++ b/src/services/sequelize.service.js @@ -11,26 +11,26 @@ function getCacheKeyForReadSqlRequest(request, modelName) { return ['cache', 'db', modelName != null ? modelName : 'unk', key].join(':') } +const loadDynamicModule = async name => { + return import(name).catch(e => { + return require(name) + }) +} + class SequelizeService { constructor({ name = '', app = null, modelName = null, cacheReads = false } = {}) { this.name = String(name) this.modelName = String(modelName || name) this.sequelize = sequelize.client(app.get('sequelize')) - import(`../models/${this.modelName}.model`) + loadDynamicModule(`../models/${this.modelName}.model`) .then(m => { - debug('MODEL', m) - if (m.Model) { - this.Model = m.Model - this.sequelizeKlass = this.Model.sequelize(this.sequelize) - } else { - this.Model = m.default - this.sequelizeKlass = this.Model.sequelize(this.sequelize) - } + this.Model = m.Model ?? m.default?.default ?? m.default ?? m + this.sequelizeKlass = this.Model.sequelize(this.sequelize) debug(`Configuring service: ${this.name} (model:${this.modelName}) success!`) }) .catch(err => { - throw new Error(`Sequelize Model not found in import: ${this.modelName}`, err) + throw new Error(`Sequelize Model not found in import: ${this.modelName}: ${err.message}`, err) }) this.cacheReads = cacheReads diff --git a/src/services/terms-of-use/terms-of-use.class.ts b/src/services/terms-of-use/terms-of-use.class.ts index d20b9028..27592cb7 100644 --- a/src/services/terms-of-use/terms-of-use.class.ts +++ b/src/services/terms-of-use/terms-of-use.class.ts @@ -71,7 +71,7 @@ export class Service { defaults: { user_id: parseInt(params.user.id, 10), // set user as authenticated user - bitmap: Buffer.from([Number(BufferUserPlanAuthUser)]), + bitmap: BufferUserPlanAuthUser, dateAcceptedTerms: new Date(), }, }) diff --git a/src/services/text-reuse-clusters/text-reuse-clusters.hooks.js b/src/services/text-reuse-clusters/text-reuse-clusters.hooks.js index 6787636d..d7a5cfe5 100644 --- a/src/services/text-reuse-clusters/text-reuse-clusters.hooks.js +++ b/src/services/text-reuse-clusters/text-reuse-clusters.hooks.js @@ -3,7 +3,13 @@ import { rateLimit } from '../../hooks/rateLimiter' import { decodeJsonQueryParameters } from '../../hooks/parameters' import { validate } from '../../hooks/params' import { parseFilters } from '../../util/queryParameters' -import { redactResponse, redactResponseDataItem, defaultCondition, inPublicApi } from '../../hooks/redaction' +import { + redactResponse, + redactResponseDataItem, + webAppTranscriptRedactionCondition, + publicApiTranscriptRedactionCondition, + inPublicApi, +} from '../../hooks/redaction' import { loadYamlFile } from '../../util/yaml' import { transformResponseDataItem, @@ -49,13 +55,15 @@ module.exports = { all: [], get: [ transformResponse(transformTextReuseCluster, inPublicApi), - redactResponse(trPassageRedactionPolicy, defaultCondition), + redactResponse(trPassageRedactionPolicy, webAppTranscriptRedactionCondition), + redactResponse(trPassageRedactionPolicy, publicApiTranscriptRedactionCondition), ], find: [ renameTopLevelField(['clusters', 'data'], inPublicApi), transformResponse(transformBaseFind, inPublicApi), transformResponseDataItem(transformTextReuseCluster, inPublicApi), - redactResponseDataItem(trPassageRedactionPolicy, defaultCondition), + redactResponseDataItem(trPassageRedactionPolicy, webAppTranscriptRedactionCondition), + redactResponseDataItem(trPassageRedactionPolicy, publicApiTranscriptRedactionCondition), ], // find: [validateWithSchema('services/text-reuse-clusters/schema/find/response.json', 'result')], // get: [validateWithSchema('services/text-reuse-clusters/schema/get/response.json', 'result')], diff --git a/src/services/text-reuse-passages/text-reuse-passages.hooks.js b/src/services/text-reuse-passages/text-reuse-passages.hooks.js index e2c5a0fc..9677a486 100644 --- a/src/services/text-reuse-passages/text-reuse-passages.hooks.js +++ b/src/services/text-reuse-passages/text-reuse-passages.hooks.js @@ -3,7 +3,13 @@ import { decodeJsonQueryParameters, decodePathParameters } from '../../hooks/par import { validate } from '../../hooks/params' import { rateLimit } from '../../hooks/rateLimiter' import { parseFilters } from '../../util/queryParameters' -import { redactResponse, redactResponseDataItem, defaultCondition, inPublicApi } from '../../hooks/redaction' +import { + redactResponse, + redactResponseDataItem, + webAppTranscriptRedactionCondition, + publicApiTranscriptRedactionCondition, + inPublicApi, +} from '../../hooks/redaction' import { loadYamlFile } from '../../util/yaml' import { transformResponseDataItem, transformResponse } from '../../hooks/transformation' import { transformTextReusePassage } from '../../transformers/textReuse' @@ -37,12 +43,14 @@ module.exports = { after: { get: [ transformResponse(transformTextReusePassage, inPublicApi), - redactResponse(trPassageRedactionPolicy, defaultCondition), + redactResponse(trPassageRedactionPolicy, webAppTranscriptRedactionCondition), + redactResponse(trPassageRedactionPolicy, publicApiTranscriptRedactionCondition), ], find: [ transformResponse(transformBaseFind, inPublicApi), transformResponseDataItem(transformTextReusePassage, inPublicApi), - redactResponseDataItem(trPassageRedactionPolicy, defaultCondition), + redactResponseDataItem(trPassageRedactionPolicy, webAppTranscriptRedactionCondition), + redactResponseDataItem(trPassageRedactionPolicy, publicApiTranscriptRedactionCondition), ], }, } diff --git a/src/useCases/getContentItemsPermissionsDetails.ts b/src/useCases/getContentItemsPermissionsDetails.ts new file mode 100644 index 00000000..ace9db57 --- /dev/null +++ b/src/useCases/getContentItemsPermissionsDetails.ts @@ -0,0 +1,149 @@ +import { SolrFacetQueryParams } from '../data/types' +import { SimpleSolrClient, SelectRequestBody, Bucket } from '../internalServices/simpleSolr' +import { SolrNamespaces } from '../solr' +import { bigIntToBitString, bigIntToLongString } from '../util/bigint' + +export type PermissionsScope = 'explore' | 'get_transcript' | 'get_images' +const permissionsScopes: PermissionsScope[] = ['explore', 'get_transcript', 'get_images'] + +type BitmapFacetField = 'rights_bm_explore_l' | 'rights_bm_get_tr_l' | 'rights_bm_get_img_l' +const bitmapFacetFields: BitmapFacetField[] = ['rights_bm_explore_l', 'rights_bm_get_tr_l', 'rights_bm_get_img_l'] + +export interface PermissionDetails { + bitmap: BigInt // this may not be displayed correctly in the browser because it may be out of safe integer range + bitmapString: string + bitmapStringBin: string + totalItems: number + sample: { + id: string + articleUrl?: string + rights_perm_use_explore_plain?: string + rights_perm_use_get_tr_plain?: string + rights_perm_use_get_img_plain?: string + } +} + +interface ScopedPermissions

{ + scope: PermissionsScope + permissions: P[] +} + +export interface ContentItemPermissionsDetails { + permissions: ScopedPermissions[] +} + +const FieldToScope: Record = { + rights_bm_explore_l: 'explore', + rights_bm_get_tr_l: 'get_transcript', + rights_bm_get_img_l: 'get_images', +} + +const ScopeToField = Object.entries(FieldToScope).reduce( + (acc, [field, scope]) => { + acc[scope] = field as BitmapFacetField + return acc + }, + {} as Record +) + +/** + * A query that returns all possible values for the bitmap fields that represent permissions. + */ +const getBitmapsFacets: SelectRequestBody = { + query: '*:*', + limit: 0, + offset: 0, + params: { hl: false }, + facet: bitmapFacetFields.reduce( + (acc, field) => { + acc[field] = { + type: 'terms', + field, + mincount: 1, + limit: 1000000, + } + return acc + }, + {} as Record + ), +} + +const getSample = (field: BitmapFacetField, permission: BigInt): SelectRequestBody => ({ + query: '*:*', + filter: `filter(${field}:${permission.toString()})`, + fields: ['id', 'rights_perm_use_explore_plain', 'rights_perm_use_get_tr_plain', 'rights_perm_use_get_img_plain'].join( + ',' + ), + limit: 1, + offset: 0, + params: { + hl: false, + }, +}) + +interface SampleSolrDocument { + id: string + rights_perm_use_explore_plain?: string + rights_perm_use_get_tr_plain?: string + rights_perm_use_get_img_plain?: string +} + +export const getContentItemsPermissionsDetails = async ( + solrClient: SimpleSolrClient +): Promise => { + const response = await solrClient.select(SolrNamespaces.Search, { + body: getBitmapsFacets, + }) + + const permissionsItems = bitmapFacetFields.map( + field => + ({ + scope: FieldToScope[field], + permissions: toPermissionDetails(response.facets?.[field]?.buckets ?? []), + }) satisfies ScopedPermissions> + ) + + const permissionsItemsWithSample = await Promise.all( + permissionsItems.map( + async ({ scope, permissions }) => + ({ + scope, + permissions: await Promise.all(permissions.map(withSample(scope, solrClient))), + }) satisfies ScopedPermissions + ) + ) + + return { permissions: permissionsItemsWithSample } +} + +const toPermissionDetails = (buckets: Bucket[]): Omit[] => { + return buckets.map(bucket => { + // Value can be either a number (if it fits) or a bigint (if it doesn't) + const bitmap = typeof bucket.val === 'bigint' ? bucket.val! : BigInt(bucket.val! as number) + + return { + bitmap, + bitmapString: bigIntToLongString(bitmap), + bitmapStringBin: bigIntToBitString(bitmap), + totalItems: bucket.count ?? 0, + } + }) +} + +const withSample = + (scope: PermissionsScope, solrClient: SimpleSolrClient) => + async (details: Omit): Promise => { + const response = await solrClient.select(SolrNamespaces.Search, { + body: getSample(ScopeToField[scope], details.bitmap), + }) + + const doc = response.response?.docs?.[0] + if (doc === undefined) throw new Error('No sample document found') + + return { + ...details, + sample: { + ...doc, + }, + } + } diff --git a/src/useCases/getUsersPermissionsDetails.ts b/src/useCases/getUsersPermissionsDetails.ts new file mode 100644 index 00000000..71e3fb54 --- /dev/null +++ b/src/useCases/getUsersPermissionsDetails.ts @@ -0,0 +1,37 @@ +import { QueryTypes, Sequelize } from 'sequelize' + +const queryDistinctUsersPermissions = ` +SELECT + b.bitmap AS bitmap, + LPAD(BIN(CONV(HEX(SUBSTRING(b.bitmap, 1, 8)), 16, 10)), 64, '0') AS bin_bitmap, + MIN(user_id) as sample_user_id, + MIN(u.email) as sample_user_email +FROM impresso_userbitmap AS b +JOIN auth_user AS u ON u.id=b.user_id +WHERE u.is_active = 1 +GROUP BY b.bitmap +ORDER BY bin_bitmap +` +// CONV(HEX(SUBSTRING(b.bitmap, 1, 8)), 16, 10) AS int64_bitmap, + +interface Record { + bitmap: Buffer + // int64_bitmap: number + bin_bitmap: string + sample_user_id: number + sample_user_email: string +} + +export type UserAccount = Omit & { + bitmap: BigInt +} + +export const getUserAccountsWithAvailablePermissions = async (dbClient: Sequelize): Promise => { + const results = await dbClient.query(queryDistinctUsersPermissions, { + type: QueryTypes.SELECT, + }) + return results.map(record => { + const { bitmap, ...rest } = record + return { ...rest, bitmap: BigInt(`0b${record.bin_bitmap}`) } + }) +} diff --git a/src/util/bigint.ts b/src/util/bigint.ts new file mode 100644 index 00000000..87e63e3f --- /dev/null +++ b/src/util/bigint.ts @@ -0,0 +1,34 @@ +import { toBufferBE, toBigIntBE } from 'bigint-buffer' + +export const bigIntToBuffer = (value: bigint): Buffer => { + return toBufferBE(value, 8) +} + +export const bufferToBigInt = (buffer: Buffer): bigint => { + return toBigIntBE(buffer) +} + +/** + * @returns a string representation of the bigint value as a bit string. + * The string is padded to 64 bits. + */ +export const bigIntToBitString = (value: bigint): string => { + return value.toString(2).padStart(64, '0') +} + +/** + * @returns a string representation of the bigint value as a decimal string. + */ +export const bigIntToLongString = (value: bigint): string => { + return value.toString(10) +} + +export const bitStringToBigInt = (bitString: string): bigint => { + return BigInt(`0b${bitString}`) +} + +const Zero = BigInt(0) + +export const bitmapsAlign = (bitmap: bigint, mask: bigint): boolean => { + return (bitmap & mask) != Zero +} diff --git a/src/util/express.ts b/src/util/express.ts new file mode 100644 index 00000000..41fd94e6 --- /dev/null +++ b/src/util/express.ts @@ -0,0 +1,42 @@ +import { Request, Response, NextFunction, Send } from 'express' +import { safeParseJson, safeStringifyJson } from './jsonCodec' + +/** + * Custom JSON middleware for Express. + * It handles JSON parsing and stringifying using a custom JSON parser + * that supports BigInts. + */ +export const customJsonMiddleware = () => { + return (req: Request, res: Response, next: NextFunction) => { + // Override response.json method + res.json = function (body: any): Response { + try { + res.header('Content-Type', 'application/json') + return res.send(Buffer.from(safeStringifyJson(body))) + } catch (error) { + next(error) + return this + } + } + + // Parse incoming JSON + if (req.is('application/json')) { + let data = '' + req.setEncoding('utf8') + req.on('data', chunk => { + data += chunk + }) + + req.on('end', () => { + try { + req.body = safeParseJson(data) + next() + } catch (error) { + next(error) + } + }) + } else { + next() + } + } +} diff --git a/src/util/jsonCodec.ts b/src/util/jsonCodec.ts new file mode 100644 index 00000000..6cdbd69d --- /dev/null +++ b/src/util/jsonCodec.ts @@ -0,0 +1,34 @@ +import JSONBigInt from 'json-bigint' +import { Encoder, Decoder } from 'socket.io-parser' + +/** + * A replacer that encodes bigint as strings. + */ +export const customJSONReplacer = (key: string, value: any) => { + if (typeof value === 'bigint') { + return value.toString(10) + } + return value +} + +export class CustomEncoder extends Encoder { + constructor() { + super(customJSONReplacer) + } +} + +export class CustomDecoder extends Decoder { + constructor() { + super() + } +} + +const parser = JSONBigInt({ useNativeBigInt: true }) + +export const safeParseJson = (json: string) => { + return parser.parse(json) +} + +export const safeStringifyJson = (o: any, space?: number) => { + return parser.stringify(o, undefined, space) +} diff --git a/src/util/jsonEncoder.ts b/src/util/jsonEncoder.ts deleted file mode 100644 index d59d9d07..00000000 --- a/src/util/jsonEncoder.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Encoder } from 'socket.io-parser' - -/** - * A replacer that encodes bigint as strings. - */ -export const customJSONReplacer = (key: string, value: any) => { - if (typeof value === 'bigint') { - return value.toString(10) - } - return value -} - -export class CustomEncoder extends Encoder { - constructor() { - super(customJSONReplacer) - } -} diff --git a/src/util/redaction.ts b/src/util/redaction.ts index 463fb27a..3bcfaf57 100644 --- a/src/util/redaction.ts +++ b/src/util/redaction.ts @@ -1,5 +1,5 @@ import { JSONPath } from 'jsonpath-plus' -import { customJSONReplacer } from './jsonEncoder' +import { safeParseJson, safeStringifyJson } from './jsonCodec' /** * Represents a redactable object with arbitrary string keys and values. @@ -20,7 +20,7 @@ export interface RedactionPolicy { items: RedactionPolicyItem[] } -const DefaultConverters: Record = { +export const DefaultConverters: Record = { redact: value => '[REDACTED]', contextNotAllowedImage: value => 'https://impresso-project.ch/assets/images/not-allowed.png', remove: value => undefined, @@ -35,7 +35,7 @@ export const redactObject = (object: T, policy: RedactionP throw new Error('The provided object is not Redactable') } - const objectCopy = JSON.parse(JSON.stringify(object, customJSONReplacer)) + const objectCopy = safeParseJson(safeStringifyJson(object)) policy.items.forEach(item => { JSONPath({ diff --git a/src/util/solr/errors.ts b/src/util/solr/errors.ts index 1fb12511..8ee03220 100644 --- a/src/util/solr/errors.ts +++ b/src/util/solr/errors.ts @@ -1,6 +1,11 @@ import { isSolrError, SolrError } from '../../solr' import { BadRequest, GeneralError } from '@feathersjs/errors' +const withErrorStack = (sourceError: Error, targetError: Error): Error => { + targetError.stack = sourceError.stack + return targetError +} + /** * Parse Solr error and convert it to a standard FetahtersJs error * with a sensible message. @@ -26,6 +31,6 @@ export function preprocessSolrError(error: Error | SolrError) { } } - if (code === 400) return new BadRequest(message) - return new GeneralError(message) + if (code === 400) return withErrorStack(error, new BadRequest(message)) + return withErrorStack(error, new GeneralError(message)) } diff --git a/src/utils/bigIntToHex.ts b/src/utils/bigIntToHex.ts new file mode 100644 index 00000000..e69de29b diff --git a/test/integration/models/subscription-datasets.model.test.ts b/test/integration/models/subscription-datasets.model.test.ts index 81aed621..1bedf80d 100644 --- a/test/integration/models/subscription-datasets.model.test.ts +++ b/test/integration/models/subscription-datasets.model.test.ts @@ -2,13 +2,13 @@ import assert from 'assert' import debug from 'debug' import { client as getSequelizeClient } from '../../../src/sequelize' -import configuration, { SequelizeConfiguration } from '../../../src/configuration' +import app from '../../../src/app' import SubscriptionDataset, { SubscriptionDatasetAttributes } from '../../../src/models/subscription-datasets.model' const logger = debug('impresso/test:models:subscription-datasets.model.test') const userId = process.env.USER_ID -const config: SequelizeConfiguration = configuration()().get('sequelize') +const config = app.get('sequelize') logger(`Sequelize configuration: ${config.host}:${config.port} db:${config.database}`) diff --git a/test/integration/models/users.model.test.ts b/test/integration/models/users.model.test.ts index 7520797e..1413d5aa 100644 --- a/test/integration/models/users.model.test.ts +++ b/test/integration/models/users.model.test.ts @@ -2,15 +2,15 @@ import assert from 'assert' import debug from 'debug' import { client as getSequelizeClient } from '../../../src/sequelize' -import configuration, { SequelizeConfiguration } from '../../../src/configuration' import User, { UserAttributes } from '../../../src/models/users.model' import UserBitmap from '../../../src/models/user-bitmap.model' import Group from '../../../src/models/groups.model' +import app from '../../../src/app' const logger = debug('impresso/test:models:users.model.test') const userId = process.env.USER_ID -const config: SequelizeConfiguration = configuration()().get('sequelize') +const config = app.get('sequelize') logger(`Test started using env variable USER_ID: ${userId} and NODE_ENV=${process.env.NODE_ENV}`) logger(`Sequelize configuration: ${config.host}:${config.port} db:${config.database}`) diff --git a/test/integration/models/users.test.ts b/test/integration/models/users.test.ts index 8260fe19..716ebcee 100644 --- a/test/integration/models/users.test.ts +++ b/test/integration/models/users.test.ts @@ -1,6 +1,6 @@ // For more information about this file see https://dove.feathersjs.com/guides/cli/service.test.html import assert from 'assert' -const app = require('../../src/app') +import app from '../../../src/app' describe('test Service method to get users representations', () => { if (!process.env.USER_ID) { diff --git a/test/integration/sanity-check.test.js b/test/integration/sanity-check.test.js index 85390abb..3ddb16e1 100644 --- a/test/integration/sanity-check.test.js +++ b/test/integration/sanity-check.test.js @@ -1,5 +1,5 @@ -const assert = require('assert') -const app = require('../../src/app') +import assert from 'assert' +import app from '../../src/app' /* ./node_modules/.bin/eslint \ @@ -100,7 +100,11 @@ describe("'OpenPrivate' behaviour", function () { describe("'Luxwort' contents", function () { this.timeout(15000) - const service = app.service('content-items') + let service + + before(async () => { + service = app.service('content-items') + }) it('registered the service', () => { assert.ok(service) @@ -133,7 +137,13 @@ describe("'Luxwort' contents", function () { describe('Test /images service for crazy contents', function () { this.timeout(5000) - const service = app.service('images') + + let service + + before(async () => { + service = app.service('images') + }) + it('get NO images from issue obermosel-1930-12-23-a without errors', async () => { assert.ok(service) const result = await service.find({ diff --git a/test/integration/services/articles-suggestions.test.js b/test/integration/services/articles-suggestions.test.js index 28332b7f..2bdba7db 100644 --- a/test/integration/services/articles-suggestions.test.js +++ b/test/integration/services/articles-suggestions.test.js @@ -11,7 +11,11 @@ src/services/articles-suggestions \ describe("'articles-suggestions' service", function () { this.timeout(10000) - const service = app.service('articles-suggestions') + let service + + before(() => { + service = app.service('articles-suggestions') + }) it('registered the service', () => { assert.ok(service, 'Registered the service') diff --git a/test/integration/services/articles-tags.test.js b/test/integration/services/articles-tags.test.js index eb7dc90a..954efa41 100644 --- a/test/integration/services/articles-tags.test.js +++ b/test/integration/services/articles-tags.test.js @@ -15,7 +15,12 @@ mocha test/services/articles-tags.test.js describe("'articles-tags' service", function () { this.timeout(10000) - const service = app.service('articles-tags') + let service + + before(() => { + service = app.service('articles-tags') + }) + let user = { username: 'guest-test-2', password: 'Apaaiiai87!!', diff --git a/test/integration/services/articles-timelines.test.js b/test/integration/services/articles-timelines.test.js index 5ea8d8ab..9b91ed10 100644 --- a/test/integration/services/articles-timelines.test.js +++ b/test/integration/services/articles-timelines.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /* ./node_modules/.bin/eslint \ @@ -10,38 +10,47 @@ src/models src/hooks \ && NODE_ENV=test DEBUG=impresso* mocha test/services/articles-timelines.test.js */ -describe('\'articles-timelines\' service', function () { - this.timeout(5000); - const service = app.service('articles-timelines'); +describe("'articles-timelines' service", function () { + this.timeout(5000) + + let service + + before(() => { + service = app.service('articles-timelines') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('should filter timeline based on topic uid', async () => { const results = await service.get('stats', { query: { - filters: [{ - type: 'topic', - q: 'tmLETEMPS_tp23_fr', - }], + filters: [ + { + type: 'topic', + q: 'tmLETEMPS_tp23_fr', + }, + ], }, - }); - assert.ok(results); - assert.ok(results.values[0].w); - assert.ok(results.format); - }); + }) + assert.ok(results) + assert.ok(results.values[0].w) + assert.ok(results.format) + }) it('should handle empty filtered timeline', async () => { const results = await service.get('stats', { query: { - filters: [{ - type: 'topic', - q: 'thisTopicDoesNotExist', - }], + filters: [ + { + type: 'topic', + q: 'thisTopicDoesNotExist', + }, + ], }, - }); - assert.ok(results.values[0].w); - assert.equal(results.extents.w1.join(','), '0,0'); - }); -}); + }) + assert.ok(results.values[0].w) + assert.equal(results.extents.w1.join(','), '0,0') + }) +}) diff --git a/test/integration/services/articles.test.js b/test/integration/services/articles.test.js index f2a9d38f..8ef70769 100644 --- a/test/integration/services/articles.test.js +++ b/test/integration/services/articles.test.js @@ -15,7 +15,13 @@ const app = require('../../../src/app') */ describe("'articles' service", function () { this.timeout(15000) - const service = app.service('content-items') + + let service + + before(() => { + service = app.service('content-items') + }) + it('registered the service', () => { assert.ok(service) }) diff --git a/test/integration/services/collectable-items.test.js b/test/integration/services/collectable-items.test.js index 884d228b..2934c56b 100644 --- a/test/integration/services/collectable-items.test.js +++ b/test/integration/services/collectable-items.test.js @@ -1,6 +1,6 @@ -const assert = require('assert'); -const app = require('../../../src/app'); -const { generateUser, removeGeneratedUser } = require('./utils'); +const assert = require('assert') +const app = require('../../../src/app') +const { generateUser, removeGeneratedUser } = require('./utils') /* ./node_modules/.bin/eslint \ @@ -10,15 +10,21 @@ const { generateUser, removeGeneratedUser } = require('./utils'); --config .eslintrc.json --fix \ && DEBUG=impresso* mocha test/services/collectable-items.test.js */ -describe('\'collectable-items\' service', function () { - this.timeout(30000); - const service = app.service('collectable-items'); - const user = {}; - const collection = {}; +describe("'collectable-items' service", function () { + this.timeout(30000) + + let service + + before(() => { + service = app.service('collectable-items') + }) + + const user = {} + const collection = {} it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) // it.only('get the items for a specific collection', async () => { // const result = await service.find({ @@ -31,42 +37,48 @@ describe('\'collectable-items\' service', function () { // }); it('setup the test', async () => { - const result = await generateUser(); - assert.ok(result.uid, 'should have an uid prop'); - assert.ok(result.id, 'should have an id'); - assert.ok(result.username, 'should have a nice username'); + const result = await generateUser() + assert.ok(result.uid, 'should have an uid prop') + assert.ok(result.id, 'should have an id') + assert.ok(result.username, 'should have a nice username') // enrich the user variable - user.username = result.username; - user.uid = result.uid; - user.id = result.id; - }); + user.username = result.username + user.uid = result.uid + user.id = result.id + }) it('create a collection', async () => { - const result = await app.service('collections').create({ - name: 'a nice name', - description: 'digitus', - }, { - user, - }); - assert.ok(result.uid, 'should have an unique uid'); - collection.uid = result.uid; - }); + const result = await app.service('collections').create( + { + name: 'a nice name', + description: 'digitus', + }, + { + user, + } + ) + assert.ok(result.uid, 'should have an unique uid') + collection.uid = result.uid + }) it('create an entry item for the current user', async () => { - const results = await service.create({ - collection_uid: collection.uid, - items: [ - { - uid: 'GDL-1967-04-25-a-i0152', - content_type: 'article', - }, - ], - }, { - authenticated: true, - user, - }); - assert.ok(results.data.length); - }); + const results = await service.create( + { + collection_uid: collection.uid, + items: [ + { + uid: 'GDL-1967-04-25-a-i0152', + content_type: 'article', + }, + ], + }, + { + authenticated: true, + user, + } + ) + assert.ok(results.data.length) + }) it('find all items for the current user', async () => { const results = await service.find({ @@ -76,11 +88,11 @@ describe('\'collectable-items\' service', function () { }, authenticated: true, user, - }); - assert.equal(results.total, 1); - assert.ok(results.data[0].collections.length, 'has at least one collection!'); - assert.equal(results.data[0].item.uid, 'GDL-1967-04-25-a-i0152', 'item has been resolved'); - }); + }) + assert.equal(results.total, 1) + assert.ok(results.data[0].collections.length, 'has at least one collection!') + assert.equal(results.data[0].item.uid, 'GDL-1967-04-25-a-i0152', 'item has been resolved') + }) it('find all collectableitems for a given set of item uids', async () => { const results = await service.find({ @@ -89,24 +101,22 @@ describe('\'collectable-items\' service', function () { }, authenticated: true, user, - }); - assert.ok(results); - }); + }) + assert.ok(results) + }) it('find all collectableitems for a given set of item uids', async () => { const results = await service.find({ query: { - collection_uids: [ - collection.uid, - ], + collection_uids: [collection.uid], resolve: 'item', }, authenticated: true, user, - }); + }) - assert.ok(results); - }); + assert.ok(results) + }) it('remove an article from a collection', async () => { const results = await service.remove(collection.uid, { @@ -119,12 +129,12 @@ describe('\'collectable-items\' service', function () { ], }, user, - }); - assert.equal(results.removed, 1); - assert.ok(results); - }); + }) + assert.equal(results.removed, 1) + assert.ok(results) + }) it('remove setup', async () => { - await removeGeneratedUser(user); - }); -}); + await removeGeneratedUser(user) + }) +}) diff --git a/test/integration/services/collections.test.js b/test/integration/services/collections.test.js index 8366c202..6acca979 100644 --- a/test/integration/services/collections.test.js +++ b/test/integration/services/collections.test.js @@ -1,6 +1,6 @@ -const assert = require('assert'); -const app = require('../../../src/app'); -const { generateUser, removeGeneratedUser } = require('./utils'); +const assert = require('assert') +const app = require('../../../src/app') +const { generateUser, removeGeneratedUser } = require('./utils') /* ./node_modules/.bin/eslint \ @@ -13,77 +13,85 @@ const user = { username: 'local-user-test-only', password: 'Impresso2018!', email: 'local-user-test-only@impresso-project.ch', -}; +} const collection = { uid: 'this-is-random-collection-id', name: 'a nice name', description: 'digitus', -}; +} -describe('\'collections\' service', function () { - this.timeout(15000); +describe("'collections' service", function () { + this.timeout(15000) + + let service before(async () => { - const result = await generateUser(user); - assert.ok(result.uid, 'should have an uid prop'); - assert.ok(result.id, 'should have an id'); - assert.equal(result.username, user.username); + service = app.service('collections') + + const result = await generateUser(user) + assert.ok(result.uid, 'should have an uid prop') + assert.ok(result.id, 'should have an id') + assert.equal(result.username, user.username) // enrich the user variable - user.uid = result.uid; - user.id = result.id; + user.uid = result.uid + user.id = result.id // runs before all tests in this block - }); + }) after(async () => { - await removeGeneratedUser(user); - }); - - const service = app.service('collections'); + await removeGeneratedUser(user) + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('create, edit then remove a collection', async () => { - console.log('create a collection', user); + console.log('create a collection', user) const created = await service.create(collection, { user, - }); + }) - const patched = await service.patch(created.uid, { - name: 'a new name', - description: '', - }, { - user, - }); - assert.deepEqual(patched.uid, created.uid); - assert.deepEqual(patched.name, 'a new name'); + const patched = await service.patch( + created.uid, + { + name: 'a new name', + description: '', + }, + { + user, + } + ) + assert.deepEqual(patched.uid, created.uid) + assert.deepEqual(patched.name, 'a new name') const getted = await service.get(created.uid, { user, - }); - assert.ok(getted.name, 'a new name'); + }) + assert.ok(getted.name, 'a new name') const found = await service.find({ user, query: { q: 'new', }, - }); - assert.deepEqual(found.data[0].uid, created.uid); + }) + assert.deepEqual(found.data[0].uid, created.uid) const removed = await service.remove(created.uid, { user, - }); + }) - await service.get(created.uid, { - user, - }).catch((err) => { - assert.deepEqual(err.name, 'NotFound'); - }); + await service + .get(created.uid, { + user, + }) + .catch(err => { + assert.deepEqual(err.name, 'NotFound') + }) - assert.deepEqual(removed.uid, created.uid); - assert.deepEqual(removed.status, 'DEL'); - }); -}); + assert.deepEqual(removed.uid, created.uid) + assert.deepEqual(removed.status, 'DEL') + }) +}) diff --git a/test/integration/services/embeddings.test.js b/test/integration/services/embeddings.test.js index 33175044..103b8ac1 100644 --- a/test/integration/services/embeddings.test.js +++ b/test/integration/services/embeddings.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /* ./node_modules/.bin/eslint \ src/services/embeddings \ @@ -7,34 +7,46 @@ test/services/embeddings.test.js \ --config .eslintrc.json --fix \ && NODE_ENV=development DEBUG=verbose*,imp* mocha test/services/embeddings.test.js */ -describe('\'embeddings\' service', () => { - const service = app.service('embeddings'); +describe("'embeddings' service", () => { + let service + + before(() => { + service = app.service('embeddings') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) - it('should not work (BadRequest) with multiple words', async () => service.find({ - query: { - q: 'amour folie', - language: 'fr', - }, - }).then(() => { - assert.fail('no no no, it\'s not possible'); - }).catch((err) => { - assert.strictEqual(err.name, 'BadRequest'); - })); + it('should not work (BadRequest) with multiple words', async () => + service + .find({ + query: { + q: 'amour folie', + language: 'fr', + }, + }) + .then(() => { + assert.fail("no no no, it's not possible") + }) + .catch(err => { + assert.strictEqual(err.name, 'BadRequest') + })) - it('should not work (NotFound) with words that do not exist in our embeddings', async () => service.find({ - query: { - q: 'Vélozzome', - language: 'fr', - }, - }).then(() => { - assert.fail('no no no, it\'s not possible'); - }).catch((err) => { - assert.strictEqual(err.name, 'NotFound'); - })); + it('should not work (NotFound) with words that do not exist in our embeddings', async () => + service + .find({ + query: { + q: 'Vélozzome', + language: 'fr', + }, + }) + .then(() => { + assert.fail("no no no, it's not possible") + }) + .catch(err => { + assert.strictEqual(err.name, 'NotFound') + })) it('should work even with accents and weird characters', async () => { const result = await service.find({ @@ -42,8 +54,8 @@ describe('\'embeddings\' service', () => { q: 'Vélo', language: 'fr', }, - }); + }) // closest one: vélo - assert.strictEqual(result.data[0], 'vélo'); - }); -}); + assert.strictEqual(result.data[0], 'vélo') + }) +}) diff --git a/test/integration/services/entities.test.js b/test/integration/services/entities.test.js index d7d4569c..5c135a86 100644 --- a/test/integration/services/entities.test.js +++ b/test/integration/services/entities.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /** ./node_modules/.bin/eslint \ test/services/entities.test.js \ @@ -8,13 +8,18 @@ src/models/entities.model.js \ --config .eslintrc.json --fix \ && NODE_ENV=development DEBUG=@feathersjs/error*,impresso* mocha test/services/entities.test.js */ -describe('\'entities\' service', function () { - this.timeout(10000); - const service = app.service('entities'); +describe("'entities' service", function () { + this.timeout(10000) + + let service + + before(() => { + service = app.service('entities') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('test find method with q param', async () => { const result = await service.find({ @@ -27,26 +32,26 @@ describe('\'entities\' service', function () { }, ], }, - }); - assert.ok(result.total); - assert.ok(result.data[0].type, 'person'); - }); + }) + assert.ok(result.total) + assert.ok(result.data[0].type, 'person') + }) it('test wikidata contents for get:aida-0001-50-"Arizona"_Charlie_Meadows (yes with quotes)', async () => { - const entity = await service.get('aida-0001-50-"Arizona"_Charlie_Meadows'); - assert.ok(entity.wikidata.images, 'Entity 1 must contain wikidata as it has wikidata id'); - }); + const entity = await service.get('aida-0001-50-"Arizona"_Charlie_Meadows') + assert.ok(entity.wikidata.images, 'Entity 1 must contain wikidata as it has wikidata id') + }) it('test get:aida-0001-54-Luxembourg_(city)', async () => { - const entity = await service.get('aida-0001-54-Luxembourg_(city)'); - assert.strictEqual(entity.name, 'Luxembourg (city)'); - }); + const entity = await service.get('aida-0001-54-Luxembourg_(city)') + assert.strictEqual(entity.name, 'Luxembourg (city)') + }) it('test get method with a location', async () => { - const entity = await service.get('aida-0001-54-Berlin'); - assert.strictEqual(entity.name, 'Berlin'); - assert.strictEqual(entity.type, 'location'); - assert.strictEqual(entity.wikidataId, 'Q64'); - assert.strictEqual(entity.uid, 'aida-0001-54-Berlin'); - assert.ok(entity.wikidata.images, 'Entity 1 must contain wikidata as it has wikidata id'); - }); -}); + const entity = await service.get('aida-0001-54-Berlin') + assert.strictEqual(entity.name, 'Berlin') + assert.strictEqual(entity.type, 'location') + assert.strictEqual(entity.wikidataId, 'Q64') + assert.strictEqual(entity.uid, 'aida-0001-54-Berlin') + assert.ok(entity.wikidata.images, 'Entity 1 must contain wikidata as it has wikidata id') + }) +}) diff --git a/test/integration/services/errors-collector.test.js b/test/integration/services/errors-collector.test.js index 7e9c02c6..0bc45d5a 100644 --- a/test/integration/services/errors-collector.test.js +++ b/test/integration/services/errors-collector.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /** ./node_modules/.bin/eslint \ @@ -8,12 +8,16 @@ src/services/errors-collector \ --config .eslintrc.json --fix \ && mocha test/services/errors-collector.test.js */ -describe('\'errors-collector\' service', () => { - const service = app.service('errors-collector'); +describe("'errors-collector' service", () => { + let service + + before(() => { + service = app.service('errors-collector') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('send BadRequest data to Error console', async () => { const stderr = await service.create({ @@ -59,7 +63,7 @@ describe('\'errors-collector\' service', () => { }, }, }, - }); - console.log(stderr); - }); -}); + }) + console.log(stderr) + }) +}) diff --git a/test/integration/services/images.test.js b/test/integration/services/images.test.js index c4251142..973faa95 100644 --- a/test/integration/services/images.test.js +++ b/test/integration/services/images.test.js @@ -10,7 +10,12 @@ src/services/images \ */ describe("'images' service", function () { this.timeout(20000) - const service = app.service('images') + + let service + + before(() => { + service = app.service('images') + }) it('registered a working service', () => { assert.ok(service, 'Registered the service') diff --git a/test/integration/services/issues.test.js b/test/integration/services/issues.test.js index 7929c2a0..e0ab1770 100644 --- a/test/integration/services/issues.test.js +++ b/test/integration/services/issues.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /** * use with @@ -10,27 +10,32 @@ const app = require('../../../src/app'); --config .eslintrc.json --fix \ && NODE_ENV=development DEBUG=impresso/* mocha test/services/issues.test.js */ -describe('\'issues\' service', function () { - this.timeout(25000); - const service = app.service('issues'); +describe("'issues' service", function () { + this.timeout(25000) + + let service + + before(() => { + service = app.service('issues') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('it should load issues, sort by -date', async () => { const results = await service.find({ query: { order_by: ['date', '-name'], }, - }); - assert.ok(results.total > 0, 'find at least one issue :)'); - assert.ok(results.data[0].iiif, 'has iiif url based on cover'); + }) + assert.ok(results.total > 0, 'find at least one issue :)') + assert.ok(results.data[0].iiif, 'has iiif url based on cover') // cannot test if is instanceOf Issue here, since we have after hooks // who transform data. - assert.ok(results.total > 0, 'find at least one issue :)'); - }); + assert.ok(results.total > 0, 'find at least one issue :)') + }) it('it should load issues from GDL only, sort by -date', async () => { const results = await service.find({ @@ -44,20 +49,19 @@ describe('\'issues\' service', function () { }, ], }, - }); - assert.ok(results.total > 0, 'find at least one issue :)'); - assert.ok(results.data[0].iiif, 'has iiif url based on cover'); + }) + assert.ok(results.total > 0, 'find at least one issue :)') + assert.ok(results.data[0].iiif, 'has iiif url based on cover') // cannot test if is instanceOf Issue here, since we have after hooks // who transform data. - assert.ok(results.total > 0, 'find at least one issue :)'); - }); + assert.ok(results.total > 0, 'find at least one issue :)') + }) it('it should load a single issue', async () => { - const result = await service.get('GDL-1811-11-22-a', { - }).catch((err) => { - console.log(err); - }); - assert.ok(result); - assert.equal(result.uid, 'GDL-1811-11-22-a'); - }); -}); + const result = await service.get('GDL-1811-11-22-a', {}).catch(err => { + console.log(err) + }) + assert.ok(result) + assert.equal(result.uid, 'GDL-1811-11-22-a') + }) +}) diff --git a/test/integration/services/jobs.test.js b/test/integration/services/jobs.test.js index f75ec3f7..a70a56df 100644 --- a/test/integration/services/jobs.test.js +++ b/test/integration/services/jobs.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /** * use with @@ -10,18 +10,25 @@ const app = require('../../../src/app'); --config .eslintrc.json --fix \ && DEBUG=impresso/* mocha test/services/jobs.test.js */ -describe('\'jobs\' service', () => { - const service = app.service('jobs'); +describe("'jobs' service", () => { + let service + + before(() => { + service = app.service('jobs') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('create test job', async () => { - const result = await service.create({}, { - user: { - id: 1, - }, - }); - console.log(result); - }); -}); + const result = await service.create( + {}, + { + user: { + id: 1, + }, + } + ) + console.log(result) + }) +}) diff --git a/test/integration/services/mentions.test.js b/test/integration/services/mentions.test.js index 3d585542..8212e3a6 100644 --- a/test/integration/services/mentions.test.js +++ b/test/integration/services/mentions.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /* ./node_modules/.bin/eslint \ @@ -11,56 +11,64 @@ src/hooks/resolvers/mentions.resolvers.js \ --config .eslintrc.json --fix \ && NODE_ENV=development DEBUG=verbose*,imp*,@feathersjs/error* mocha test/services/mentions.test.js */ -describe('\'mentions\' service', function () { - this.timeout(30000); +describe("'mentions' service", function () { + this.timeout(30000) - const service = app.service('mentions'); + let service + + before(() => { + service = app.service('mentions') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('find mentions for Leipzig', async () => { - const result = await service.find({ - query: { - order_by: 'id', - filters: [ - { - type: 'entity', - q: 'aida-0001-54-Leipzig', - }, - ], - }, - }).catch((err) => { - console.log(err); - }); - assert.ok(result.total, 'there are results'); - assert.ok(result.data[0].entityId, 'aida-0001-54-Leipzig'); - assert.ok(result.data[0].type, 'location'); - assert.ok(result.data[0].articleUid, 'there should be an article attached'); - assert.ok(result.data[0].article.uid, 'there is an article attached'); - }); + const result = await service + .find({ + query: { + order_by: 'id', + filters: [ + { + type: 'entity', + q: 'aida-0001-54-Leipzig', + }, + ], + }, + }) + .catch(err => { + console.log(err) + }) + assert.ok(result.total, 'there are results') + assert.ok(result.data[0].entityId, 'aida-0001-54-Leipzig') + assert.ok(result.data[0].type, 'location') + assert.ok(result.data[0].articleUid, 'there should be an article attached') + assert.ok(result.data[0].article.uid, 'there is an article attached') + }) it('find mentions for a person', async () => { - const result = await service.find({ - query: { - order_by: 'name', - filters: [ - { - type: 'entity', - q: 'aida-0001-50-"Arizona"_Charlie_Meadows', - }, - ], - }, - }).catch((err) => { - console.log(err); - }); - assert.ok(result.total, 'there are results'); - assert.ok(result.data[0].type, 'person'); - assert.ok(result.data[0].ancillary); - assert.ok(result.data[0].articleUid, 'there should be an article attached'); + const result = await service + .find({ + query: { + order_by: 'name', + filters: [ + { + type: 'entity', + q: 'aida-0001-50-"Arizona"_Charlie_Meadows', + }, + ], + }, + }) + .catch(err => { + console.log(err) + }) + assert.ok(result.total, 'there are results') + assert.ok(result.data[0].type, 'person') + assert.ok(result.data[0].ancillary) + assert.ok(result.data[0].articleUid, 'there should be an article attached') if (result.data[0].article) { - assert.ok(result.data[0].article.uid, 'there is an article attached'); + assert.ok(result.data[0].article.uid, 'there is an article attached') } - }); -}); + }) +}) diff --git a/test/integration/services/newspapers.test.js b/test/integration/services/newspapers.test.js index 6a0db13b..800092f1 100644 --- a/test/integration/services/newspapers.test.js +++ b/test/integration/services/newspapers.test.js @@ -1,49 +1,56 @@ -const assert = require('assert'); -const app = require('../../../src/app'); -const Newspaper = require('../../../src/models/newspapers.model'); +const assert = require('assert') +const app = require('../../../src/app') +const Newspaper = require('../../../src/models/newspapers.model') /** ./node_modules/.bin/eslint \ src/models src/services/newspapers src/hooks test/services/newspapers.test.js \ --config .eslintrc.json --fix \ && NODE_ENV=test DEBUG=impresso*,verbose* mocha test/services/newspapers.test.js */ -describe('\'newspapers\' service', function () { - this.timeout(15000); - const service = app.service('newspapers'); +describe("'newspapers' service", function () { + this.timeout(15000) + + let service + + before(() => { + service = app.service('newspapers') + }) it('registered the service', async () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('handle 404 not found', async () => { const result = await service.get('JDGRRRRRRRRR').catch(({ code }) => { - assert.strictEqual(code, 404); - }); - assert.strictEqual(result, undefined); - }); + assert.strictEqual(code, 404) + }) + assert.strictEqual(result, undefined) + }) it('get single newspaper', async () => { - const result = await service.get('JDG'); - assert.strictEqual(result.uid, 'JDG'); - assert.strictEqual(result.included, true); - }); + const result = await service.get('JDG') + assert.strictEqual(result.uid, 'JDG') + assert.strictEqual(result.included, true) + }) it('get single newspaper existing, not yet included', async () => { - const result = await service.get('DVF'); - assert.strictEqual(result.uid, 'DVF'); - assert.strictEqual(result.included, false); - }); + const result = await service.get('DVF') + assert.strictEqual(result.uid, 'DVF') + assert.strictEqual(result.included, false) + }) it('get newspapers, should fail because of JSON schema', async () => { - const result = await service.find({ - query: { - included: true, - }, - }).catch(({ code }) => { - assert.strictEqual(code, 400); - }); - assert.strictEqual(result, undefined); - }); + const result = await service + .find({ + query: { + included: true, + }, + }) + .catch(({ code }) => { + assert.strictEqual(code, 400) + }) + assert.strictEqual(result, undefined) + }) it('get only included newspaper', async () => { const results = await service.find({ @@ -55,36 +62,36 @@ describe('\'newspapers\' service', function () { }, ], }, - }); - assert.ok(results.total > 0, 'has a total greater than zero'); + }) + assert.ok(results.total > 0, 'has a total greater than zero') if (!results.cached) { - assert.ok(results.data[0] instanceof Newspaper, 'is an instance of Newspaper'); + assert.ok(results.data[0] instanceof Newspaper, 'is an instance of Newspaper') } else { - assert.ok(results.data[0].acronym, 'is a valid newspaper'); + assert.ok(results.data[0].acronym, 'is a valid newspaper') } - }); + }) it('get newspapers, order by countIssues desc', async () => { const results = await service.find({ query: { order_by: '-countIssues', }, - }); - assert.ok(results.data[0].countIssues > results.data[1].countIssues, 'the biggest newspaper first'); - }); + }) + assert.ok(results.data[0].countIssues > results.data[1].countIssues, 'the biggest newspaper first') + }) it('get newspapers!', async () => { const results = await service.find({ query: {}, - }); + }) - assert.ok(results.total > 0, 'has a total greater than zero'); + assert.ok(results.total > 0, 'has a total greater than zero') if (!results.cached) { - assert.ok(results.data[0] instanceof Newspaper, 'is an instance of Newspaper'); + assert.ok(results.data[0] instanceof Newspaper, 'is an instance of Newspaper') } else { - assert.ok(results.data[0].acronym, 'is a valid newspaper'); + assert.ok(results.data[0].acronym, 'is a valid newspaper') } - }); + }) it('get newspapers containing "gazette", ordered by start year!', async () => { const results = await service.find({ @@ -92,12 +99,12 @@ describe('\'newspapers\' service', function () { q: 'gazette', order_by: 'startYear', }, - }); - assert.ok(results.total > 0, 'has a total greater than zero'); + }) + assert.ok(results.total > 0, 'has a total greater than zero') if (!results.cached) { - assert.ok(results.data[0] instanceof Newspaper, 'is an instance of Newspaper'); + assert.ok(results.data[0] instanceof Newspaper, 'is an instance of Newspaper') } else { - assert.ok(results.data[0].acronym, 'is a valid newspaper'); + assert.ok(results.data[0].acronym, 'is a valid newspaper') } - }); -}); + }) +}) diff --git a/test/integration/services/pages-timelines.test.js b/test/integration/services/pages-timelines.test.js index e30da06a..b4b29fa5 100644 --- a/test/integration/services/pages-timelines.test.js +++ b/test/integration/services/pages-timelines.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const app = require('../../../src/app') +const assert = require('assert') /* ./node_modules/.bin/eslint \ @@ -10,38 +10,44 @@ src/models src/hooks \ && NODE_ENV=test DEBUG=impresso* mocha test/services/pages-timelines.test.js */ -describe('\'pages-timelines\' service', function () { - this.timeout(15000); +describe("'pages-timelines' service", function () { + this.timeout(15000) - const service = app.service('pages-timelines'); + let service + + before(() => { + service = app.service('pages-timelines') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('timeline fn not found', async () => { - await app.service('pages-timelines').get('not-valid', {}) + await app + .service('pages-timelines') + .get('not-valid', {}) .then(() => { - assert.fail('not here'); + assert.fail('not here') + }) + .catch(err => { + assert.equal(err.name, 'NotFound', 'should get a 404 NotFound error') + assert.ok(err, 'should get an error') }) - .catch((err) => { - assert.equal(err.name, 'NotFound', 'should get a 404 NotFound error'); - assert.ok(err, 'should get an error'); - }); - }); + }) it.only('global pages timelines', async () => { - const results = await app.service('pages-timelines').get('stats', {}); - console.log(results); - assert.ok(results); - }); + const results = await app.service('pages-timelines').get('stats', {}) + console.log(results) + assert.ok(results) + }) it('JDG pages timelines', async () => { const results = await app.service('pages-timelines').get('stats', { query: { newspaper_uid: 'GDL', }, - }); - console.log(results); - }); -}); + }) + console.log(results) + }) +}) diff --git a/test/integration/services/pages.test.js b/test/integration/services/pages.test.js index 7903f67e..5eba328c 100644 --- a/test/integration/services/pages.test.js +++ b/test/integration/services/pages.test.js @@ -1,6 +1,6 @@ -const assert = require('assert'); -const app = require('../../../src/app'); -const { generateUser, removeGeneratedUser } = require('./utils'); +const assert = require('assert') +const app = require('../../../src/app') +const { generateUser, removeGeneratedUser } = require('./utils') /* @@ -9,48 +9,53 @@ test/services/pages.test.js src/services/pages --fix \ && DEBUG=impresso* mocha test/services/pages.test.js */ -describe('\'pages\' service', function () { - this.timeout(10000); - const service = app.service('pages'); +describe("'pages' service", function () { + this.timeout(10000) + + let service + + before(() => { + service = app.service('pages') + }) let user = { username: 'guest-test-2', password: 'Apaaiiai87!!', email: 'guest-test-2@impresso-project.ch', - }; + } it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('setup the test', async () => { - user = await generateUser(user); - assert.ok(user.uid); - }); + user = await generateUser(user) + assert.ok(user.uid) + }) it('get complete page item', async () => { - const pag = await service.get('GDL-1902-05-12-a-p0002'); + const pag = await service.get('GDL-1902-05-12-a-p0002') - console.log(pag); - assert.ok(pag.labels, 'has label'); - assert.ok(pag.iiif, 'has iiif proxied'); + console.log(pag) + assert.ok(pag.labels, 'has label') + assert.ok(pag.iiif, 'has iiif proxied') // assert.ok(pag.labbellos, 'has label'); - }); + }) - it('if there\'s an user_uid, get the buckets associate to the page', async () => { + it("if there's an user_uid, get the buckets associate to the page", async () => { const pag = await service.get('GDL-1882-06-01-a-p0002', { user: { uid: 'local-user-test-only', }, - }); + }) // console.log(pag); - assert.ok(pag.labels, 'has label'); - assert.ok(pag.iiif, 'has iiif proxied'); + assert.ok(pag.labels, 'has label') + assert.ok(pag.iiif, 'has iiif proxied') // assert.ok(pag.labbellos, 'has label'); - }); + }) it('remove setup', async () => { - await removeGeneratedUser(user); - }); -}); + await removeGeneratedUser(user) + }) +}) diff --git a/test/integration/services/search-exporter.test.js b/test/integration/services/search-exporter.test.js index a62048ab..f3cef8dd 100644 --- a/test/integration/services/search-exporter.test.js +++ b/test/integration/services/search-exporter.test.js @@ -11,7 +11,12 @@ const app = require('../../../src/app') */ describe("'search-exporter' service", function () { this.timeout(20000) - const service = app.service('search-exporter') + + let service + + before(() => { + service = app.service('search-exporter') + }) it('registered the service', () => { assert.ok(service, 'Registered the service') diff --git a/test/integration/services/search-facets.test.js b/test/integration/services/search-facets.test.js index 5812757b..f6582bce 100644 --- a/test/integration/services/search-facets.test.js +++ b/test/integration/services/search-facets.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /* ./node_modules/.bin/eslint \ @@ -9,12 +9,16 @@ test/services/search-facets.test.js \ --config .eslintrc.json --fix \ && NODE_ENV=development DEBUG=verbose*,imp* mocha test/services/search-facets.test.js */ -describe('\'search-facets\' service', () => { - const service = app.service('search-facets'); +describe("'search-facets' service", () => { + let service + + before(() => { + service = app.service('search-facets') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('registered the service', async () => { const results = await service.find({ @@ -22,7 +26,7 @@ describe('\'search-facets\' service', () => { group_by: 'articles', facets: ['person'], }, - }); - console.log(results); - }); -}); + }) + console.log(results) + }) +}) diff --git a/test/integration/services/search.create.test.js b/test/integration/services/search.create.test.js index b0a5a136..9a5ecb52 100644 --- a/test/integration/services/search.create.test.js +++ b/test/integration/services/search.create.test.js @@ -1,7 +1,7 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') -const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) /* ./node_modules/.bin/eslint \ src/services/search src/hooks \ @@ -9,47 +9,60 @@ src/services/search src/hooks \ && NODE_ENV=development DEBUG=impresso* mocha test/services/search.create.test.js */ -describe('\'search\' service, \'create\' method', function () { - this.timeout(20000); - const service = app.service('search'); +describe("'search' service, 'create' method", function () { + this.timeout(20000) + + let service + + before(() => { + service = app.service('search') + }) it('throw an error bad request', async () => { - await service.create({ - collection_uid: 'local-abc', - }, { - query: { - group_by: 'articles', - }, - }).catch((err) => { - assert.equal(err.name, 'BadRequest', 'should be 400 Bad Request'); - assert.equal(err.data.filters.code, 'NotFound', 'filters should be compulsory'); - }); - }); + await service + .create( + { + collection_uid: 'local-abc', + }, + { + query: { + group_by: 'articles', + }, + } + ) + .catch(err => { + assert.equal(err.name, 'BadRequest', 'should be 400 Bad Request') + assert.equal(err.data.filters.code, 'NotFound', 'filters should be compulsory') + }) + }) it('save a search', async () => { - await sleep(2000); - const result = await service.create({}, { - user: { - id: 12, - }, - query: { - collection_uid: 'local-abc', - group_by: 'articles', - filters: [ - { - type: 'string', - context: 'include', - q: 'europ*', - }, - { - type: 'daterange', - context: 'include', - daterange: '1900-01-01T00:00:00Z TO 1945-01-01T00:00:00Z', - }, - ], - }, - }); - console.log('save a search', result); - assert.ok(result, 'should not throw any exception'); - }); -}); + await sleep(2000) + const result = await service.create( + {}, + { + user: { + id: 12, + }, + query: { + collection_uid: 'local-abc', + group_by: 'articles', + filters: [ + { + type: 'string', + context: 'include', + q: 'europ*', + }, + { + type: 'daterange', + context: 'include', + daterange: '1900-01-01T00:00:00Z TO 1945-01-01T00:00:00Z', + }, + ], + }, + } + ) + console.log('save a search', result) + assert.ok(result, 'should not throw any exception') + }) +}) diff --git a/test/integration/services/search.test.js b/test/integration/services/search.test.js index d23be750..1c83f9d3 100644 --- a/test/integration/services/search.test.js +++ b/test/integration/services/search.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /** * Test for search endpoint. Usage: @@ -16,17 +16,23 @@ const app = require('../../../src/app'); && NODE_ENV=test DEBUG=impresso* mocha test/services/search.test.js */ -describe('\'search\' service', function () { - this.timeout(15000); - const service = app.service('search'); +describe("'search' service", function () { + this.timeout(15000) + + let service + + before(() => { + service = app.service('search') + }) + // const staff = { // uid: 'local-user-test-only', // is_staff: true, // }; it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('get user collection facets (if any, but should not be any err)', async () => { const result = await service.find({ @@ -38,19 +44,20 @@ describe('\'search\' service', function () { facets: ['collection'], filters: [], }, - }); + }) - assert.ok(result.data); - assert.ok(result.info.facets); + assert.ok(result.data) + assert.ok(result.info.facets) - console.log(result.info.facets.collection); - }); + console.log(result.info.facets.collection) + }) it('get search results for one specific topic', async () => { // find one topic - const [t1, t2] = await app.service('topics') + const [t1, t2] = await app + .service('topics') .find({ query: { limit: 2 } }) - .then(({ data }) => data); + .then(({ data }) => data) const result = await service.find({ query: { @@ -69,11 +76,11 @@ describe('\'search\' service', function () { }, ], }, - }); + }) // console.log(result.info.facets.topic); - assert.ok(result.info.facets.topic, 'should expose topic in facet'); - assert.ok(result.info.facets.year, 'should expose year in facet'); - }); + assert.ok(result.info.facets.topic, 'should expose topic in facet') + assert.ok(result.info.facets.year, 'should expose year in facet') + }) it('get search results with regex queries', async () => { const result = await service.find({ query: { @@ -87,10 +94,10 @@ describe('\'search\' service', function () { }, ], }, - }); + }) // console.log(result) - assert.ok(result.info.facets.year); - }); + assert.ok(result.info.facets.year) + }) it('get search results when no filters is given', async () => { // return; @@ -102,10 +109,10 @@ describe('\'search\' service', function () { limit: 12, order_by: '-relevance', }, - }); - assert.ok(result.info.queryComponents); - assert.ok(result.info.facets.year); - }); + }) + assert.ok(result.info.queryComponents) + assert.ok(result.info.facets.year) + }) it('get search results with daterange filters', async () => { const result = await service.find({ @@ -125,9 +132,9 @@ describe('\'search\' service', function () { }, ], }, - }); - assert.ok(result.info.facets.year); - }); + }) + assert.ok(result.info.facets.year) + }) it('get search results with newspaper filters, without searching for text', async () => { const result = await service.find({ @@ -142,10 +149,10 @@ describe('\'search\' service', function () { limit: 1, order_by: '-relevance', }, - }); - assert.ok(result); - assert.deepEqual(result.data[0].newspaper.uid, 'NZZ'); - }); + }) + assert.ok(result) + assert.deepEqual(result.data[0].newspaper.uid, 'NZZ') + }) it('get search results with newspaper filters, with string filter', async () => { const result = await service.find({ @@ -160,24 +167,26 @@ describe('\'search\' service', function () { limit: 12, order_by: '-relevance', }, - }); - assert.ok(result); - assert.deepEqual(result.data[0].newspaper.uid, 'NZZ'); - }); + }) + assert.ok(result) + assert.deepEqual(result.data[0].newspaper.uid, 'NZZ') + }) it('loaded solr content', async () => { - const results = await service.find({ - query: { - q: 'avion accident', - group_by: 'articles', - facets: ['year'], - }, - }).catch((err) => { - assert.fail(err); - }); + const results = await service + .find({ + query: { + q: 'avion accident', + group_by: 'articles', + facets: ['year'], + }, + }) + .catch(err => { + assert.fail(err) + }) - assert.ok(results.data.length); - }); + assert.ok(results.data.length) + }) // it('loaded solr content, filters & facets, with current user having a bucket ;)', async () => { // // remove the bucket if any @@ -254,9 +263,7 @@ describe('\'search\' service', function () { group_by: 'articles', order_by: '-date,-relevance', limit: 1, - facets: [ - 'language', - ], + facets: ['language'], filters: [ { type: 'string', @@ -268,7 +275,7 @@ describe('\'search\' service', function () { user: { uid: 'local-user-test-only', }, - }); - assert.ok(res); - }); -}); + }) + assert.ok(res) + }) +}) diff --git a/test/integration/services/suggestions.test.js b/test/integration/services/suggestions.test.js index db796615..b9881be3 100644 --- a/test/integration/services/suggestions.test.js +++ b/test/integration/services/suggestions.test.js @@ -1,6 +1,6 @@ -const assert = require('assert'); -const app = require('../../../src/app'); -const { toPlainText } = require('../../../src/helpers'); +const assert = require('assert') +const app = require('../../../src/app') +const { toPlainText } = require('../../../src/helpers') /* ./node_modules/.bin/eslint \ @@ -11,102 +11,109 @@ const { toPlainText } = require('../../../src/helpers'); */ -describe('\'suggestions\' service', function () { - this.timeout(10000); - const service = app.service('suggestions'); +describe("'suggestions' service", function () { + this.timeout(10000) + let service + + before(() => { + service = app.service('suggestions') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('test getTopics sub service', async () => { const results = await service.suggestTopics({ q: 'suiss', - }); - assert.strictEqual(results[0].type, 'topic'); - }); + }) + assert.strictEqual(results[0].type, 'topic') + }) it('test getMentions sub service', async () => { const results = await service.suggestMentions({ q: 'Rome', - }); - assert.strictEqual(results[0].type, 'mention'); - assert.strictEqual(results[0].item.type, 'location', 'correct mention type in mention item'); - assert.strictEqual(results[0].q, 'Rome'); - assert.strictEqual(results[0].item.name, 'Rome'); - }); + }) + assert.strictEqual(results[0].type, 'mention') + assert.strictEqual(results[0].item.type, 'location', 'correct mention type in mention item') + assert.strictEqual(results[0].q, 'Rome') + assert.strictEqual(results[0].item.name, 'Rome') + }) it('test getMentions sub service with partial regex', async () => { const results = await service.suggestMentions({ q: toPlainText('/go[uû]t.*parfait.*'), - }); - assert.ok(results); - }); + }) + assert.ok(results) + }) it('only one year', async () => { const suggestions = await service.find({ query: { q: '1947', }, - }); + }) - assert.strictEqual(suggestions.data[0].daterange, '1947-01-01T00:00:00Z TO 1947-12-31T23:59:59Z'); - }); + assert.strictEqual(suggestions.data[0].daterange, '1947-01-01T00:00:00Z TO 1947-12-31T23:59:59Z') + }) it('two years', async () => { const suggestions = await app.service('suggestions').find({ query: { q: '1950-1951', }, - }); - assert.strictEqual(suggestions.data[0].daterange, '1950-01-01T00:00:00Z TO 1951-12-31T23:59:59Z'); - }); + }) + assert.strictEqual(suggestions.data[0].daterange, '1950-01-01T00:00:00Z TO 1951-12-31T23:59:59Z') + }) it('one year', async () => { const suggestions = await app.service('suggestions').find({ query: { q: 'october 1950 to october 1951', }, - }); - assert.strictEqual(suggestions.data[0].daterange, '1950-10-01T10:00:00Z TO 1951-10-31T23:59:59Z'); - }); + }) + assert.strictEqual(suggestions.data[0].daterange, '1950-10-01T10:00:00Z TO 1951-10-31T23:59:59Z') + }) it('one month', async () => { const suggestions = await app.service('suggestions').find({ query: { q: 'october 1956', }, - }); + }) - assert.strictEqual(suggestions.data[0].daterange, '1956-10-01T10:00:00Z TO 1956-10-31T23:59:59Z'); - }); + assert.strictEqual(suggestions.data[0].daterange, '1956-10-01T10:00:00Z TO 1956-10-31T23:59:59Z') + }) it('exact match with partial q', async () => { - const suggestions = await app.service('suggestions').find({ - query: { - q: '"louis', - }, - }).catch((err) => { - console.log(err); - throw err; - }); - assert.ok(suggestions.data.length); - }); + const suggestions = await app + .service('suggestions') + .find({ + query: { + q: '"louis', + }, + }) + .catch(err => { + console.log(err) + throw err + }) + assert.ok(suggestions.data.length) + }) it('recognizes an invalid regexp', async () => { const suggestions = await app.service('suggestions').find({ query: { q: '/*.*[/', }, - }); - assert.strictEqual(suggestions.data.length, 0); - }); + }) + assert.strictEqual(suggestions.data.length, 0) + }) it('recognizes a valid regexp and split on spaces', async () => { const suggestions = await app.service('suggestions').find({ query: { q: '/go[uû]t.*parfait.*/', }, - }); - assert.strictEqual(suggestions.data[0].type, 'regex'); - }); -}); + }) + assert.strictEqual(suggestions.data[0].type, 'regex') + }) +}) diff --git a/test/integration/services/topics.test.js b/test/integration/services/topics.test.js index 2f4ee06a..e76d4536 100644 --- a/test/integration/services/topics.test.js +++ b/test/integration/services/topics.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /** ./node_modules/.bin/eslint \ @@ -7,21 +7,25 @@ src/models src/services/topics src/hooks test/services/topics.test.js \ --config .eslintrc.json --fix \ && DEBUG=impresso* mocha test/services/topics.test.js */ -describe('\'topics\' service', () => { - const service = app.service('topics'); +describe("'topics' service", () => { + let service + + before(() => { + service = app.service('topics') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('should not raise an issue when q is null (from socket)', async () => { const results = await service.find({ query: { q: null, }, - }); - assert.ok(results.data[0]); - }); + }) + assert.ok(results.data[0]) + }) // it('use filters to get topics from one model only', async () => { // const results = await service.find({ @@ -52,4 +56,4 @@ describe('\'topics\' service', () => { // console.log(results.info); // assert.equal(results.data[0].model, 'tmJDG'); // }); -}); +}) diff --git a/test/integration/services/uploaded-images.test.js b/test/integration/services/uploaded-images.test.js index a0675c18..262c55c7 100644 --- a/test/integration/services/uploaded-images.test.js +++ b/test/integration/services/uploaded-images.test.js @@ -1,26 +1,30 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /** ./node_modules/.bin/eslint \ src/services/uploaded-images \ --config .eslintrc.json --fix \ && NODE_ENV=development DEBUG=imp* mocha test/services/uploaded-images.test.js */ -describe('\'uploaded-images\' service', function () { - this.timeout(10000); +describe("'uploaded-images' service", function () { + this.timeout(10000) - const service = app.service('uploaded-images'); + let service + + before(() => { + service = app.service('uploaded-images') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); + assert.ok(service, 'Registered the service') + }) it('get all images for one user', async () => { const result = await service.find({ user: { id: 1, }, - }); - console.log(result); - }); -}); + }) + console.log(result) + }) +}) diff --git a/test/integration/services/user-requests.test.ts b/test/integration/services/user-requests.test.ts index 37250dc7..cc3cc884 100644 --- a/test/integration/services/user-requests.test.ts +++ b/test/integration/services/user-requests.test.ts @@ -1,6 +1,6 @@ // For more information about this file see https://dove.feathersjs.com/guides/cli/service.test.html import assert from 'assert' -const app = require('../../src/app') +import app from '../../../src/app' describe('test Service method to list all my requests', () => { if (!process.env.USER_ID) { diff --git a/test/integration/services/users.test.js b/test/integration/services/users.test.js index 22d4bb08..80aebd48 100644 --- a/test/integration/services/users.test.js +++ b/test/integration/services/users.test.js @@ -10,9 +10,14 @@ const User = require('../../../src/models/users.model') && DEBUG=impresso/* mocha test/services/users.test.js */ describe("'users' service", function () { - const service = app.service('users') this.timeout(10000) + let service + + before(() => { + service = app.service('users') + }) + it('registered the service', () => { assert.ok(service, 'Registered the service') }) diff --git a/test/integration/services/version.test.js b/test/integration/services/version.test.js index f4b3e260..71f2270b 100644 --- a/test/integration/services/version.test.js +++ b/test/integration/services/version.test.js @@ -1,5 +1,5 @@ -const assert = require('assert'); -const app = require('../../../src/app'); +const assert = require('assert') +const app = require('../../../src/app') /** * ./node_modules/.bin/eslint \ @@ -7,10 +7,14 @@ src/services/version test/services/version.test.js --fix && mocha test/services/version.test.js */ -describe('\'version\' service', () => { - const service = app.service('version'); +describe("'version' service", () => { + let service + + before(() => { + service = app.service('version') + }) it('registered the service', () => { - assert.ok(service, 'Registered the service'); - }); -}); + assert.ok(service, 'Registered the service') + }) +}) diff --git a/test/integration/use_cases/permissions.test.ts b/test/integration/use_cases/permissions.test.ts new file mode 100644 index 00000000..013a04d9 --- /dev/null +++ b/test/integration/use_cases/permissions.test.ts @@ -0,0 +1,153 @@ +import assert from 'assert' +import app from '../../../src/app' +import { SlimUser } from '../../../src/authentication' +import { + ContentItemPermissionsDetails, + getContentItemsPermissionsDetails, + PermissionDetails, + PermissionsScope, +} from '../../../src/useCases/getContentItemsPermissionsDetails' +import { UserAccount, getUserAccountsWithAvailablePermissions } from '../../../src/useCases/getUsersPermissionsDetails' +import { bigIntToBitString, bitmapsAlign } from '../../../src/util/bigint' +import { safeStringifyJson } from '../../../src/util/jsonCodec' + +import { + contentItemRedactionPolicy, + contentItemRedactionPolicyWebApp, +} from '../../../src/services/articles/articles.hooks' +import { DefaultConverters, RedactionPolicy } from '../../../src/util/redaction' +import { JSONPath } from 'jsonpath-plus' + +interface RedactionTestContext { + scope: PermissionsScope + + contentItemId: string + contentItemPermissions: bigint + contentSample: PermissionDetails['sample'] + + userAccountId: number + userAccountPermissions: bigint + + accessAllowed: boolean +} + +const buildSlimUser = (context: RedactionTestContext): SlimUser => ({ + bitmap: context.userAccountPermissions, + groups: [], + id: context.userAccountId, + isStaff: false, + uid: context.userAccountId.toString(), +}) + +const buildTestMatrix = ( + contentItemDetails: ContentItemPermissionsDetails, + userAccounts: UserAccount[] +): RedactionTestContext[] => { + const items = contentItemDetails.permissions.map(scopeItem => { + return scopeItem.permissions.map(permissionItem => { + return userAccounts.map(userAccount => { + const contentBitmap = permissionItem.bitmap.valueOf() // as bigint + const userBitmap = userAccount.bitmap.valueOf() // as bigint + + return { + scope: scopeItem.scope, + + contentItemId: permissionItem.sample.id, + contentItemPermissions: contentBitmap, + contentSample: permissionItem.sample, + + userAccountId: userAccount.sample_user_id, + userAccountPermissions: userBitmap, + + accessAllowed: bitmapsAlign(contentBitmap, userBitmap), + } satisfies RedactionTestContext + }) + }) + }) + return items.flat(3) as RedactionTestContext[] +} + +const getPaths = (policy: RedactionPolicy, object: any, redactionExpectation: 'redacted' | 'notRedacted') => { + const paths: string[] = [] + + policy.items.forEach(item => { + JSONPath({ + path: item.jsonPath, + json: object, + resultType: 'value', + callback: (value, type, payload) => { + const valueConverter = DefaultConverters[item.valueConverterName] + const redactedValue = valueConverter(value) + const actualValue = payload.parent[payload.parentProperty] + + if (redactionExpectation === 'redacted' && actualValue != null && actualValue !== redactedValue) { + paths.push(item.jsonPath) + } + if (redactionExpectation === 'notRedacted' && actualValue === redactedValue) { + paths.push(item.jsonPath) + } + }, + }) + }) + + return paths +} + +describe('Bitmap permissions', function () { + this.timeout(30000) + + let isPublicApi = false + let testMatrix: RedactionTestContext[] = [] + + before(async () => { + isPublicApi = app.get('isPublicApi') ?? false + + const solrCilent = app.service('simpleSolrClient') + const sequelize = app.get('sequelizeClient')! + + const [userAccounts, contentPermissionsDetails] = await Promise.all([ + getUserAccountsWithAvailablePermissions(sequelize), + getContentItemsPermissionsDetails(solrCilent), + ]) + testMatrix = buildTestMatrix(contentPermissionsDetails, userAccounts) + const totalContentItemPermissions = contentPermissionsDetails.permissions.reduce( + (acc, item) => acc + item.permissions.length, + 0 + ) + + console.log( + `${totalContentItemPermissions} various content item permissions across ${contentPermissionsDetails.permissions.length} scopes found` + ) + console.log(`${userAccounts.length} various user accounts permissions found`) + console.log(`${testMatrix.length} test cases to run`) + }) + + it('Web App: get article', async () => { + if (isPublicApi) return + + const redactionPolicy = contentItemRedactionPolicyWebApp + const getTranscriptCases = testMatrix.filter(test => test.scope === 'explore') + const service = app.service('articles') + + const indices = Array.from({ length: getTranscriptCases.length }, (_, i) => i) + + for await (const idx of indices) { + const testCase = getTranscriptCases[idx] + console.log(`Testing case ${idx} of ${getTranscriptCases.length}...`) + const params = { user: buildSlimUser(testCase) } + const result = await service.get(testCase.contentItemId, params) + + const paths = getPaths(redactionPolicy, result, testCase.accessAllowed ? 'notRedacted' : 'redacted') + + const failMessage = ` + Content item ${testCase.contentItemId} is supposed to be ${testCase.accessAllowed ? 'not redacted' : 'redacted'} for user ${testCase.userAccountId}, + but these paths are ${testCase.accessAllowed ? 'redacted' : 'not redacted'}: ${paths.join(',')}. + Content bitmap:\t${bigIntToBitString(testCase.contentItemPermissions)} + User bitmap:\t${bigIntToBitString(testCase.userAccountPermissions)} + Content: ${safeStringifyJson(result, 2)} + ` + + assert.strictEqual(paths.length, 0, failMessage) + } + }) +}) diff --git a/test/util/bigint.test.ts b/test/util/bigint.test.ts new file mode 100644 index 00000000..3ffc6f9c --- /dev/null +++ b/test/util/bigint.test.ts @@ -0,0 +1,66 @@ +import { + bigIntToBitString, + bigIntToBuffer, + bigIntToLongString, + bitmapsAlign, + bufferToBigInt, +} from '../../src/util/bigint' +import assert from 'assert' + +const testBigInts: [bigint, string][] = [ + [BigInt(0), 'AAAAAAAAAAA='], + [BigInt(1), 'AAAAAAAAAAE='], + [BigInt('0b' + '1' + [...Array(63)].map(() => '0').join('')), 'gAAAAAAAAAA='], + [BigInt('0b' + '1' + [...Array(62)].map(() => '0').join('') + '1'), 'gAAAAAAAAAE='], + [BigInt('0b' + [...Array(64)].map(() => '1').join('')), '//////////8='], +] + +function bufferToBinaryString(buffer) { + return buffer + .toString('hex') + .match(/.{1,2}/g) + .map(byte => parseInt(byte, 16).toString(2).padStart(8, '0')) + .join(' ') +} + +describe('bigint utils', () => { + it('should convert bigint to buffer and back', () => { + testBigInts.forEach(([bigint, base64Representation]) => { + const buffer = bigIntToBuffer(bigint) + assert.strictEqual( + buffer.toString('base64'), + base64Representation, + ` bigint: ${bigint}, buffer: ${buffer.toString('hex')}` + ) + const bigint2 = bufferToBigInt(buffer) + assert.strictEqual(bigint, bigint2) + }) + }) + + it('should represent bigint as a bit string', () => { + const maxValue = BigInt('0b' + [...Array(64)].map(() => '1').join('')) + const maxBitString = bigIntToBitString(maxValue) + assert.strictEqual(maxBitString, '1111111111111111111111111111111111111111111111111111111111111111') + + const minValue = BigInt(0) + const minBitString = bigIntToBitString(minValue) + assert.strictEqual(minBitString, '0000000000000000000000000000000000000000000000000000000000000000') + }) + + it('should represent bigint as a long string', () => { + const maxValue = BigInt('0b' + [...Array(64)].map(() => '1').join('')) + const maxLongString = bigIntToLongString(maxValue) + assert.strictEqual(maxLongString, '18446744073709551615') + + const minValue = BigInt(0) + const minLongString = bigIntToLongString(minValue) + assert.strictEqual(minLongString, '0') + }) + + it('checks bitmaps alignment', () => { + assert.ok(bitmapsAlign(BigInt(0b0010), BigInt(0b1010))) + assert.ok(bitmapsAlign(BigInt(0b0001), BigInt(0b0001))) + assert.ok(!bitmapsAlign(BigInt(0b0001), BigInt(0b1000))) + assert.ok(!bitmapsAlign(BigInt(0b0001), BigInt(0b0100))) + }) +}) diff --git a/test/util/jsonCodec.test.ts b/test/util/jsonCodec.test.ts new file mode 100644 index 00000000..0e2a57e8 --- /dev/null +++ b/test/util/jsonCodec.test.ts @@ -0,0 +1,113 @@ +import assert from 'assert' +import { customJSONReplacer, CustomEncoder, safeParseJson, safeStringifyJson } from '../../src/util/jsonCodec' + +describe('jsonCodec', () => { + describe('customJSONReplacer', () => { + it('should convert BigInt to string', () => { + const bigIntValue = BigInt('9007199254740991') + const result = customJSONReplacer('test', bigIntValue) + assert.strictEqual(result, '9007199254740991') + }) + + it('should return original value for non-BigInt types', () => { + const testCases = [123, 'string', true, { key: 'value' }, [1, 2, 3], null] + + testCases.forEach(value => { + const result = customJSONReplacer('test', value) + assert.strictEqual(result, value) + }) + }) + }) + + describe('CustomEncoder', () => { + it('should create encoder instance with customJSONReplacer', () => { + const encoder = new CustomEncoder() + assert(encoder instanceof CustomEncoder) + }) + + it('should properly encode object with BigInt', () => { + const encoder = new CustomEncoder() + const testObj = { id: BigInt('9007199254740991') } + const encoded = JSON.stringify(testObj, customJSONReplacer) + assert.strictEqual(encoded, '{"id":"9007199254740991"}') + }) + }) + + describe('safeParseJson', () => { + it('should convert unsafe integers to BigInt', () => { + const json = '{"unsafe":9223372036854775807,"safe":123,"s":"123"}' + const result = safeParseJson(json) + + assert(typeof result.unsafe === 'bigint') + assert(typeof result.safe === 'number') + assert.strictEqual(result.unsafe, BigInt('9223372036854775807')) + assert.strictEqual(result.safe, 123) + assert.strictEqual(result.s, '123') + }) + + it('should handle nested objects and arrays', () => { + const json = '{"nested":{"unsafe":9223372036854775807},"arr":[9223372036854775807]}' + const result = safeParseJson(json) + + assert.strictEqual(result.nested.unsafe, BigInt('9223372036854775807')) + assert.strictEqual(result.arr[0], BigInt('9223372036854775807')) + }) + }) + + describe('safeStringifyJson', () => { + it('should stringify object with BigInt values', () => { + const obj = { + bigInt: BigInt('9223372036854775807'), + number: 123, + string: 'test', + } + const result = safeStringifyJson(obj) + assert.strictEqual(result, '{"bigInt":9223372036854775807,"number":123,"string":"test"}') + }) + + it('should handle nested objects with BigInt values', () => { + const obj = { + nested: { + bigInt: BigInt('9223372036854775807'), + }, + arr: [BigInt('9223372036854775807')], + } + const result = safeStringifyJson(obj) + assert.strictEqual(result, '{"nested":{"bigInt":9223372036854775807},"arr":[9223372036854775807]}') + }) + + it('should handle arrays of mixed types', () => { + const arr = [BigInt('9223372036854775807'), 123, 'test', { bigInt: BigInt('9223372036854775807') }] + const result = safeStringifyJson(arr) + assert.strictEqual(result, '[9223372036854775807,123,"test",{"bigInt":9223372036854775807}]') + }) + + it('should handle null and undefined values', () => { + const obj = { + nullValue: null, + undefinedValue: undefined, + bigInt: BigInt('9223372036854775807'), + } + const result = safeStringifyJson(obj) + assert.strictEqual(result, '{"nullValue":null,"bigInt":9223372036854775807}') + }) + + it('should handle round trip from safeStringifyJson to safeParseJson', () => { + const original = { + bigInt: BigInt('9223372036854775807'), + nested: { + unsafe: BigInt('9223372036854775807'), + safe: 123, + string: 'test', + }, + array: [BigInt('9223372036854775807'), 456, 'test'], + string: 'test', + } + + const stringified = safeStringifyJson(original) + const parsed = safeParseJson(stringified) + + assert.deepEqual(parsed, original) + }) + }) +}) diff --git a/test/util/redaction.test.ts b/test/util/redaction.test.ts index 61160a98..8d85230d 100644 --- a/test/util/redaction.test.ts +++ b/test/util/redaction.test.ts @@ -41,7 +41,7 @@ describe('redactObject', () => { secret: undefined, } satisfies TestDocument - assert.deepStrictEqual(redactObject(input, policy), expectedOutput) + assert.deepEqual(redactObject(input, policy), expectedOutput) }) incorrectInputs.forEach(input => {