Merge pull request #441 from impresso/develop

Release v3.0.3

.vscode/settings.json

@@ -21,5 +21,6 @@
   },
   "[json]": {
     "editor.formatOnSave": true
-  }
+  },
+  "mochaExplorer.files": "test/**/*.test.{ts,js}"
 }

package.json

@@ -25,8 +25,8 @@
     "watch": "tsc -p ./tsconfig.json -w & tscp -w",
     "build": "tsc -p ./tsconfig.json",
     "copy-files": "tscp",
-    "test": "mocha 'test/**/*.test.js'",
-    "test-watch": "mocha 'test/**/*.test.js' --watch",
+    "test": "mocha --require ts-node/register 'test/**/*.test.{js,ts}'",
+    "test-watch": "mocha --require ts-node/register --watch 'test/**/*.test.{js,ts}'",
     "integration-test": "NODE_ENV=test mocha --config ./.mocharc-integration.json 'test/integration/**/*.test.js'",
     "lintfix": "eslint src/. --config .eslintrc.js --fix",
     "lint": "eslint src/. --config .eslintrc.js",
@@ -92,6 +92,7 @@
     "http-proxy-middleware": "^2.0.1",
     "impresso-jscommons": "https://github.com/impresso/impresso-jscommons/tarball/v1.4.3",
     "json2csv": "^4.3.3",
+    "jsonpath-plus": "^10.0.1",
     "jsonschema": "^1.4.1",
     "lodash": "^4.17.21",
     "lodash.first": "^3.0.0",
@@ -123,7 +124,7 @@
     "wikidata-sdk": "^5.15.10",
     "winston": "3.13.0",
     "xml2js": "^0.6.2",
-    "yaml": "^2.1.1",
+    "yaml": "^2.6.0",
     "undici": "6.19.8"
   },
   "devDependencies": {

src/authentication.ts

@@ -59,6 +59,7 @@ export interface SlimUser {
   uid: string
   id: number
   isStaff: boolean
+  groups: string[]
 }
 
 /**
@@ -91,6 +92,7 @@ class NoDBJWTStrategy extends JWTStrategy {
       uid: payload.userId,
       id: parseInt(payload.sub),
       isStaff: payload.isStaff ?? false,
+      groups: payload.userGroups ?? [],
     }
     return {
       ...result,

src/hooks/redaction.ts

@@ -0,0 +1,88 @@
+import { HookContext, HookFunction } from '@feathersjs/feathers'
+import { FindResponse } from '../models/common'
+import { ImpressoApplication } from '../types'
+import { Redactable, RedactionPolicy, redactObject } from '../util/redaction'
+import { SlimUser } from '../authentication'
+
+export type RedactCondition = (context: HookContext<ImpressoApplication>) => boolean
+
+/**
+ * Redact the response object using the provided redaction policy.
+ * If the condition is provided, the redaction will only be applied if the condition is met.
+ */
+export const redactResponse = <S>(
+  policy: RedactionPolicy,
+  condition?: (context: HookContext<ImpressoApplication>) => boolean
+): HookFunction<ImpressoApplication, S> => {
+  return context => {
+    if (context.type != 'after') throw new Error('The redactResponse hook should be used as an after hook only')
+
+    if (condition != null && !condition(context)) return context
+
+    if (context.result != null) {
+      context.result = redactObject(context.result, policy)
+    }
+    return context
+  }
+}
+
+/**
+ * Redact the response object using the provided redaction policy.
+ * Assumes that the response is a FindResponse object (has a `data` field with
+ * an array of objects).
+ * If the condition is provided, the redaction will only be applied if the condition is met.
+ */
+export const redactResponseDataItem = <S>(
+  policy: RedactionPolicy,
+  condition?: (context: HookContext<ImpressoApplication>) => boolean,
+  dataItemsField?: string
+): HookFunction<ImpressoApplication, S> => {
+  return context => {
+    if (context.type != 'after') throw new Error('The redactResponseDataItem hook should be used as an after hook only')
+
+    if (condition != null && !condition(context)) return context
+
+    if (context.result != null) {
+      if (dataItemsField != null) {
+        const result = context.result as Record<string, any>
+        result[dataItemsField] = result[dataItemsField].map((item: Redactable) => redactObject(item, policy))
+      } else {
+        const result = context.result as any as FindResponse<Redactable>
+        result.data = result.data.map(item => redactObject(item, policy))
+      }
+    }
+    return context
+  }
+}
+
+/**
+ * Below are conditions that can be used in the redactResponse hook.
+ */
+export const inPublicApi: RedactCondition = context => {
+  return context.app.get('isPublicApi') == true
+}
+
+/**
+ * Condition is:
+ *  - user is not authenticated
+ *  - OR user is authenticated and is not in the specified group
+ */
+export const notInGroup =
+  (groupName: string): RedactCondition =>
+  context => {
+    const user = context.params?.user as any as SlimUser
+    return user == null || !user.groups.includes(groupName)
+  }
+
+const NoRedactionGroup = 'NoRedaction'
+
+/**
+ * Default condition we should currently use:
+ * - running as Public API
+ * - AND user is not in the NoRedaction group
+ */
+export const defaultCondition: RedactCondition = context => {
+  return inPublicApi(context) && notInGroup(NoRedactionGroup)(context)
+}
+
+export type { RedactionPolicy }

src/schema/common/redactionPolicy.json

@@ -0,0 +1,32 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "RedactionPolicy",
+  "type": "object",
+  "properties": {
+    "name": {
+      "type": "string"
+    },
+    "items": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/RedactionPolicyItem"
+      }
+    }
+  },
+  "required": ["name", "items"],
+  "definitions": {
+    "RedactionPolicyItem": {
+      "type": "object",
+      "properties": {
+        "jsonPath": {
+          "type": "string"
+        },
+        "valueConverterName": {
+          "type": "string",
+          "enum": ["redact", "contextNotAllowedImage", "remove", "emptyArray"]
+        }
+      },
+      "required": ["jsonPath", "valueConverterName"]
+    }
+  }
+}

src/schema/schemas/Topic.json

@@ -47,6 +47,10 @@
           "w": {
             "type": "number",
             "description": "TODO"
+          },
+          "avg": {
+            "type": "number",
+            "description": "TODO"
           }
         },
         "required": ["uid", "w"]

src/services/articles/articles.hooks.js

@@ -1,5 +1,7 @@
 import { rateLimit } from '../../hooks/rateLimiter'
 import { authenticateAround as authenticate } from '../../hooks/authenticate'
+import { redactResponse, redactResponseDataItem, defaultCondition } from '../../hooks/redaction'
+import { loadYamlFile } from '../../util/yaml'
 
 const {
   utils,
@@ -17,6 +19,8 @@ const { resolveTopics, resolveUserAddons } = require('../../hooks/resolvers/arti
 const { obfuscate } = require('../../hooks/access-rights')
 const { SolrMappings } = require('../../data/constants')
 
+const articleRedactionPolicy = loadYamlFile(`${__dirname}/resources/articleRedactionPolicy.yml`)
+
 module.exports = {
   around: {
     all: [authenticate({ allowUnauthenticated: true }), rateLimit()],
@@ -90,6 +94,7 @@ module.exports = {
       resolveTopics(),
       saveResultsInCache(),
       obfuscate(),
+      redactResponseDataItem(articleRedactionPolicy, defaultCondition),
     ],
     get: [
       // save here cache, flush cache here
@@ -100,6 +105,7 @@ module.exports = {
       saveResultsInCache(),
       resolveUserAddons(),
       obfuscate(),
+      redactResponse(articleRedactionPolicy, defaultCondition),
     ],
     create: [],
     update: [],

src/services/articles/resources/articleRedactionPolicy.yml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=../../../schema/common/redactionPolicy.json
+name: artice-redaction-policy
+items:
+  - jsonPath: $.title
+    valueConverterName: redact
+  - jsonPath: $.excerpt
+    valueConverterName: redact
+  - jsonPath: $.content
+    valueConverterName: redact
+  - jsonPath: $.regions
+    valueConverterName: emptyArray
+  - jsonPath: $.matches
+    valueConverterName: emptyArray
+  - jsonPath: $.pages[*].iiif
+    valueConverterName: contextNotAllowedImage
+  - jsonPath: $.pages[*].iiifThumbnail
+    valueConverterName: contextNotAllowedImage

src/services/search/search.hooks.js

@@ -1,5 +1,7 @@
 import { authenticateAround as authenticate } from '../../hooks/authenticate'
 import { rateLimit } from '../../hooks/rateLimiter'
+import { redactResponseDataItem, defaultCondition } from '../../hooks/redaction'
+import { loadYamlFile } from '../../util/yaml'
 
 const { protect } = require('@feathersjs/authentication-local').hooks
 const {
@@ -16,6 +18,8 @@ const { paramsValidator, eachFilterValidator, eachFacetFilterValidator } = requi
 const { SolrMappings } = require('../../data/constants')
 const { SolrNamespaces } = require('../../solr')
 
+const articleRedactionPolicy = loadYamlFile(`${__dirname}/../articles/resources/articleRedactionPolicy.yml`)
+
 module.exports = {
   around: {
     find: [authenticate({ allowUnauthenticated: true }), rateLimit()],
@@ -93,7 +97,12 @@ module.exports = {
 
   after: {
     all: [],
-    find: [displayQueryParams(['queryComponents', 'filters']), resolveQueryComponents(), protect('content')],
+    find: [
+      displayQueryParams(['queryComponents', 'filters']),
+      resolveQueryComponents(),
+      protect('content'),
+      redactResponseDataItem(articleRedactionPolicy, defaultCondition),
+    ],
     get: [],
     create: [],
     update: [],