diff --git a/config/default.json b/config/default.json index c7456c42..2e3290ab 100644 --- a/config/default.json +++ b/config/default.json @@ -179,6 +179,20 @@ "enableSentryMonitoring": false, "sentryDsn": "", "segmentApiKey": "", + "rateLimit": { + "disable": false, + "ttlSeconds": 60, + "threshold": 60, + "getTtlSeconds":60, + "getThreshold":200, + "createProjectTtlSeconds":86400, + "createProjectThreshold":10, + "createDonationTtlSeconds":3600, + "createDonationThreshold":300, + "createAuthenticationTtlSeconds":60, + "createAuthenticationThreshold":10 + }, + "givethIoUrl": "https://serve.giveth.io/graphql", "givethIoProjectsReviewerAddress": "0x90F8bf6A479f320ead074411a4B0e7944Ea8c9C1" } diff --git a/config/test.json b/config/test.json index fe980b24..bbe88565 100644 --- a/config/test.json +++ b/config/test.json @@ -84,7 +84,6 @@ { "name": "ANT", "address": "0xe78A0F7E598Cc8b0Bb87894B0F60dD2a88d6a8Ab", - "foreignAddress": "0x8F086f895deBc23473dfe507dd4BF35D6184552c", "foreignAddress": "0x7283b97f7546ba8eff68167cab2da526e7d7f88a", "symbol": "ANT", "coingeckoId": "aragon", @@ -187,6 +186,9 @@ "dappMailerUrl": "https://fake.dappmailer.giveth.io", "dappMailerSecret": "fakeSecret", "enablePayoutEmail": true, + "rateLimit": { + "disable": true + }, "givethIoProjectsReviewerAddress": "0xd00cc82a132f421bA6414D196BC830Db95e2e7Dd", "mockGivethIo": true } diff --git a/docker-compose-develop.yml b/docker-compose-develop.yml index 5b5e0155..9caaec33 100644 --- a/docker-compose-develop.yml +++ b/docker-compose-develop.yml @@ -13,6 +13,7 @@ services: environment: - logDir=/usr/src/app/logs - NODE_ENV=develop + - LOG_LEVEL=info volumes: # You should have a develop.json file in the config folder - type: bind diff --git a/docker-compose-production.yml b/docker-compose-production.yml index 5707f931..a2fe773f 100644 --- a/docker-compose-production.yml +++ b/docker-compose-production.yml @@ -13,6 +13,7 @@ services: environment: - logDir=/usr/src/app/logs - NODE_ENV=production + - LOG_LEVEL=info volumes: # You should have a production.json file in the config folder - type: bind diff --git a/package-lock.json b/package-lock.json index 63410a55..f0f2c3c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12325,15 +12325,16 @@ "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" }, "ioredis": { - "version": "4.24.5", - "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.24.5.tgz", - "integrity": "sha512-a1uk8WXM4Xe9bfYUJH17Up9ODEASjYCWiD/BKojPHp5YDDMX/QBOWxgSmrtpRE+ARdLYUoXSLeyGZyegLdHcOg==", + "version": "4.27.7", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.7.tgz", + "integrity": "sha512-lqvFFmUyGIHlrNyDvBoakzy1+ioJzNyoP6CP97GWtdTjWq9IOAnv6l0HUTsqhvd/z9etGgtrDHZ4kWCMAwNkug==", "requires": { "cluster-key-slot": "^1.1.0", "debug": "^4.3.1", "denque": "^1.1.0", "lodash.defaults": "^4.2.0", "lodash.flatten": "^4.4.0", + "lodash.isarguments": "^3.1.0", "p-map": "^2.1.0", "redis-commands": "1.7.0", "redis-errors": "^1.2.0", @@ -12342,9 +12343,9 @@ }, "dependencies": { "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", "requires": { "ms": "2.1.2" } @@ -14570,8 +14571,7 @@ "lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", - "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=", - "dev": true + "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=" }, "lodash.isarray": { "version": "3.0.4", @@ -18618,6 +18618,11 @@ "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==" }, + "rate-limiter-flexible": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/rate-limiter-flexible/-/rate-limiter-flexible-2.2.4.tgz", + "integrity": "sha512-8u4k5b1afuBcfydX0L0l3J2PNjgcuo3zua8plhvIisyDqOBldrCwfSFut/Fj00LAB1nxJYVM9jeszr2rZyDhQw==" + }, "raw-body": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz", diff --git a/package.json b/package.json index 6bcf9528..0bc935e1 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ "graphql": "^15.5.1", "graphql-request": "^3.5.0", "helmet": "^3.8.1", + "ioredis": "^4.27.7", "ipfs-api": "^24.0.0", "is-ipfs": "^0.4.2", "json2csv": "^4.5.4", @@ -104,6 +105,7 @@ "multer": "^1.3.0", "nyc": "^15.1.0", "passport-strategy": "^1.0.0", + "rate-limiter-flexible": "^2.2.4", "request-promise": "^4.2.2", "sanitize-html": "^2.4.0", "semaphore": "^1.1.0", diff --git a/src/app.hooks.js b/src/app.hooks.js index 1cd70fb7..062a360e 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -1,11 +1,19 @@ // Application hooks that run for every service const auth = require('@feathersjs/authentication'); +const config = require('config'); const { discard } = require('feathers-hooks-common'); const { NotAuthenticated } = require('@feathersjs/errors'); const { DonationStatus } = require('./models/donations.model'); const { isRequestInternal } = require('./utils/feathersUtils'); const { responseLoggerHook, startMonitoring } = require('./hooks/logger'); +const { rateLimit } = require('./utils/rateLimit'); +const { + getTtlSeconds, + getThreshold, + threshold: rateLimitThreshold, + ttlSeconds: rateLimitTtlSeconds, +} = config.rateLimit; const authenticate = () => context => { // No need to authenticate internal calls if (isRequestInternal(context)) return context; @@ -50,12 +58,41 @@ const convertVerifiedToBoolean = () => context => { module.exports = { before: { all: [startMonitoring()], - find: [convertVerifiedToBoolean()], - get: [], + find: [ + convertVerifiedToBoolean(), + rateLimit({ + threshold: getThreshold, + ttl: getTtlSeconds, + }), + ], + get: [ + rateLimit({ + threshold: getThreshold, + ttl: getTtlSeconds, + }), + ], create: [authenticate()], - update: [authenticate()], - patch: [authenticate()], - remove: [authenticate()], + update: [ + authenticate(), + rateLimit({ + threshold: rateLimitThreshold, + ttl: rateLimitTtlSeconds, + }), + ], + patch: [ + authenticate(), + rateLimit({ + threshold: rateLimitThreshold, + ttl: rateLimitTtlSeconds, + }), + ], + remove: [ + authenticate(), + rateLimit({ + threshold: rateLimitThreshold, + ttl: rateLimitTtlSeconds, + }), + ], }, after: { diff --git a/src/authentication.js b/src/authentication.js index 0a6ecc13..eb654afd 100644 --- a/src/authentication.js +++ b/src/authentication.js @@ -1,7 +1,9 @@ const { JWTStrategy } = require('@feathersjs/authentication'); const { expressOauth } = require('@feathersjs/authentication-oauth'); +const config = require('config'); const { MyAuthenticationService } = require('./authenticationService'); const { Web3Strategy } = require('./Web3Strategy'); +const { rateLimit } = require('./utils/rateLimit'); module.exports = app => { const authentication = new MyAuthenticationService(app); @@ -43,5 +45,17 @@ module.exports = app => { }; app.use('/authentication', authentication); + const hooks = { + before: { + create: [ + rateLimit({ + threshold: config.rateLimit.createAuthenticationThreshold, + ttl: config.rateLimit.createAuthenticationTtlSeconds, + }), + ], + }, + }; + app.service('authentication').hooks(hooks); + app.configure(expressOauth()); }; diff --git a/src/hooks/logger.js b/src/hooks/logger.js index 71788568..d7a86761 100644 --- a/src/hooks/logger.js +++ b/src/hooks/logger.js @@ -14,7 +14,9 @@ const startMonitoring = () => context => { !config.enableSentryMonitoring || isRequestInternal(context) || // internal calls that use the external context doesnt have headers - !context.params.headers + !context.params.headers || + // for requests that use _populate it will fill after first call + context.params._populate ) return context; const transaction = Sentry.startTransaction({ diff --git a/src/services/analytics/analytics.hooks.js b/src/services/analytics/analytics.hooks.js new file mode 100644 index 00000000..a358ec6c --- /dev/null +++ b/src/services/analytics/analytics.hooks.js @@ -0,0 +1,14 @@ +const config = require('config'); + +const { rateLimit } = require('../../utils/rateLimit'); + +module.exports = { + before: { + create: [ + rateLimit({ + threshold: config.rateLimit.threshold, + ttl: config.rateLimit.ttlSeconds, + }), + ], + }, +}; diff --git a/src/services/analytics/analytics.service.js b/src/services/analytics/analytics.service.js index e7599f53..0da002a1 100644 --- a/src/services/analytics/analytics.service.js +++ b/src/services/analytics/analytics.service.js @@ -1,4 +1,5 @@ const { sendAnalytics } = require('../../utils/analyticsUtils'); +const hooks = require('./analytics.hooks'); module.exports = function analytics() { const app = this; @@ -8,5 +9,40 @@ module.exports = function analytics() { return result; }, }; + + analyticsService.docs = { + operations: { + update: false, + patch: false, + remove: false, + find: false, + create: { + description: + 'This is for sending analytics event, Brave block analytics, so the gievth-dapp send analytic event by feathers-giveth in this case', + }, + }, + definition: { + type: 'object', + properties: { + properties: { + type: 'object', + }, + userId: { + type: 'string', + }, + event: { + type: 'string', + }, + anonymousId: { + type: 'string', + }, + reportType: { + type: 'string', + enum: ['track', 'page'], + }, + }, + }, + }; app.use('/analytics', analyticsService); + app.service('analytics').hooks(hooks); }; diff --git a/src/services/campaigns/campaigns.hooks.js b/src/services/campaigns/campaigns.hooks.js index 5b7cbd89..75d34d50 100644 --- a/src/services/campaigns/campaigns.hooks.js +++ b/src/services/campaigns/campaigns.hooks.js @@ -1,6 +1,8 @@ const commons = require('feathers-hooks-common'); const errors = require('@feathersjs/errors'); +const config = require('config'); +const { rateLimit } = require('../../utils/rateLimit'); const sanitizeAddress = require('../../hooks/sanitizeAddress'); const setAddress = require('../../hooks/setAddress'); const sanitizeHtml = require('../../hooks/sanitizeHtml'); @@ -127,6 +129,12 @@ module.exports = { checkCampaignOwner(), sanitizeHtml('description'), createModelSlug('campaigns'), + + // We dont count failed requests so I put it in last before hook + rateLimit({ + threshold: config.rateLimit.createProjectThreshold, + ttl: config.rateLimit.createProjectTtlSeconds, + }), ], update: [commons.disallow()], patch: [ diff --git a/src/services/communities/communities.hooks.js b/src/services/communities/communities.hooks.js index d1b037d9..6a8ba852 100644 --- a/src/services/communities/communities.hooks.js +++ b/src/services/communities/communities.hooks.js @@ -1,6 +1,9 @@ const commons = require('feathers-hooks-common'); const errors = require('@feathersjs/errors'); const { restrictToOwner } = require('feathers-authentication-hooks'); +const config = require('config'); + +const { rateLimit } = require('../../utils/rateLimit'); const sanitizeAddress = require('../../hooks/sanitizeAddress'); const setAddress = require('../../hooks/setAddress'); const sanitizeHtml = require('../../hooks/sanitizeHtml'); @@ -101,6 +104,12 @@ module.exports = { sanitizeAddress('ownerAddress', { required: true, validate: true }), sanitizeHtml('description'), createModelSlug('communities'), + + // We dont count failed requests so I put it in last before hook + rateLimit({ + threshold: config.rateLimit.createProjectThreshold, + ttl: config.rateLimit.createProjectTtlSeconds, + }), ], update: [commons.disallow()], patch: [ diff --git a/src/services/conversations/conversations.hooks.js b/src/services/conversations/conversations.hooks.js index 582cdd6c..3c9dc8aa 100644 --- a/src/services/conversations/conversations.hooks.js +++ b/src/services/conversations/conversations.hooks.js @@ -3,6 +3,9 @@ const commons = require('feathers-hooks-common'); const { disallow } = require('feathers-hooks-common'); const errors = require('@feathersjs/errors'); const { getItems } = require('feathers-hooks-common'); +const config = require('config'); + +const { rateLimit } = require('../../utils/rateLimit'); const sanitizeAddress = require('../../hooks/sanitizeAddress'); const sanitizeHtml = require('../../hooks/sanitizeHtml'); const resolveFiles = require('../../hooks/resolveFiles'); @@ -183,7 +186,16 @@ module.exports = { all: [], find: [sanitizeAddress(['ownerAddress'])], get: [], - create: [restrictAndSetOwner(), checkMessageContext(), sanitizeHtml('message')], + create: [ + restrictAndSetOwner(), + checkMessageContext(), + sanitizeHtml('message'), + // We dont count failed requests so I put it in last before hook + rateLimit({ + threshold: config.rateLimit.threshold, + ttl: config.rateLimit.ttlSeconds, + }), + ], update: [disallow()], patch: [onlyInternal()], remove: [disallow()], diff --git a/src/services/conversionRates/conversionRates.hooks.js b/src/services/conversionRates/conversionRates.hooks.js index 9440164c..7fd1a77f 100644 --- a/src/services/conversionRates/conversionRates.hooks.js +++ b/src/services/conversionRates/conversionRates.hooks.js @@ -1,5 +1,7 @@ const { disallow } = require('feathers-hooks-common'); +const config = require('config'); +const { rateLimit } = require('../../utils/rateLimit'); const onlyInternal = require('../../hooks/onlyInternal'); const { getConversionRates, @@ -39,7 +41,12 @@ const findConversionRates = () => async context => { module.exports = { before: { all: [], - find: [], + find: [ + rateLimit({ + threshold: config.rateLimit.threshold, + ttl: config.rateLimit.ttlSeconds, + }), + ], get: [disallow()], create: [onlyInternal()], update: [disallow()], diff --git a/src/services/donations/donations.hooks.js b/src/services/donations/donations.hooks.js index ac5b765d..955f54b5 100644 --- a/src/services/donations/donations.hooks.js +++ b/src/services/donations/donations.hooks.js @@ -4,6 +4,9 @@ const errors = require('@feathersjs/errors'); const commons = require('feathers-hooks-common'); const logger = require('winston'); const { ObjectId } = require('mongoose').Types; +const config = require('config'); + +const { rateLimit } = require('../../utils/rateLimit'); const sanitizeAddress = require('../../hooks/sanitizeAddress'); const addConfirmations = require('../../hooks/addConfirmations'); @@ -505,6 +508,12 @@ module.exports = { updateMilestoneIfNotPledged(), addActionTakerAddress(), convertTokenToTokenAddress(), + + // We dont count failed requests so I put it in last before hook + rateLimit({ + threshold: config.rateLimit.createDonationThreshold, + ttl: config.rateLimit.createDonationTtlSeconds, + }), ], update: [commons.disallow()], patch: [ diff --git a/src/services/subscriptions/subscription.hooks.js b/src/services/subscriptions/subscription.hooks.js index e6011ea2..044dd6b6 100644 --- a/src/services/subscriptions/subscription.hooks.js +++ b/src/services/subscriptions/subscription.hooks.js @@ -1,5 +1,8 @@ const errors = require('@feathersjs/errors'); const commons = require('feathers-hooks-common'); +const config = require('config'); + +const { rateLimit } = require('../../utils/rateLimit'); const { ProjectTypes } = require('../../models/subscription.model'); const validatePayload = () => async context => { @@ -39,7 +42,15 @@ module.exports = { all: [], find: [], get: [], - create: [validatePayload()], + create: [ + validatePayload(), + + // We dont count failed requests so I put it in last before hook + rateLimit({ + threshold: config.rateLimit.threshold, + ttl: config.rateLimit.ttlSeconds, + }), + ], update: [commons.disallow()], patch: [commons.disallow()], remove: [commons.disallow()], diff --git a/src/services/traces/traces.hooks.js b/src/services/traces/traces.hooks.js index 169f5260..28f022d3 100644 --- a/src/services/traces/traces.hooks.js +++ b/src/services/traces/traces.hooks.js @@ -3,9 +3,11 @@ const BatchLoader = require('@feathers-plus/batch-loader'); const errors = require('@feathersjs/errors'); const logger = require('winston'); const { restrictToOwner } = require('feathers-authentication-hooks'); +const config = require('config'); -const { getResultsByKey, getUniqueKeys } = BatchLoader; +const { rateLimit } = require('../../utils/rateLimit'); +const { getResultsByKey, getUniqueKeys } = BatchLoader; const sanitizeAddress = require('../../hooks/sanitizeAddress'); const setAddress = require('../../hooks/setAddress'); const sanitizeHtml = require('../../hooks/sanitizeHtml'); @@ -346,6 +348,12 @@ module.exports = { sanitizeHtml('description'), convertTokenToTokenAddress(), createModelSlug('traces'), + + // We dont count failed requests so I put it in last before hook + rateLimit({ + threshold: config.rateLimit.createProjectThreshold, + ttl: config.rateLimit.createProjectTtlSeconds, + }), ], update: [ removeProtectedFields(), diff --git a/src/services/users/users.hooks.js b/src/services/users/users.hooks.js index 013c3c34..f642cfbd 100644 --- a/src/services/users/users.hooks.js +++ b/src/services/users/users.hooks.js @@ -1,7 +1,9 @@ const commons = require('feathers-hooks-common'); const { toChecksumAddress } = require('web3-utils'); const errors = require('@feathersjs/errors'); +const config = require('config'); +const { rateLimit } = require('../../utils/rateLimit'); const notifyOfChange = require('../../hooks/notifyOfChange'); const sanitizeAddress = require('../../hooks/sanitizeAddress'); const setAddress = require('../../hooks/setAddress'); @@ -85,7 +87,16 @@ module.exports = { all: [commons.discardQuery('$disableStashBefore')], find: [sanitizeAddress('address')], get: [normalizeId()], - create: [commons.discard('_id'), ...address], + create: [ + commons.discard('_id'), + ...address, + + // We dont count failed requests so I put it in last before hook + rateLimit({ + threshold: config.rateLimit.threshold, + ttl: config.rateLimit.ttlSeconds, + }), + ], update: [...restrict, commons.stashBefore()], patch: [...restrict, commons.stashBefore()], remove: [commons.disallow()], diff --git a/src/utils/rateLimit.js b/src/utils/rateLimit.js new file mode 100644 index 00000000..bb5cb8f8 --- /dev/null +++ b/src/utils/rateLimit.js @@ -0,0 +1,51 @@ +const config = require('config'); +const { RateLimiterRedis } = require('rate-limiter-flexible'); +const Redis = require('ioredis'); + +const redisClient = new Redis({ ...config.redis, enableOfflineQueue: false }); +const errors = require('@feathersjs/errors'); +const { isRequestInternal } = require('./feathersUtils'); + +/** + * + * @param options {ttl: number(seconds), threshold:number, errorMessage:string} + * @returns {function(*): *} + */ +const rateLimit = (options = {}) => { + const { threshold, ttl, errorMessage } = options; + /** + * @see {@link https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#create-rate-limiter-and-consume-points-on-every-request} + */ + const opts = { + storeClient: redisClient, + points: threshold, + duration: ttl, // Per second + }; + const rateLimiter = new RateLimiterRedis(opts); + + return async context => { + if ( + isRequestInternal(context) || + // internal calls that use the external context doesnt have headers + !context.params.headers || + // for requests that use _populate it will fill after first call + context.params._populate || + config.rateLimit.disable + ) { + // Should not count internal requests + return context; + } + const ip = context.params.headers['x-real-ip'] || context.params.headers.cookie; + // if we just use ip as key, can not use separate rate limit for separate web services + const key = `${context.path}-${context.method}-${ip}`; + try { + // await messageLimiter.consume(ip); + await rateLimiter.consume(key); + } catch (e) { + throw new errors.TooManyRequests(errorMessage || 'Too many requests'); + } + + return context; + }; +}; +module.exports = { rateLimit };