From 4e495a143330e5ecd8884f862278185546b96985 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Wed, 25 Aug 2021 09:29:45 +0430 Subject: [PATCH 1/5] Allow to delete multiple entries we using multiple remove and everytime we got error --- src/services/serviceCommons.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/serviceCommons.js b/src/services/serviceCommons.js index f9dc2da0..0b5f29d6 100644 --- a/src/services/serviceCommons.js +++ b/src/services/serviceCommons.js @@ -1,5 +1,5 @@ const defaultFeatherMongooseOptions = { - multi: ['patch'], + multi: ['patch', 'remove'], whitelist: [ '$exists', '$and', From 73ab4fbf3d46c16bf2f05fcdf16a9df26d56c3bb Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Wed, 25 Aug 2021 09:31:19 +0430 Subject: [PATCH 2/5] Add emails update and create to elastik search related to #597 --- src/auditLog/feathersElasticSearch.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/auditLog/feathersElasticSearch.js b/src/auditLog/feathersElasticSearch.js index a486893c..ab5ab3a1 100644 --- a/src/auditLog/feathersElasticSearch.js +++ b/src/auditLog/feathersElasticSearch.js @@ -67,6 +67,7 @@ const configureAuditLog = app => { setAuditLogToFeathersService({ app, serviceName: 'donations' }); setAuditLogToFeathersService({ app, serviceName: 'pledgeAdmins' }); setAuditLogToFeathersService({ app, serviceName: 'events' }); + setAuditLogToFeathersService({ app, serviceName: 'emails' }); }; module.exports = { From 43e0ec81fd1095b7cf867a81e81fd2fe1eae803d Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Wed, 25 Aug 2021 09:34:10 +0430 Subject: [PATCH 3/5] Remove jobs from queue if it doesnt remove after two minutes (prevent blocking queue) related to #594 --- src/blockchain/lib/eventHandlerQueue.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/blockchain/lib/eventHandlerQueue.js b/src/blockchain/lib/eventHandlerQueue.js index 93886215..d5564703 100644 --- a/src/blockchain/lib/eventHandlerQueue.js +++ b/src/blockchain/lib/eventHandlerQueue.js @@ -10,6 +10,7 @@ const { EventStatus } = require('../../models/events.model'); const handleEventQueue = new Queue('eventHandler', { redis: config.get('redis') }); const pendingEventQueue = new Queue('NewEventQueue', { redis: config.get('redis') }); +const TWO_MINUTES = 1000 * 60 * 2; setInterval(async () => { const eventHandlerQueueCount = await handleEventQueue.count(); @@ -18,7 +19,7 @@ setInterval(async () => { eventHandlerQueueCount, NewEventQueueCount, }); -}, 1000 * 60 * 2); +}, TWO_MINUTES); const removeEvent = async (app, event) => { const { id, transactionHash } = event; @@ -155,9 +156,17 @@ const initEventHandlerQueue = app => { handleEventQueue.process(1, async (job, done) => { const { event } = job.data; + const callDoneTimeout = setTimeout(() => { + logger.error('The event handler didnt respond, call done() to prevent stocking queue'); + done(); + }, TWO_MINUTES); + try { + const remainingEventsInQueue = await handleEventQueue.count(); const handler = handlers[event.event]; + logger.info('Handling Event: ', { + remainingEventsInQueue, event: event.event, transactionHash: event.transactionHash, status: event.status, @@ -181,6 +190,7 @@ const initEventHandlerQueue = app => { processingError: e.toString(), }); } finally { + clearTimeout(callDoneTimeout); done(); } }); From 8cd0dca7c7bd705a0a18a1dcedbf5f526f6a8c10 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Wed, 25 Aug 2021 09:35:57 +0430 Subject: [PATCH 4/5] Dont process jobs multiple times if the event status is Processed already related to #594 --- src/blockchain/lib/eventHandlerQueue.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/blockchain/lib/eventHandlerQueue.js b/src/blockchain/lib/eventHandlerQueue.js index d5564703..cf89dbe3 100644 --- a/src/blockchain/lib/eventHandlerQueue.js +++ b/src/blockchain/lib/eventHandlerQueue.js @@ -174,6 +174,18 @@ const initEventHandlerQueue = app => { logIndex: event.logIndex, _id: event._id, }); + const eventInDb = await eventService.get(event._id); + if (eventInDb === EventStatus.PROCESSED) { + logger.info('Event is already processed, so dont need to handle again', { + event: event.event, + _id: event._id, + transactionHash: event.transactionHash, + transactionIndex: event.transactionIndex, + }); + clearTimeout(callDoneTimeout); + done(); + return; + } if (typeof handler === 'function') { await handler(event); } else { From 0f636e8c3a0e338bc2cb7c2e13b072aca2f18ef6 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Wed, 25 Aug 2021 13:54:39 +0430 Subject: [PATCH 5/5] Fix bug in check eventInDb status --- src/blockchain/lib/eventHandlerQueue.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blockchain/lib/eventHandlerQueue.js b/src/blockchain/lib/eventHandlerQueue.js index 1d7aafee..139b88a6 100644 --- a/src/blockchain/lib/eventHandlerQueue.js +++ b/src/blockchain/lib/eventHandlerQueue.js @@ -175,7 +175,7 @@ const initEventHandlerQueue = app => { _id: event._id, }); const eventInDb = await eventService.get(event._id); - if (eventInDb === EventStatus.PROCESSED) { + if (eventInDb.status === EventStatus.PROCESSED) { logger.info('Event is already processed, so dont need to handle again', { event: event.event, _id: event._id,