From fdb283243d5df40988c13af08252f9a0f7833097 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Wed, 9 Jun 2021 22:37:16 +0430 Subject: [PATCH 01/47] Add /verifiedProjectsGiversReport --- package.json | 1 + src/models/traces.model.js | 1 + src/repositories/donationRepository.js | 100 ++++++++++++++++++ src/services/index.js | 2 + src/services/traces/traces.hooks.js | 11 ++ src/services/traces/traces.service.test.js | 63 +++++------ .../verifiedPerojectsGiversReport.service.js | 41 +++++++ ...ifiedPerojectsGiversReport.service.test.js | 41 +++++++ 8 files changed, 229 insertions(+), 31 deletions(-) create mode 100644 src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js create mode 100644 src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.test.js diff --git a/package.json b/package.json index 92a01ddb..a3ee53d8 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "memory-cache": "^0.2.0", "migrate-mongo": "^8.1.4", "mkdirp": "^0.5.1", + "moment": "^2.29.1", "mongodb": "^3.6.3", "mongoose": "^5.11.9", "mongoose-long": "^0.3.2", diff --git a/src/models/traces.model.js b/src/models/traces.model.js index e9f0248e..c03b56ca 100644 --- a/src/models/traces.model.js +++ b/src/models/traces.model.js @@ -74,6 +74,7 @@ function Milestone(app) { donationCounters: [DonationCounter], peopleCount: { type: Number }, mined: { type: Boolean, required: true, default: false }, + verified: { type: Boolean, default: false }, prevStatus: { type: String }, url: { type: String }, customThanksMessage: { type: String }, diff --git a/src/repositories/donationRepository.js b/src/repositories/donationRepository.js index 72570576..fbb9c1d2 100644 --- a/src/repositories/donationRepository.js +++ b/src/repositories/donationRepository.js @@ -45,8 +45,108 @@ const isAllDonationsPaidOut = async (app, { txHash, traceId }) => { return notPaidOutDonationsCount === 0; }; +/** + * + * @param app: feathers instance + * @param from: Date, example: 2021-06-08T16:05:28.005Z + * @param to: Date: example: 2021-06-08T16:05:28.005Z + * @param projectIds: Array, example: [1340, 2723] + * @returns { + * Promise< + [{ + "_id" : "0x3ba20bC27c2B65C98A5aBF922a34e6Eb3108A9CB", + "totalAmount" : 10.43, + "donations" : [ + { + donation + } + ], + "giver" : { + //userInfo + } + }] + >} + */ +const listOfUserDonorsOnVerifiedProjects = async (app, { projectIds, from, to }) => { + const donationModel = app.service('donations').Model; + return donationModel.aggregate([ + { + $match: { + status: { + $in: ['Waiting', 'Committed'], + }, + createdAt: { + $gte: from, + $lte: to, + }, + + homeTxHash: { $exists: true }, + $or: [ + { + // it's for communities + delegateId: { $in: projectIds }, + intendedProjectId: { $exists: false }, + }, + + // it's for traces and campaigns + { ownerId: { $in: projectIds } }, + ], + amount: { $ne: '0' }, + usdValue: { $ne: 0 }, + isReturn: false, + }, + }, + + { + $group: { + _id: '$giverAddress', + totalAmount: { $sum: '$usdValue' }, + donationIds: { $push: '$_id' }, + }, + }, + + { + $lookup: { + from: 'users', + let: { giverAddress: '$_id' }, + pipeline: [ + { + $match: { $expr: { $eq: ['$address', '$$giverAddress'] } }, + }, + ], + as: 'giver', + }, + }, + { + $unwind: '$giver', + }, + { + $lookup: { + from: 'donations', + let: { donationIds: '$donationIds' }, + pipeline: [ + { + $match: { $expr: { $in: ['$_id', '$$donationIds'] } }, + }, + ], + as: 'donations', + }, + }, + { + $project: { + // giverAddress: '$_id', + donations: 1, + giver: 1, + totalAmount: 1, + _id: 0, + }, + }, + ]); +}; + module.exports = { updateBridgePaymentExecutedTxHash, updateBridgePaymentAuthorizedTxHash, isAllDonationsPaidOut, + listOfUserDonorsOnVerifiedProjects, }; diff --git a/src/services/index.js b/src/services/index.js index 622d2d6b..6beb8fd1 100644 --- a/src/services/index.js +++ b/src/services/index.js @@ -18,6 +18,7 @@ const whitelist = require('./whitelist/whitelist.service.js'); const gasprice = require('./gasprice/gasprice.service.js'); const conversionRates = require('./conversionRates/conversionRates.service.js'); const conversations = require('./conversations/conversations.service.js'); +const givbackReportDonations = require('./verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service'); module.exports = function configure() { const app = this; @@ -40,4 +41,5 @@ module.exports = function configure() { app.configure(subscription); app.configure(conversations); app.configure(campaigncsv); + app.configure(givbackReportDonations); }; diff --git a/src/services/traces/traces.hooks.js b/src/services/traces/traces.hooks.js index e5e4f254..6fd24fa6 100644 --- a/src/services/traces/traces.hooks.js +++ b/src/services/traces/traces.hooks.js @@ -24,6 +24,14 @@ const { getBlockTimestamp, ZERO_ADDRESS } = require('../../blockchain/lib/web3He const { getTokenByAddress } = require('../../utils/tokenHelper'); const createModelSlug = require('../createModelSlug'); +const removeProtectedFields = async context => { + return context; + // if (context && context.data) { + // delete context.data.verified; + // } + // return context; +}; + const traceResolvers = { before: context => { context._loaders = { @@ -327,6 +335,7 @@ module.exports = { ], get: [], create: [ + removeProtectedFields(), checkConversionRates(), checkTraceDates(), checkTraceName(), @@ -339,6 +348,7 @@ module.exports = { createModelSlug('traces'), ], update: [ + removeProtectedFields(), restrict(), checkTraceDates(), ...address, @@ -347,6 +357,7 @@ module.exports = { checkTraceName(), ], patch: [ + removeProtectedFields(), restrict(), sanitizeAddress( ['pluginAddress', 'reviewerAddress', 'campaignReviewerAddress', 'recipientAddress'], diff --git a/src/services/traces/traces.service.test.js b/src/services/traces/traces.service.test.js index 611d5d95..b3053e9d 100644 --- a/src/services/traces/traces.service.test.js +++ b/src/services/traces/traces.service.test.js @@ -8,7 +8,7 @@ let app; const baseUrl = config.get('givethFathersBaseUrl'); const relativeUrl = '/traces'; -async function createMilestone(data) { +async function createTrace(data) { const response = await request(baseUrl) .post(relativeUrl) .send(data) @@ -17,13 +17,13 @@ async function createMilestone(data) { } function getMilestoneTestCases() { - it('should get successful result', async function() { + it('should get successful result', async () => { const response = await request(baseUrl).get(relativeUrl); assert.equal(response.statusCode, 200); assert.exists(response.body.data); assert.notEqual(response.body.data.length, 0); }); - it('getMileStoneDetail', async function() { + it('getMileStoneDetail', async () => { const response = await request(baseUrl).get(`${relativeUrl}/${SAMPLE_DATA.TRACE_ID}`); assert.equal(response.statusCode, 200); assert.equal(response.body.ownerAddress, SAMPLE_DATA.USER_ADDRESS); @@ -31,7 +31,7 @@ function getMilestoneTestCases() { } function postMilestoneTestCases() { - it('should create milestone successfully', async () => { + it('should create trace successfully', async () => { const response = await request(baseUrl) .post(relativeUrl) .send(SAMPLE_DATA.createTraceData()) @@ -39,7 +39,7 @@ function postMilestoneTestCases() { assert.equal(response.statusCode, 201); assert.equal(response.body.ownerAddress, SAMPLE_DATA.USER_ADDRESS); }); - it('should create milestone successfully including category', async () => { + it('should create trace successfully including category', async () => { const formType = 'expense'; const response = await request(baseUrl) .post(relativeUrl) @@ -50,10 +50,10 @@ function postMilestoneTestCases() { assert.equal(response.body.formType, formType); }); - it('should create milestone , token must be returned', async function() { - // In milestone hooks based on token.symbol set a tokenSymbol field - // in milestone, and after all http methods hook when returning milestone - // add token to milestone based on tokenSymbol + it('should create trace , token must be returned', async () => { + // In trace hooks based on token.symbol set a tokenSymbol field + // in trace, and after all http methods hook when returning trace + // add token to trace based on tokenSymbol const ethToken = config.get('tokenWhitelist').find(token => token.symbol === 'ETH'); const response = await request(baseUrl) @@ -81,7 +81,7 @@ function postMilestoneTestCases() { assert.equal(response.statusCode, 401); assert.equal(response.body.code, 401); }); - it('should get different slugs for two traces with same title successfully', async function() { + it('should get different slugs for two traces with same title successfully', async () => { const response1 = await request(baseUrl) .post(relativeUrl) .send(SAMPLE_DATA.createTraceData()) @@ -94,10 +94,11 @@ function postMilestoneTestCases() { assert.isNotNull(response2.body.slug); assert.notEqual(response1.body.slug, response2.body.slug); }); + } function patchMilestoneTestCases() { - it('should update milestone successfully', async function() { + it('should update trace successfully', async () => { const description = 'Description updated by test'; const response = await request(baseUrl) .patch(`${relativeUrl}/${SAMPLE_DATA.TRACE_ID}`) @@ -107,7 +108,7 @@ function patchMilestoneTestCases() { assert.equal(response.body.description, description); }); - it('should not update milestone because status not sent in payload', async function() { + it('should not update trace because status not sent in payload', async () => { const description = String(new Date()); const response = await request(baseUrl) .patch(`${relativeUrl}/${SAMPLE_DATA.TRACE_ID}`) @@ -117,7 +118,7 @@ function patchMilestoneTestCases() { assert.notEqual(response.body.description, description); }); - it('should not update , because data that stored on-chain cant be updated', async function() { + it('should not update , because data that stored on-chain cant be updated', async () => { const updateData = { // this should exists otherwise without status mileston should not updated status: SAMPLE_DATA.TRACE_STATUSES.IN_PROGRESS, @@ -158,7 +159,7 @@ function patchMilestoneTestCases() { assert.notEqual(response.body.type, updateData.type); assert.notEqual(response.body.token, updateData.token); }); - it('should get unAuthorized error', async function() { + it('should get unAuthorized error', async () => { const response = await request(baseUrl) .patch(`${relativeUrl}/${SAMPLE_DATA.TRACE_ID}`) .send(SAMPLE_DATA.createTraceData()); @@ -166,7 +167,7 @@ function patchMilestoneTestCases() { assert.equal(response.body.code, 401); }); - it('should get unAuthorized error because Only the Milestone and Campaign Manager can edit milestone', async function() { + it('should get unAuthorized error because Only the Milestone and Campaign Manager can edit trace', async () => { const description = 'Description updated by test'; const response = await request(baseUrl) .patch(`${relativeUrl}/${SAMPLE_DATA.TRACE_ID}`) @@ -178,7 +179,7 @@ function patchMilestoneTestCases() { } function deleteMilestoneTestCases() { - it('should not delete because status is not Proposed or Rejected ', async function() { + it('should not delete because status is not Proposed or Rejected ', async () => { const statusThatCantBeDeleted = [ SAMPLE_DATA.TRACE_STATUSES.IN_PROGRESS, SAMPLE_DATA.TRACE_STATUSES.ARCHIVED, @@ -196,61 +197,61 @@ function deleteMilestoneTestCases() { createMileStoneData.status = status; createMileStoneData.ownerAddress = SAMPLE_DATA.USER_ADDRESS; - const milestone = await createMilestone(createMileStoneData); + const trace = await createTrace(createMileStoneData); const response = await request(baseUrl) - .delete(`${relativeUrl}/${milestone._id}`) + .delete(`${relativeUrl}/${trace._id}`) .set({ Authorization: getJwt() }); assert.equal(response.statusCode, 403); } }); - it('should be successful for milestone with status Proposed', async function() { + it('should be successful for deleting trace with status Proposed', async () => { const createMileStoneData = { ...SAMPLE_DATA.createTraceData() }; createMileStoneData.status = SAMPLE_DATA.TRACE_STATUSES.PROPOSED; createMileStoneData.ownerAddress = SAMPLE_DATA.USER_ADDRESS; - const milestone = await createMilestone(createMileStoneData); + const trace = await createTrace(createMileStoneData); const response = await request(baseUrl) - .delete(`${relativeUrl}/${milestone._id}`) + .delete(`${relativeUrl}/${trace._id}`) .set({ Authorization: getJwt() }); assert.equal(response.statusCode, 200); }); - it('should be successful for milestone with status Rejected', async function() { + it('should be successful for trace with status Rejected', async () => { const createMileStoneData = { ...SAMPLE_DATA.createTraceData() }; createMileStoneData.status = SAMPLE_DATA.TRACE_STATUSES.REJECTED; createMileStoneData.ownerAddress = SAMPLE_DATA.USER_ADDRESS; - const milestone = await createMilestone(createMileStoneData); + const trace = await createTrace(createMileStoneData); const response = await request(baseUrl) - .delete(`${relativeUrl}/${milestone._id}`) + .delete(`${relativeUrl}/${trace._id}`) .set({ Authorization: getJwt() }); assert.equal(response.statusCode, 200); }); - it("should get 403 , users cant delete other's milestone", async function() { + it("should get 403 , users cant delete other's trace", async () => { const createMileStoneData = { ...SAMPLE_DATA.createTraceData() }; createMileStoneData.status = SAMPLE_DATA.TRACE_STATUSES.REJECTED; createMileStoneData.ownerAddress = SAMPLE_DATA.USER_ADDRESS; - const milestone = await createMilestone(createMileStoneData); + const trace = await createTrace(createMileStoneData); const response = await request(baseUrl) - .delete(`${relativeUrl}/${milestone._id}`) + .delete(`${relativeUrl}/${trace._id}`) .set({ Authorization: getJwt(SAMPLE_DATA.SECOND_USER_ADDRESS) }); // TODO this testCase is for a bug, when bug fixed this testCase should fix and probably the status should be 403 instead of 200 assert.equal(response.statusCode, 403); }); - it('should be successful , delete Proposed milestone', async function() { + it('should be successful , delete Proposed trace', async () => { const createMileStoneData = { ...SAMPLE_DATA.createTraceData() }; createMileStoneData.status = SAMPLE_DATA.TRACE_STATUSES.REJECTED; createMileStoneData.ownerAddress = SAMPLE_DATA.USER_ADDRESS; - const milestone = await createMilestone(createMileStoneData); + const trace = await createTrace(createMileStoneData); const response = await request(baseUrl) - .delete(`${relativeUrl}/${milestone._id}`) + .delete(`${relativeUrl}/${trace._id}`) .set({ Authorization: getJwt(SAMPLE_DATA.USER_ADDRESS) }); // TODO this testCase is for a bug, when bug fixed this testCase should fix and probably the status should be 403 instead of 200 assert.equal(response.statusCode, 200); }); - it('should get unAuthorized error', async function() { + it('should get unAuthorized error', async () => { const response = await request(baseUrl).delete(`${relativeUrl}/${SAMPLE_DATA.TRACE_ID}`); assert.equal(response.statusCode, 401); assert.equal(response.body.code, 401); diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js new file mode 100644 index 00000000..5492fc7c --- /dev/null +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -0,0 +1,41 @@ +const moment = require('moment'); +const errors = require('@feathersjs/errors'); +const { listOfUserDonorsOnVerifiedProjects } = require('../../repositories/donationRepository'); + +module.exports = function aggregateDonations() { + const app = this; + + const givbackReportDonations = { + async find({ query }) { + const { fromDate, toDate, projectIds } = query; + if (!projectIds) { + throw new errors.BadRequest('projectIds are required'); + } + if (!fromDate) { + throw new errors.BadRequest('fromDate is required with this format: YYYY/MM/DD-hh:mm:ss'); + } + if (!toDate) { + throw new errors.BadRequest('toDate is required with this format: YYYY/MM/DD-hh:mm:ss'); + } + if (!projectIds) { + throw new errors.BadRequest('projectIds are required'); + } + const from = moment(fromDate, 'YYYY/MM/DD-hh:mm:ss').toDate(); + const to = moment(toDate, 'YYYY/MM/DD-hh:mm:ss').toDate(); + + const result = await listOfUserDonorsOnVerifiedProjects(app, { + projectIds: projectIds.split(',').map(projectId => Number(projectId)), + from, + to, + }); + + return { + total: result.length, + from, + to, + data: result, + }; + }, + }; + app.use('/verifiedProjectsGiversReport', givbackReportDonations); +}; diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.test.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.test.js new file mode 100644 index 00000000..57b5aa20 --- /dev/null +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.test.js @@ -0,0 +1,41 @@ +const request = require('supertest'); +const config = require('config'); +const { assert } = require('chai'); + +const baseUrl = config.get('givethFathersBaseUrl'); +const relativeUrl = '/verifiedProjectsGiversReport'; + +function getGasPriceTestCases() { + it('get error if projectIds didnt send', async () => { + const response = await request(baseUrl).get(relativeUrl); + assert.equal(response.statusCode, 400); + assert.equal(response.body.message, 'projectIds are required'); + }); + + it('get error if fromDate didnt send', async () => { + const response = await request(baseUrl).get(`${relativeUrl}?projectIds=1,2,3,4,5,22`); + assert.equal(response.statusCode, 400); + assert.equal( + response.body.message, + 'fromDate is required with this format: YYYY/MM/DD-hh:mm:ss', + ); + }); + it('get error if toDate didnt send', async () => { + const response = await request(baseUrl).get( + `${relativeUrl}?projectIds=1,2,3,4,5,22&fromDate=2018/01/01-00:00:00`, + ); + assert.equal(response.statusCode, 400); + assert.equal(response.body.message, 'toDate is required with this format: YYYY/MM/DD-hh:mm:ss'); + }); + it('get success response', async () => { + const response = await request(baseUrl).get( + `${relativeUrl}?projectIds=1,2,3,4,5,22&fromDate=2020/01/01-00:00:00&toDate=2020/01/01-00:00:00`, + ); + assert.equal(response.statusCode, 200); + assert.exists(response.body.total); + assert.exists(response.body.data); + assert.isArray(response.body.data); + }); +} + +describe(`Test GET ${relativeUrl}`, getGasPriceTestCases); From 04079274131e3a881e962beeae50fce2c66d9f0b Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 12 Jun 2021 20:50:25 +0430 Subject: [PATCH 02/47] Add traceRepository, change Api /verifiedProjectsGiversReport output related to Giveth/giveth-dapp#2253 --- src/models/campaigns.model.js | 1 + src/models/communities.model.js | 1 + src/repositories/campaignRepository.js | 9 +++ src/repositories/communityRepository.js | 7 +++ src/repositories/donationRepository.js | 55 +++++++++---------- src/repositories/traceRepository.js | 18 ++++++ src/services/campaigns/campaigns.hooks.js | 10 ++++ .../campaigns/campaigns.service.test.js | 9 +++ src/services/communities/communities.hooks.js | 10 ++++ .../communities/communities.service.test.js | 10 ++++ src/services/traces/traces.hooks.js | 10 ++-- src/services/traces/traces.service.test.js | 7 +++ .../verifiedPerojectsGiversReport.service.js | 34 +++++++++--- 13 files changed, 138 insertions(+), 43 deletions(-) create mode 100644 src/repositories/campaignRepository.js create mode 100644 src/repositories/traceRepository.js diff --git a/src/models/campaigns.model.js b/src/models/campaigns.model.js index 0fc2edc2..874f8c12 100644 --- a/src/models/campaigns.model.js +++ b/src/models/campaigns.model.js @@ -33,6 +33,7 @@ function createModel(app) { pluginAddress: { type: String }, tokenAddress: { type: String }, mined: { type: Boolean, required: true, default: false }, + verified: { type: Boolean, default: false }, status: { type: String, require: true, diff --git a/src/models/communities.model.js b/src/models/communities.model.js index 2a259711..4afe9d0b 100644 --- a/src/models/communities.model.js +++ b/src/models/communities.model.js @@ -43,6 +43,7 @@ function createModel(app) { commitTime: { type: Number }, campaigns: { type: [String], default: [] }, mined: { type: Boolean }, + verified: { type: Boolean, default: false }, url: { type: String }, customThanksMessage: { type: String }, prevUrl: { type: String }, // To store deleted/cleared lost ipfs values diff --git a/src/repositories/campaignRepository.js b/src/repositories/campaignRepository.js new file mode 100644 index 00000000..33e17345 --- /dev/null +++ b/src/repositories/campaignRepository.js @@ -0,0 +1,9 @@ +const findVerifiedCampaigns = async app => { + const campaignsService = app.service('campaigns'); + const campaignsModel = campaignsService.Model; + return campaignsModel.find({ verified: true }); +}; + +module.exports = { + findVerifiedCampaigns, +}; diff --git a/src/repositories/communityRepository.js b/src/repositories/communityRepository.js index ecd96d42..7de79285 100644 --- a/src/repositories/communityRepository.js +++ b/src/repositories/communityRepository.js @@ -6,6 +6,13 @@ const findParentCommunities = async (app, { campaignId }) => { return communityModel.find({ campaigns: campaignId }); }; +const findVerifiedCommunities = async app => { + const communityService = app.service('communities'); + const communityModel = communityService.Model; + return communityModel.find({ verified: true }); +}; + module.exports = { findParentCommunities, + findVerifiedCommunities, }; diff --git a/src/repositories/donationRepository.js b/src/repositories/donationRepository.js index fbb9c1d2..74377781 100644 --- a/src/repositories/donationRepository.js +++ b/src/repositories/donationRepository.js @@ -48,26 +48,28 @@ const isAllDonationsPaidOut = async (app, { txHash, traceId }) => { /** * * @param app: feathers instance - * @param from: Date, example: 2021-06-08T16:05:28.005Z + * @param from: Date, example: 2018-06-08T16:05:28.005Z * @param to: Date: example: 2021-06-08T16:05:28.005Z * @param projectIds: Array, example: [1340, 2723] * @returns { * Promise< [{ - "_id" : "0x3ba20bC27c2B65C98A5aBF922a34e6Eb3108A9CB", - "totalAmount" : 10.43, + "giverAddress" : string, + "totalAmount" : number, "donations" : [ { - donation + "usdValue": number, + "amount": number, + "homeTxHash": string, + "createdAt": string // sample: "2019-04-22T22:14:23.046Z", + "token": string //sample : "ETH", + "projectId": number } - ], - "giver" : { - //userInfo - } + ] }] >} */ -const listOfUserDonorsOnVerifiedProjects = async (app, { projectIds, from, to }) => { +const listOfUserDonorsOnVerifiedProjects = async (app, { verifiedProjectIds, from, to }) => { const donationModel = app.service('donations').Model; return donationModel.aggregate([ { @@ -84,12 +86,12 @@ const listOfUserDonorsOnVerifiedProjects = async (app, { projectIds, from, to }) $or: [ { // it's for communities - delegateId: { $in: projectIds }, + delegateId: { $in: verifiedProjectIds }, intendedProjectId: { $exists: false }, }, // it's for traces and campaigns - { ownerId: { $in: projectIds } }, + { ownerId: { $in: verifiedProjectIds } }, ], amount: { $ne: '0' }, usdValue: { $ne: 0 }, @@ -104,22 +106,6 @@ const listOfUserDonorsOnVerifiedProjects = async (app, { projectIds, from, to }) donationIds: { $push: '$_id' }, }, }, - - { - $lookup: { - from: 'users', - let: { giverAddress: '$_id' }, - pipeline: [ - { - $match: { $expr: { $eq: ['$address', '$$giverAddress'] } }, - }, - ], - as: 'giver', - }, - }, - { - $unwind: '$giver', - }, { $lookup: { from: 'donations', @@ -128,15 +114,26 @@ const listOfUserDonorsOnVerifiedProjects = async (app, { projectIds, from, to }) { $match: { $expr: { $in: ['$_id', '$$donationIds'] } }, }, + { + $project: { + _id: 0, + createdAt: 1, + token: '$token.symbol', + usdValue: 1, + amount: 1, + homeTxHash: 1, + delegateId: 1, + ownerId: 1, + }, + }, ], as: 'donations', }, }, { $project: { - // giverAddress: '$_id', + giverAddress: '$_id', donations: 1, - giver: 1, totalAmount: 1, _id: 0, }, diff --git a/src/repositories/traceRepository.js b/src/repositories/traceRepository.js new file mode 100644 index 00000000..e17788db --- /dev/null +++ b/src/repositories/traceRepository.js @@ -0,0 +1,18 @@ +const { findVerifiedCampaigns } = require('./campaignRepository'); + +const findVerifiedTraces = async app => { + const tracesService = app.service('traces'); + const tracesModel = tracesService.Model; + const verifiedCampaigns = await findVerifiedCampaigns(app); + return tracesModel.find({ + projectId: { $nin: [0, -1, null] }, + $or: [ + { verified: true }, + { campaignId: { $in: verifiedCampaigns.map(campaign => String(campaign._id)) } }, + ], + }); +}; + +module.exports = { + findVerifiedTraces, +}; diff --git a/src/services/campaigns/campaigns.hooks.js b/src/services/campaigns/campaigns.hooks.js index 5bc07eab..c79cde09 100644 --- a/src/services/campaigns/campaigns.hooks.js +++ b/src/services/campaigns/campaigns.hooks.js @@ -9,6 +9,7 @@ const { checkReviewer, checkOwner } = require('../../hooks/isProjectAllowed'); const addConfirmations = require('../../hooks/addConfirmations'); const { CampaignStatus } = require('../../models/campaigns.model'); const createModelSlug = require('../createModelSlug'); +const { isRequestInternal } = require('../../utils/feathersUtils'); const schema = { include: [ @@ -70,6 +71,13 @@ const restrict = () => context => { ); }; +const removeProtectedFields = () => context => { + if (context && context.data && !isRequestInternal(context)) { + delete context.data.verified; + } + return context; +}; + const countTraces = (item, service) => service.Model.countDocuments({ campaignId: item._id, @@ -104,6 +112,7 @@ module.exports = { find: [sanitizeAddress('ownerAddress')], get: [], create: [ + removeProtectedFields(), setAddress('coownerAddress'), sanitizeAddress('coownerAddress', { required: true, @@ -121,6 +130,7 @@ module.exports = { ], update: [commons.disallow()], patch: [ + removeProtectedFields(), restrict(), sanitizeAddress('ownerAddress', { validate: true }), sanitizeHtml('description'), diff --git a/src/services/campaigns/campaigns.service.test.js b/src/services/campaigns/campaigns.service.test.js index 48702600..8e0d4b0d 100644 --- a/src/services/campaigns/campaigns.service.test.js +++ b/src/services/campaigns/campaigns.service.test.js @@ -40,6 +40,15 @@ function postCampaignTestCases() { assert.equal(response.statusCode, 201); assert.equal(response.body.ownerAddress, SAMPLE_DATA.CREATE_CAMPAIGN_DATA.ownerAddress); }); + + it('should create campaign successfully, should not set verified', async () => { + const response = await request(baseUrl) + .post(relativeUrl) + .send({ ...SAMPLE_DATA.CREATE_CAMPAIGN_DATA, verified: true }) + .set({ Authorization: getJwt(SAMPLE_DATA.CREATE_CAMPAIGN_DATA.ownerAddress) }); + assert.equal(response.statusCode, 201); + assert.isFalse(response.body.verified); + }); it('should get unAuthorized error', async () => { const response = await request(baseUrl) .post(relativeUrl) diff --git a/src/services/communities/communities.hooks.js b/src/services/communities/communities.hooks.js index 2f1e85c6..d1b037d9 100644 --- a/src/services/communities/communities.hooks.js +++ b/src/services/communities/communities.hooks.js @@ -8,6 +8,7 @@ const addConfirmations = require('../../hooks/addConfirmations'); const resolveFiles = require('../../hooks/resolveFiles'); const createModelSlug = require('../createModelSlug'); const { isUserInDelegateWhiteList } = require('../../utils/roleUtility'); +const { isRequestInternal } = require('../../utils/feathersUtils'); const restrict = [ context => commons.deleteByDot(context.data, 'txHash'), @@ -17,6 +18,13 @@ const restrict = [ }), ]; +const removeProtectedFields = () => context => { + if (context && context.data && !isRequestInternal(context)) { + delete context.data.verified; + } + return context; +}; + const schema = { include: [ { @@ -87,6 +95,7 @@ module.exports = { find: [sanitizeAddress('ownerAddress')], get: [], create: [ + removeProtectedFields(), setAddress('ownerAddress'), isDacAllowed(), sanitizeAddress('ownerAddress', { required: true, validate: true }), @@ -95,6 +104,7 @@ module.exports = { ], update: [commons.disallow()], patch: [ + removeProtectedFields(), ...restrict, sanitizeAddress('ownerAddress', { validate: true }), sanitizeHtml('description'), diff --git a/src/services/communities/communities.service.test.js b/src/services/communities/communities.service.test.js index 5b53dbd3..53beaf32 100644 --- a/src/services/communities/communities.service.test.js +++ b/src/services/communities/communities.service.test.js @@ -40,6 +40,16 @@ function postCommunityTestCases() { assert.equal(response.statusCode, 201); assert.equal(response.body.ownerAddress, SAMPLE_DATA.USER_ADDRESS); }); + + it('should create community successfully, but verified should not set', async () => { + const response = await request(baseUrl) + .post(relativeUrl) + .send({ ...SAMPLE_DATA.CREATE_COMMUNITY_DATA, verified: true }) + .set({ Authorization: getJwt() }); + assert.equal(response.statusCode, 201); + assert.isFalse(response.body.verified); + }); + it('should get unAuthorized error', async () => { const response = await request(baseUrl) .post(relativeUrl) diff --git a/src/services/traces/traces.hooks.js b/src/services/traces/traces.hooks.js index 6fd24fa6..ae3964d0 100644 --- a/src/services/traces/traces.hooks.js +++ b/src/services/traces/traces.hooks.js @@ -23,13 +23,13 @@ const checkTraceName = require('./checkTraceName'); const { getBlockTimestamp, ZERO_ADDRESS } = require('../../blockchain/lib/web3Helpers'); const { getTokenByAddress } = require('../../utils/tokenHelper'); const createModelSlug = require('../createModelSlug'); +const { isRequestInternal } = require('../../utils/feathersUtils'); -const removeProtectedFields = async context => { +const removeProtectedFields = () => context => { + if (context && context.data && !isRequestInternal(context)) { + delete context.data.verified; + } return context; - // if (context && context.data) { - // delete context.data.verified; - // } - // return context; }; const traceResolvers = { diff --git a/src/services/traces/traces.service.test.js b/src/services/traces/traces.service.test.js index b3053e9d..a53c5317 100644 --- a/src/services/traces/traces.service.test.js +++ b/src/services/traces/traces.service.test.js @@ -95,6 +95,13 @@ function postMilestoneTestCases() { assert.notEqual(response1.body.slug, response2.body.slug); }); + it('should create trace but verified field should not set', async () => { + const createMileStoneData = { ...SAMPLE_DATA.createTraceData(), verified: true }; + createMileStoneData.status = SAMPLE_DATA.TRACE_STATUSES.PROPOSED; + createMileStoneData.ownerAddress = SAMPLE_DATA.USER_ADDRESS; + const trace = await createTrace(createMileStoneData); + assert.isFalse(trace.verified); + }); } function patchMilestoneTestCases() { diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index 5492fc7c..ca9d8ca8 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -1,38 +1,54 @@ const moment = require('moment'); const errors = require('@feathersjs/errors'); const { listOfUserDonorsOnVerifiedProjects } = require('../../repositories/donationRepository'); +const { findVerifiedCommunities } = require('../../repositories/communityRepository'); +const { findVerifiedCampaigns } = require('../../repositories/campaignRepository'); +const { findVerifiedTraces } = require('../../repositories/traceRepository'); module.exports = function aggregateDonations() { const app = this; const givbackReportDonations = { async find({ query }) { - const { fromDate, toDate, projectIds } = query; - if (!projectIds) { - throw new errors.BadRequest('projectIds are required'); - } + const { fromDate, toDate } = query; if (!fromDate) { throw new errors.BadRequest('fromDate is required with this format: YYYY/MM/DD-hh:mm:ss'); } if (!toDate) { throw new errors.BadRequest('toDate is required with this format: YYYY/MM/DD-hh:mm:ss'); } - if (!projectIds) { - throw new errors.BadRequest('projectIds are required'); - } + const [traces, campaigns, communities] = await Promise.all([ + findVerifiedTraces(app), + findVerifiedCampaigns(app), + findVerifiedCommunities(app), + ]); + const verifiedProjectIds = traces + .map(trace => trace.projectId) + .concat(campaigns.map(campaign => campaign.projectId)) + .concat(communities.map(community => community.delegateId)); + const from = moment(fromDate, 'YYYY/MM/DD-hh:mm:ss').toDate(); const to = moment(toDate, 'YYYY/MM/DD-hh:mm:ss').toDate(); const result = await listOfUserDonorsOnVerifiedProjects(app, { - projectIds: projectIds.split(',').map(projectId => Number(projectId)), + verifiedProjectIds, from, to, }); - + result.forEach(giverInfo => { + giverInfo.donations.forEach(donation => { + // donations to communities may have both delegateId and ownerId but we should consider delegateId for them + donation.projectId = donation.delegateId || donation.ownerId; + delete donation.delegateId; + delete donation.ownerId; + donation.amount = Number(donation.amount) / 10 ** 18; + }); + }); return { total: result.length, from, to, + verifiedProjectIds, data: result, }; }, From 5e83e6e6e44299ad26d7dce27621464f8e8edf1f Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 12 Jun 2021 21:48:06 +0430 Subject: [PATCH 03/47] Remove unused test case --- .../verifiedPerojectsGiversReport.service.test.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.test.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.test.js index 57b5aa20..853ff6b5 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.test.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.test.js @@ -6,12 +6,6 @@ const baseUrl = config.get('givethFathersBaseUrl'); const relativeUrl = '/verifiedProjectsGiversReport'; function getGasPriceTestCases() { - it('get error if projectIds didnt send', async () => { - const response = await request(baseUrl).get(relativeUrl); - assert.equal(response.statusCode, 400); - assert.equal(response.body.message, 'projectIds are required'); - }); - it('get error if fromDate didnt send', async () => { const response = await request(baseUrl).get(`${relativeUrl}?projectIds=1,2,3,4,5,22`); assert.equal(response.statusCode, 400); From 5270c865e3aaf61e60baae7b025fba3768ddd279 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 13 Jun 2021 14:49:40 +0430 Subject: [PATCH 04/47] Get tokenInfo from tokenAddress of donations reated to Giveth/gievth-dapp#2253 --- src/repositories/donationRepository.js | 2 +- .../verifiedPerojectsGiversReport.service.js | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/repositories/donationRepository.js b/src/repositories/donationRepository.js index 74377781..76855267 100644 --- a/src/repositories/donationRepository.js +++ b/src/repositories/donationRepository.js @@ -118,7 +118,7 @@ const listOfUserDonorsOnVerifiedProjects = async (app, { verifiedProjectIds, fro $project: { _id: 0, createdAt: 1, - token: '$token.symbol', + tokenAddress: 1, usdValue: 1, amount: 1, homeTxHash: 1, diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index ca9d8ca8..2e415af0 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -4,6 +4,7 @@ const { listOfUserDonorsOnVerifiedProjects } = require('../../repositories/donat const { findVerifiedCommunities } = require('../../repositories/communityRepository'); const { findVerifiedCampaigns } = require('../../repositories/campaignRepository'); const { findVerifiedTraces } = require('../../repositories/traceRepository'); +const { getTokenByAddress } = require('../../utils/tokenHelper'); module.exports = function aggregateDonations() { const app = this; @@ -37,11 +38,14 @@ module.exports = function aggregateDonations() { }); result.forEach(giverInfo => { giverInfo.donations.forEach(donation => { + const token = getTokenByAddress(donation.tokenAddress); + donation.amount = Number(donation.amount) / 10 ** 18; + donation.token = token.symbol; + // donations to communities may have both delegateId and ownerId but we should consider delegateId for them donation.projectId = donation.delegateId || donation.ownerId; delete donation.delegateId; delete donation.ownerId; - donation.amount = Number(donation.amount) / 10 ** 18; }); }); return { From cff494ee190eb94c957b82515d2b3eefe3e65935 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 13 Jun 2021 16:26:41 +0430 Subject: [PATCH 05/47] Stop supporting conversion rate by transaction hash --- .../conversionRates/conversionRates.hooks.js | 32 ++----------------- 1 file changed, 3 insertions(+), 29 deletions(-) diff --git a/src/services/conversionRates/conversionRates.hooks.js b/src/services/conversionRates/conversionRates.hooks.js index 104a79a2..9440164c 100644 --- a/src/services/conversionRates/conversionRates.hooks.js +++ b/src/services/conversionRates/conversionRates.hooks.js @@ -1,5 +1,4 @@ const { disallow } = require('feathers-hooks-common'); -const errors = require('@feathersjs/errors'); const onlyInternal = require('../../hooks/onlyInternal'); const { @@ -7,7 +6,6 @@ const { getHourlyCryptoConversion, getHourlyMultipleCryptoConversion, } = require('./getConversionRatesService'); -const { getTransaction } = require('../../blockchain/lib/web3Helpers'); const findConversionRates = () => async context => { const { app, params } = context; @@ -15,34 +13,10 @@ const findConversionRates = () => async context => { // return context to avoid recursion // getConversionRates also calls this hook if (params.internal) return context; - const { - date: queryDate, - to, - symbol, - from, - interval: queryInterval, - txHash, - isHome, - } = params.query; + const { date: queryDate, to, symbol, from, interval: queryInterval } = params.query; - let date = Number(queryDate); - let interval = queryInterval; - - if (txHash) { - let error; - try { - const tx = await getTransaction(app, txHash, isHome === 'true'); - if (tx) { - date = tx.timestamp; - interval = 'hourly'; - } - } catch (e) { - error = e; - } - if (error) throw new errors.BadRequest(`Invalid tx ${error}`); - } - - if (interval === 'hourly') { + const date = Number(queryDate); + if (queryInterval === 'hourly') { if (Array.isArray(to)) { return getHourlyMultipleCryptoConversion(app, date, from, to).then(res => { context.result = res; From 0115443dc1b3fb5721a9595ca02291b7fb847e49 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 13 Jun 2021 19:31:16 +0430 Subject: [PATCH 06/47] Block bulk update of users related Giveth/giveth-dapp#1980 --- src/services/users/user.service.test.js | 11 +++++++++++ src/services/users/users.hooks.js | 8 +++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/services/users/user.service.test.js b/src/services/users/user.service.test.js index 29c1fd6f..db2d96d4 100644 --- a/src/services/users/user.service.test.js +++ b/src/services/users/user.service.test.js @@ -138,6 +138,17 @@ function patchUserTestCases() { assert.isTrue(response.body.isProjectOwner); assert.isTrue(response.body.isDelegator); }); + + it('Bulk edit should be disabled', async () => { + const response = await request(baseUrl) + .patch(`${relativeUrl}`) + .send({ + isReviewer: true, + }) + .set({ Authorization: getJwt(SAMPLE_DATA.ADMIN_USER_ADDRESS) }); + assert.equal(response.statusCode, 400); + assert.equal(response.body.message, 'Bulk edit for users entity is disabled'); + }); } function deleteUserTestCases() { diff --git a/src/services/users/users.hooks.js b/src/services/users/users.hooks.js index b023a197..013c3c34 100644 --- a/src/services/users/users.hooks.js +++ b/src/services/users/users.hooks.js @@ -16,6 +16,12 @@ const normalizeId = () => context => { } return context; }; +const disableBulkEdit = () => context => { + if (!context.id) { + throw new errors.BadRequest('Bulk edit for users entity is disabled'); + } + return context; +}; const roleAccessKeys = ['isReviewer', 'isProjectOwner', 'isDelegator']; const restrictUserdataAndAccess = () => async context => { @@ -50,7 +56,7 @@ const restrictUserdataAndAccess = () => async context => { throw new errors.Forbidden(); }; -const restrict = [normalizeId(), restrictUserdataAndAccess()]; +const restrict = [normalizeId(), disableBulkEdit(), restrictUserdataAndAccess()]; const address = [ setAddress('address'), From b7f41685f896ff4f3baa1eb6b61781b4bd4c0ce3 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Mon, 14 Jun 2021 15:29:15 +0430 Subject: [PATCH 07/47] Sort leaderboard donations by createdAt descendin related Giveth/giveth-dapp#2278 --- src/services/aggregateDonations/aggregateDonations.service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/aggregateDonations/aggregateDonations.service.js b/src/services/aggregateDonations/aggregateDonations.service.js index 61e49e50..4894b25d 100644 --- a/src/services/aggregateDonations/aggregateDonations.service.js +++ b/src/services/aggregateDonations/aggregateDonations.service.js @@ -58,7 +58,7 @@ module.exports = function aggregateDonations() { paginate: false, query: { _id: { $in: item.donations }, - $sort: { createAt: -1 }, + $sort: { createdAt: -1 }, }, }), usersService.find({ From 445786fa6951996898883f9145eedd964b7a5491 Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Mon, 14 Jun 2021 17:08:11 +0430 Subject: [PATCH 08/47] Renaming some variable --- src/repositories/donationRepository.js | 4 ++-- .../verifiedPerojectsGiversReport.service.js | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/repositories/donationRepository.js b/src/repositories/donationRepository.js index 76855267..3142c636 100644 --- a/src/repositories/donationRepository.js +++ b/src/repositories/donationRepository.js @@ -69,7 +69,7 @@ const isAllDonationsPaidOut = async (app, { txHash, traceId }) => { }] >} */ -const listOfUserDonorsOnVerifiedProjects = async (app, { verifiedProjectIds, from, to }) => { +const listOfDonorsToVerifiedProjects = async (app, { verifiedProjectIds, from, to }) => { const donationModel = app.service('donations').Model; return donationModel.aggregate([ { @@ -145,5 +145,5 @@ module.exports = { updateBridgePaymentExecutedTxHash, updateBridgePaymentAuthorizedTxHash, isAllDonationsPaidOut, - listOfUserDonorsOnVerifiedProjects, + listOfDonorsToVerifiedProjects, }; diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index 2e415af0..24089ab7 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -1,6 +1,6 @@ const moment = require('moment'); const errors = require('@feathersjs/errors'); -const { listOfUserDonorsOnVerifiedProjects } = require('../../repositories/donationRepository'); +const { listOfDonorsToVerifiedProjects } = require('../../repositories/donationRepository'); const { findVerifiedCommunities } = require('../../repositories/communityRepository'); const { findVerifiedCampaigns } = require('../../repositories/campaignRepository'); const { findVerifiedTraces } = require('../../repositories/traceRepository'); @@ -23,15 +23,16 @@ module.exports = function aggregateDonations() { findVerifiedCampaigns(app), findVerifiedCommunities(app), ]); - const verifiedProjectIds = traces - .map(trace => trace.projectId) - .concat(campaigns.map(campaign => campaign.projectId)) - .concat(communities.map(community => community.delegateId)); + const verifiedProjectIds = [ + ...traces.map(trace => trace.projectId), + ...campaigns.map(campaign => campaign.projectId), + ...communities.map(community => community.delegateId), + ]; const from = moment(fromDate, 'YYYY/MM/DD-hh:mm:ss').toDate(); const to = moment(toDate, 'YYYY/MM/DD-hh:mm:ss').toDate(); - const result = await listOfUserDonorsOnVerifiedProjects(app, { + const result = await listOfDonorsToVerifiedProjects(app, { verifiedProjectIds, from, to, From e2ea97ea340cca27f45630f034d7671e34e2b465 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Mon, 14 Jun 2021 18:40:18 +0430 Subject: [PATCH 09/47] Make verifiedProjectsGivers query efficient --- src/repositories/donationRepository.js | 43 +++++++++++--------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/repositories/donationRepository.js b/src/repositories/donationRepository.js index 76855267..8ca5f24f 100644 --- a/src/repositories/donationRepository.js +++ b/src/repositories/donationRepository.js @@ -61,9 +61,11 @@ const isAllDonationsPaidOut = async (app, { txHash, traceId }) => { "usdValue": number, "amount": number, "homeTxHash": string, + "giverAddress": string, "createdAt": string // sample: "2019-04-22T22:14:23.046Z", "token": string //sample : "ETH", - "projectId": number + "delegateId": number + "ownerId": number } ] }] @@ -98,37 +100,28 @@ const listOfUserDonorsOnVerifiedProjects = async (app, { verifiedProjectIds, fro isReturn: false, }, }, - + { + $project: { + giverAddress: 1, + usdValue: 1, + amount: 1, + homeTxHash: 1, + ownerId: 1, + delegateId: 1, + tokenAddress: 1, + createdAt: 1, + _id: 0, + }, + }, { $group: { _id: '$giverAddress', totalAmount: { $sum: '$usdValue' }, - donationIds: { $push: '$_id' }, + donations: { $push: '$$ROOT' }, }, }, { - $lookup: { - from: 'donations', - let: { donationIds: '$donationIds' }, - pipeline: [ - { - $match: { $expr: { $in: ['$_id', '$$donationIds'] } }, - }, - { - $project: { - _id: 0, - createdAt: 1, - tokenAddress: 1, - usdValue: 1, - amount: 1, - homeTxHash: 1, - delegateId: 1, - ownerId: 1, - }, - }, - ], - as: 'donations', - }, + $sort: { totalAmount: -1 }, }, { $project: { From f4aa3481d3326e558d4db8cbda24b8e0d6c07971 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Mon, 14 Jun 2021 18:44:38 +0430 Subject: [PATCH 10/47] Add sort for donations --- src/repositories/donationRepository.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/repositories/donationRepository.js b/src/repositories/donationRepository.js index d35dad61..0e5eaa72 100644 --- a/src/repositories/donationRepository.js +++ b/src/repositories/donationRepository.js @@ -100,6 +100,11 @@ const listOfDonorsToVerifiedProjects = async (app, { verifiedProjectIds, from, t isReturn: false, }, }, + { + $sort: { + createdAt: -1, + }, + }, { $project: { giverAddress: 1, From 72fdea3d9f26122c442abdd8c0d462606447951d Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Tue, 15 Jun 2021 21:55:21 +0430 Subject: [PATCH 11/47] Add project info to response, can get response for all projects related to Giveth/giveth-dapp#2253 --- src/repositories/donationRepository.js | 99 ++++++++++++++++--- .../verifiedPerojectsGiversReport.service.js | 63 +++++++++--- src/utils/urlUtils.js | 17 ++++ src/utils/urlUtils.test.js | 29 ++++++ 4 files changed, 186 insertions(+), 22 deletions(-) create mode 100644 src/utils/urlUtils.js create mode 100644 src/utils/urlUtils.test.js diff --git a/src/repositories/donationRepository.js b/src/repositories/donationRepository.js index 0e5eaa72..0d1a267b 100644 --- a/src/repositories/donationRepository.js +++ b/src/repositories/donationRepository.js @@ -50,7 +50,7 @@ const isAllDonationsPaidOut = async (app, { txHash, traceId }) => { * @param app: feathers instance * @param from: Date, example: 2018-06-08T16:05:28.005Z * @param to: Date: example: 2021-06-08T16:05:28.005Z - * @param projectIds: Array, example: [1340, 2723] + * @param projectIds ?: Array, example: [1340, 2723] * @returns { * Promise< [{ @@ -73,6 +73,29 @@ const isAllDonationsPaidOut = async (app, { txHash, traceId }) => { */ const listOfDonorsToVerifiedProjects = async (app, { verifiedProjectIds, from, to }) => { const donationModel = app.service('donations').Model; + // If verifiedProjectIds is falsy it means we should use all donations + const orCondition = verifiedProjectIds + ? [ + { + // it's for communities + delegateId: { $in: verifiedProjectIds }, + intendedProjectId: { $exists: false }, + }, + + // it's for traces and campaigns + { ownerId: { $in: verifiedProjectIds } }, + ] + : [ + { + // it's for communities + delegateId: { $exists: true }, + intendedProjectId: { $exists: false }, + }, + + // it's for traces and campaigns + { ownerId: { $exists: true }, ownerType: { $in: ['campaign', 'trace'] } }, + ]; + return donationModel.aggregate([ { $match: { @@ -85,16 +108,7 @@ const listOfDonorsToVerifiedProjects = async (app, { verifiedProjectIds, from, t }, homeTxHash: { $exists: true }, - $or: [ - { - // it's for communities - delegateId: { $in: verifiedProjectIds }, - intendedProjectId: { $exists: false }, - }, - - // it's for traces and campaigns - { ownerId: { $in: verifiedProjectIds } }, - ], + $or: orCondition, amount: { $ne: '0' }, usdValue: { $ne: 0 }, isReturn: false, @@ -118,6 +132,69 @@ const listOfDonorsToVerifiedProjects = async (app, { verifiedProjectIds, from, t _id: 0, }, }, + { + $lookup: { + from: 'communities', + let: { delegateId: '$delegateId' }, + pipeline: [ + { + $match: { + $expr: { $eq: ['$delegateId', '$$delegateId'] }, + }, + }, + { + $project: { + _id: 1, + title: 1, + }, + }, + ], + as: 'community', + }, + }, + { + $lookup: { + from: 'campaigns', + let: { projectId: '$ownerId' }, + pipeline: [ + { + $match: { + $expr: { $eq: ['$projectId', '$$projectId'] }, + }, + }, + { + $project: { + _id: 1, + title: 1, + }, + }, + ], + as: 'campaign', + }, + }, + { + $lookup: { + from: 'traces', + let: { projectId: '$ownerId' }, + pipeline: [ + { + $match: { + projectId: { $exists: true }, + $expr: { $eq: ['$projectId', '$$projectId'] }, + }, + }, + { + $project: { + _id: 1, + title: 1, + campaignId: 1, + }, + }, + ], + as: 'trace', + }, + }, + { $group: { _id: '$giverAddress', diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index 24089ab7..abe005a2 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -1,33 +1,69 @@ const moment = require('moment'); +const logger = require('winston'); const errors = require('@feathersjs/errors'); const { listOfDonorsToVerifiedProjects } = require('../../repositories/donationRepository'); const { findVerifiedCommunities } = require('../../repositories/communityRepository'); const { findVerifiedCampaigns } = require('../../repositories/campaignRepository'); const { findVerifiedTraces } = require('../../repositories/traceRepository'); const { getTokenByAddress } = require('../../utils/tokenHelper'); +const { getTraceUrl, getCampaignUrl, getCommunityUrl } = require('../../utils/urlUtils'); + +const fillProjectInfo = donation => { + const { campaign, trace, community } = donation; + if (campaign.length === 1) { + return { + title: campaign[0].title, + type: 'Campaign', + url: getCampaignUrl(campaign[0]), + }; + } + if (trace.length === 1) { + return { + title: trace[0].title, + type: 'Trace', + url: getTraceUrl(trace[0]), + }; + } + if (community.length === 1) { + return { + title: community[0].title, + type: 'Community', + url: getCommunityUrl(community[0]), + }; + } + logger.error('donation should have trace, campaign or community', donation); + throw new Error('donation should have trace, campaign or community'); +}; + +const getAllVerfiedProjectdIds = async app => { + const [traces, campaigns, communities] = await Promise.all([ + findVerifiedTraces(app), + findVerifiedCampaigns(app), + findVerifiedCommunities(app), + ]); + return [ + ...traces.map(trace => trace.projectId), + ...campaigns.map(campaign => campaign.projectId), + ...communities.map(community => community.delegateId), + ]; +}; module.exports = function aggregateDonations() { const app = this; const givbackReportDonations = { async find({ query }) { - const { fromDate, toDate } = query; + const { fromDate, toDate, allProjects } = query; if (!fromDate) { throw new errors.BadRequest('fromDate is required with this format: YYYY/MM/DD-hh:mm:ss'); } if (!toDate) { throw new errors.BadRequest('toDate is required with this format: YYYY/MM/DD-hh:mm:ss'); } - const [traces, campaigns, communities] = await Promise.all([ - findVerifiedTraces(app), - findVerifiedCampaigns(app), - findVerifiedCommunities(app), - ]); - const verifiedProjectIds = [ - ...traces.map(trace => trace.projectId), - ...campaigns.map(campaign => campaign.projectId), - ...communities.map(community => community.delegateId), - ]; + let verifiedProjectIds; + if (!allProjects || allProjects === 'false') { + verifiedProjectIds = await getAllVerfiedProjectdIds(app); + } const from = moment(fromDate, 'YYYY/MM/DD-hh:mm:ss').toDate(); const to = moment(toDate, 'YYYY/MM/DD-hh:mm:ss').toDate(); @@ -45,8 +81,13 @@ module.exports = function aggregateDonations() { // donations to communities may have both delegateId and ownerId but we should consider delegateId for them donation.projectId = donation.delegateId || donation.ownerId; + + donation.projectInfo = fillProjectInfo(donation); delete donation.delegateId; delete donation.ownerId; + delete donation.campaign; + delete donation.community; + delete donation.trace; }); }); return { diff --git a/src/utils/urlUtils.js b/src/utils/urlUtils.js new file mode 100644 index 00000000..5f759251 --- /dev/null +++ b/src/utils/urlUtils.js @@ -0,0 +1,17 @@ +const config = require('config'); + +const getTraceUrl = ({ _id, campaignId }) => { + return `${config.dappUrl}/campaigns/${campaignId}/traces/${_id}`; +}; +const getCampaignUrl = ({ _id }) => { + return `${config.dappUrl}/campaigns/${_id}`; +}; +const getCommunityUrl = ({ _id }) => { + return `${config.dappUrl}/communities/${_id}`; +}; + +module.exports = { + getTraceUrl, + getCampaignUrl, + getCommunityUrl, +}; diff --git a/src/utils/urlUtils.test.js b/src/utils/urlUtils.test.js new file mode 100644 index 00000000..cc479e6f --- /dev/null +++ b/src/utils/urlUtils.test.js @@ -0,0 +1,29 @@ +const { assert } = require('chai'); +const config = require('config'); +const { getTraceUrl, getCampaignUrl, getCommunityUrl } = require('./urlUtils'); +const { generateRandomMongoId } = require('../../test/testUtility'); + +describe('getTraceUrl() test cases', () => { + it('should return traceUrl', () => { + const traceId = generateRandomMongoId(); + const campaignId = generateRandomMongoId(); + const traceUrl = getTraceUrl({ _id: traceId, campaignId }); + assert.equal(traceUrl, `${config.dappUrl}/campaigns/${campaignId}/traces/${traceId}`); + }); +}); + +describe('getCampaignUrl() test cases', () => { + it('should return traceUrl', () => { + const campaignId = generateRandomMongoId(); + const campaignUrl = getCampaignUrl({ _id: campaignId }); + assert.equal(campaignUrl, `${config.dappUrl}/campaigns/${campaignId}`); + }); +}); + +describe('getCommunityUrl() test cases', () => { + it('should return traceUrl', () => { + const communityId = generateRandomMongoId(); + const communityUrl = getCommunityUrl({ _id: communityId }); + assert.equal(communityUrl, `${config.dappUrl}/communities/${communityId}`); + }); +}); From 5dcd26b50f7bfd39073d2e2e5db60c7d3f9f71dd Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Tue, 15 Jun 2021 22:11:21 +0430 Subject: [PATCH 12/47] Add sample log for donations event --- src/app.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/app.js b/src/app.js index 6ca83654..d7a9bfba 100644 --- a/src/app.js +++ b/src/app.js @@ -23,6 +23,16 @@ const channels = require('./channels'); const app = express(feathers()); +const addLogerToServices = () => { + const traces = app.service('traces'); + + // Listen to a normal service event + traces.on('patched', trace => console.log('trace patched', trace)); + + // Only listen to an event once + traces.on('created', trace => console.log('trace has been created', trace)); +}; + function initFeatherApp() { // Load app configuration app.configure(configuration()); @@ -83,9 +93,11 @@ function initFeatherApp() { ); app.hooks(appHooks); + addLogerToServices(app); return app; } + function getFeatherAppInstance() { return app; } From 5473e878eb0128b625c3bffe9424632034c79818 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Tue, 15 Jun 2021 22:17:48 +0430 Subject: [PATCH 13/47] rename fillProjectInfo() to extractProjectInfo() --- .../verifiedPerojectsGiversReport.service.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index abe005a2..b4f9c17a 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -8,7 +8,7 @@ const { findVerifiedTraces } = require('../../repositories/traceRepository'); const { getTokenByAddress } = require('../../utils/tokenHelper'); const { getTraceUrl, getCampaignUrl, getCommunityUrl } = require('../../utils/urlUtils'); -const fillProjectInfo = donation => { +const extractProjectInfo = donation => { const { campaign, trace, community } = donation; if (campaign.length === 1) { return { @@ -82,7 +82,7 @@ module.exports = function aggregateDonations() { // donations to communities may have both delegateId and ownerId but we should consider delegateId for them donation.projectId = donation.delegateId || donation.ownerId; - donation.projectInfo = fillProjectInfo(donation); + donation.projectInfo = extractProjectInfo(donation); delete donation.delegateId; delete donation.ownerId; delete donation.campaign; From dbb75890b97d051a3861caca9d30c7e411b80307 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 19 Jun 2021 19:55:41 +0430 Subject: [PATCH 14/47] Add elastic search and kibana docker-compose related to #346 --- elk/.env | 2 ++ elk/README.md | 12 +++++++ elk/docker-compose.yml | 51 +++++++++++++++++++++++++++++ elk/elasticsearch/Dockerfile | 7 ++++ elk/elasticsearch/elasticsearch.yml | 13 ++++++++ elk/kibana/Dockerfile | 7 ++++ elk/kibana/kibana.yml | 13 ++++++++ 7 files changed, 105 insertions(+) create mode 100644 elk/.env create mode 100644 elk/README.md create mode 100644 elk/docker-compose.yml create mode 100644 elk/elasticsearch/Dockerfile create mode 100644 elk/elasticsearch/elasticsearch.yml create mode 100644 elk/kibana/Dockerfile create mode 100644 elk/kibana/kibana.yml diff --git a/elk/.env b/elk/.env new file mode 100644 index 00000000..3dd09acb --- /dev/null +++ b/elk/.env @@ -0,0 +1,2 @@ +ELK_VERSION=7.13.1 +ELASTIC_SEARCH_PASSWORD=changeme diff --git a/elk/README.md b/elk/README.md new file mode 100644 index 00000000..2dff77f1 --- /dev/null +++ b/elk/README.md @@ -0,0 +1,12 @@ +## Installation +The Dockerfiles, docker-composes and configs are inspired from this repository +https://github.com/deviantony/docker-elk + +## Run +docker-compose up + +## Usage +After running docker-compose you will see : +* **Kibana:** localhost:5601 +* **Elastic search:** localhost:9200 + diff --git a/elk/docker-compose.yml b/elk/docker-compose.yml new file mode 100644 index 00000000..8774cd3d --- /dev/null +++ b/elk/docker-compose.yml @@ -0,0 +1,51 @@ +version: '3.2' + +services: + elasticsearch: + build: + context: elasticsearch/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - type: bind + source: ./elasticsearch/elasticsearch.yml + target: /usr/share/elasticsearch/config/elasticsearch.yml + read_only: true + - type: volume + source: elasticsearch + target: /usr/share/elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + ELASTIC_PASSWORD: $ELASTIC_SEARCH_PASSWORD + # Use single node discovery in order to disable production mode and avoid bootstrap checks. + # see: https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html + discovery.type: single-node + networks: + - elk + + kibana: + build: + context: kibana/ + args: + ELK_VERSION: $ELK_VERSION + volumes: + - type: bind + source: ./kibana/kibana.yml + target: /usr/share/kibana/config/kibana.yml + read_only: true + ports: + - "5601:5601" + networks: + - elk + depends_on: + - elasticsearch + +networks: + elk: + driver: bridge + +volumes: + elasticsearch: diff --git a/elk/elasticsearch/Dockerfile b/elk/elasticsearch/Dockerfile new file mode 100644 index 00000000..39285445 --- /dev/null +++ b/elk/elasticsearch/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://www.docker.elastic.co/ +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} + +# Add your elasticsearch plugins setup here +# Example: RUN elasticsearch-plugin install analysis-icu diff --git a/elk/elasticsearch/elasticsearch.yml b/elk/elasticsearch/elasticsearch.yml new file mode 100644 index 00000000..b06c1d21 --- /dev/null +++ b/elk/elasticsearch/elasticsearch.yml @@ -0,0 +1,13 @@ +--- +## Default Elasticsearch configuration from Elasticsearch base image. +## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml +# +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +## X-Pack settings +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html +# +xpack.license.self_generated.type: basic +xpack.security.enabled: true +xpack.monitoring.collection.enabled: true diff --git a/elk/kibana/Dockerfile b/elk/kibana/Dockerfile new file mode 100644 index 00000000..2fb3659b --- /dev/null +++ b/elk/kibana/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://www.docker.elastic.co/ +FROM docker.elastic.co/kibana/kibana:${ELK_VERSION} + +# Add your kibana plugins setup here +# Example: RUN kibana-plugin install diff --git a/elk/kibana/kibana.yml b/elk/kibana/kibana.yml new file mode 100644 index 00000000..0e1dc60c --- /dev/null +++ b/elk/kibana/kibana.yml @@ -0,0 +1,13 @@ +--- +## Default Kibana configuration from Kibana base image. +## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.ts +# +server.name: kibana +server.host: 0.0.0.0 +elasticsearch.hosts: [ "http://elasticsearch:9200" ] +monitoring.ui.container.elasticsearch.enabled: true + +## X-Pack security credentials +# +elasticsearch.username: elastic +elasticsearch.password: changeme From b2cb10fdb4bd02a9fe38799850f0aefeb1c0a7d8 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 20 Jun 2021 10:20:30 +0430 Subject: [PATCH 15/47] Send feathers patch,remove,post,create events to elastic search related to #346 --- README.md | 11 ++++ config/default.json | 6 ++- src/app.js | 18 ++----- src/auditLog/elatikSearchUtils.js | 38 ++++++++++++++ src/auditLog/feathersElasticSearch.js | 74 +++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 14 deletions(-) create mode 100644 src/auditLog/elatikSearchUtils.js create mode 100644 src/auditLog/feathersElasticSearch.js diff --git a/README.md b/README.md index b0027623..b89443d0 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,17 @@ module.exports = { yarn start:rsk ``` +## Audit Log +The Audit log system logs every Create, Update, Patch and +Remove on **Campaigns**, **Traces**, **Events**, **Users**, +**PledgeAdmins**, **Communities**, **Donations** +For enabling audit log locally you should change `enableAuditLog` +in config to `true`, then +* cd elk +* docker-compose up + +And then after logging in `localhost:5601` with user:`elastic`, password: `changeme` +you can see the logs ## Help diff --git a/config/default.json b/config/default.json index e827bc8f..eee48f80 100644 --- a/config/default.json +++ b/config/default.json @@ -171,5 +171,9 @@ "dappMailerSecret": "xxMv8a13I3YOZSKq9XmX285N0o4m1qHyKjgQWb2g83daGtPI4VmEw7dImu8kvO2ovG4EMC1bvjO906o365BaJBZtArpnxJCoYsCJ", "minimumPayoutUsdValue": 2, "givethAccounts": [], - "enablePayoutEmail": true + "enablePayoutEmail": true, + "elasticSearchUrl": "http://localhost:9200/entities/_doc", + "elasticSearchUsername": "elastic", + "elasticSearchPassword": "changeme", + "enableAuditLog": false } diff --git a/src/app.js b/src/app.js index d7a9bfba..fb8fcf80 100644 --- a/src/app.js +++ b/src/app.js @@ -1,5 +1,6 @@ const logger = require('winston'); const path = require('path'); +const config = require('config'); const favicon = require('serve-favicon'); const compress = require('compression'); const cors = require('cors'); @@ -18,21 +19,11 @@ const blockchain = require('./blockchain'); const mongoose = require('./mongoose'); const ipfsFetcher = require('./utils/ipfsFetcher'); const ipfsPinner = require('./utils/ipfsPinner'); - +const { configureAuditLog } = require('./auditLog/feathersElasticSearch'); const channels = require('./channels'); const app = express(feathers()); -const addLogerToServices = () => { - const traces = app.service('traces'); - - // Listen to a normal service event - traces.on('patched', trace => console.log('trace patched', trace)); - - // Only listen to an event once - traces.on('created', trace => console.log('trace has been created', trace)); -}; - function initFeatherApp() { // Load app configuration app.configure(configuration()); @@ -93,11 +84,12 @@ function initFeatherApp() { ); app.hooks(appHooks); - addLogerToServices(app); + if(config.enableAuditLog){ + configureAuditLog(app); + } return app; } - function getFeatherAppInstance() { return app; } diff --git a/src/auditLog/elatikSearchUtils.js b/src/auditLog/elatikSearchUtils.js new file mode 100644 index 00000000..9fe71efa --- /dev/null +++ b/src/auditLog/elatikSearchUtils.js @@ -0,0 +1,38 @@ +const axios = require('axios'); +const config = require('config'); +const logger = require('winston'); + +const createBasicAuthentication = ({ username, password }) => { + return `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`; +}; +const removeUdefinedFieldFromObject = object => { + // eslint-disable-next-line no-restricted-syntax + for (const key of Object.keys(object)) { + if (object[key] === undefined) { + delete object[key]; + } + } +}; +const sendEventToElasticSearch = async data => { + const basicAuthentication = createBasicAuthentication({ + username: config.elasticSearchUsername, + password: config.elasticSearchPassword, + }); + // if sending some data undefined may cause elastic search index dont work properly + removeUdefinedFieldFromObject(data); + try { + await axios.post(config.elasticSearchUrl, data, { + headers: { + Authorization: basicAuthentication, + 'Content-Type': 'application/json', + }, + }); + } catch (e) { + logger.info('sendEventToElasticSearch error', { e, message: e.message }); + } +}; + +module.exports = { + createBasicAuthentication, + sendEventToElasticSearch, +}; diff --git a/src/auditLog/feathersElasticSearch.js b/src/auditLog/feathersElasticSearch.js new file mode 100644 index 00000000..f4a35864 --- /dev/null +++ b/src/auditLog/feathersElasticSearch.js @@ -0,0 +1,74 @@ +const { sendEventToElasticSearch } = require('./elatikSearchUtils'); + +const unifyData = ({ item, context, serviceName }) => { + return { + entity: JSON.stringify(item), + entityType: serviceName, + provider: (context && context.params && context.params.provider) || 'internal', + user: context && context.params && context.params.user && context.params.user.address, + inputData: context && context.data && JSON.stringify(context.data, null, 4), + txHash: item.txHash || item.transactionHash, + updatedAt: item.updatedAt, + status: item.status, + homeTxHash: item.homeTxHash, + entityId: item._id || item.address, + }; +}; + +const setAuditLogToFeathersService = ({ app, serviceName }) => { + const service = app.service(serviceName); + service.on('patched', (item, context) => { + sendEventToElasticSearch({ + ...unifyData({ + item, + serviceName, + context, + }), + action: 'patch', + }); + }); + service.on('updated', (item, context) => { + sendEventToElasticSearch({ + ...unifyData({ + item, + serviceName, + context, + }), + action: 'patch', + }); + }); + service.on('removed', (item, context) => { + sendEventToElasticSearch({ + ...unifyData({ + item, + serviceName, + context, + }), + action: 'remove', + }); + }); + service.on('created', (item, context) => { + sendEventToElasticSearch({ + ...unifyData({ + item, + serviceName, + context, + }), + action: 'create', + }); + }); +}; + +const configureAuditLog = app => { + setAuditLogToFeathersService({ app, serviceName: 'traces' }); + setAuditLogToFeathersService({ app, serviceName: 'campaigns' }); + setAuditLogToFeathersService({ app, serviceName: 'users' }); + setAuditLogToFeathersService({ app, serviceName: 'communities' }); + setAuditLogToFeathersService({ app, serviceName: 'donations' }); + setAuditLogToFeathersService({ app, serviceName: 'pledgeAdmins' }); + setAuditLogToFeathersService({ app, serviceName: 'events' }); +}; + +module.exports = { + configureAuditLog, +}; From b0bfa43872dd8550667985506aba4b48ee896d08 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 20 Jun 2021 14:11:11 +0430 Subject: [PATCH 16/47] Fix eslint error --- src/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.js b/src/app.js index fb8fcf80..0ffcdb73 100644 --- a/src/app.js +++ b/src/app.js @@ -84,7 +84,7 @@ function initFeatherApp() { ); app.hooks(appHooks); - if(config.enableAuditLog){ + if (config.enableAuditLog) { configureAuditLog(app); } return app; From b36ef4e91ee1b0ee67cc0f420b41fcedf9fe5af0 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 20 Jun 2021 15:16:47 +0430 Subject: [PATCH 17/47] Update readme --- README.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b0027623..f7f6b5fb 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Welcome to the server code for Giveth's [dapp](https://github.com/Giveth/giveth- ``` cd feathers-giveth ``` - 5. Make sure you have [NodeJS](https://nodejs.org/) (v8.4.0 or higher), [yarn](https://www.yarnpkg.com/) (v0.27.5 or higher), and npm (5.4.1 or higher) installed. + 5. Make sure you have [NodeJS](https://nodejs.org/) (v10.24.0 or higher), [yarn](https://www.yarnpkg.com/) (v0.27.5 or higher), and npm (5.4.1 or higher) installed. 6. Install dependencies from within feathers-giveth directory: ``` npm install @@ -120,9 +120,11 @@ The `feathers-giveth/scripts` directory contains a few scripts to help developme * `confirm.js` - confirms any payments that are pending in the vault * `makeUserAdmin.js` - make a user admin + ## Testing -Simply run `yarn test` and all your tests in the `test/` directory will be run. +Simply run `yarn test` and all your tests in the `/src` directory will be run. +It's included some integration tests so for running tests, you need to run a mongodb in your local system (on port 27017) ## Debugging @@ -136,15 +138,21 @@ Each of these services are available via rest or websockets: ``` campaigns -dacs +communities donations donationsHistory -milestones +traces uploads users +emails +homePaymentsTransactions +subscriptions ``` If the server is using default configurations, you can see data for any of these services through your web browser at `http://localhost:3030/SERVICE_NAME` +PS: For accessing all features like creating `communities` and `campaigns` it's suggested to +make `isAdmin` field true, for your user in you local MongoDb + ## Production @@ -172,7 +180,7 @@ module.exports = { ], }; ``` - +PS: It's good to see [Github Actions config](./.github/workflows/CI-CD.yml) to better understanding of deploy structure ## RSK 1. You will need to download the [rsk node](https://github.com/rsksmart/rskj/wiki/Install-RskJ-and-join-the-RSK-Orchid-Mainnet-Beta). After installing, you will run the node w/ the `regtest` network for local development. From 66992f06509ff8794864c583f4944f5ed4907a1c Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 20 Jun 2021 15:30:45 +0430 Subject: [PATCH 18/47] Add running migration hint --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f7f6b5fb..e3da0c3d 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,10 @@ The configuration param `blockchain.nodeUrl` is used to establish a connection. ``` ipfs daemon ``` - +5. Run db migration files + ``` + ./node_modules/.bin/migrate-mongo up + ``` 5. Start your app ``` From a92edd4098c19dbaa0d71df5c7e23185cba5c8bd Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 20 Jun 2021 18:03:43 +0430 Subject: [PATCH 19/47] Fix exception if the project of donation didnt found --- .../verifiedPerojectsGiversReport.service.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index b4f9c17a..9aecc483 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -32,7 +32,8 @@ const extractProjectInfo = donation => { }; } logger.error('donation should have trace, campaign or community', donation); - throw new Error('donation should have trace, campaign or community'); + // If we should throw exception we get error in UAT env, but in beta all donations have community, campaign or trace + return {}; }; const getAllVerfiedProjectdIds = async app => { From dcad0d75d47e5a061427bae5a9a31a504ae652a8 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 20 Jun 2021 18:06:52 +0430 Subject: [PATCH 20/47] Change comment message --- .../verifiedPerojectsGiversReport.service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index 9aecc483..4db1e07e 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -32,7 +32,7 @@ const extractProjectInfo = donation => { }; } logger.error('donation should have trace, campaign or community', donation); - // If we should throw exception we get error in UAT env, but in beta all donations have community, campaign or trace + // If we throw exception we get error in UAT env, but in beta all donations have community, campaign or trace return {}; }; From f5cc7533034400a154f2f10a4c39c80c31af662d Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Sun, 20 Jun 2021 18:13:26 +0430 Subject: [PATCH 21/47] Revert "Fix exception if the project of donation didnt found" --- .../verifiedPerojectsGiversReport.service.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index 4db1e07e..b4f9c17a 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -32,8 +32,7 @@ const extractProjectInfo = donation => { }; } logger.error('donation should have trace, campaign or community', donation); - // If we throw exception we get error in UAT env, but in beta all donations have community, campaign or trace - return {}; + throw new Error('donation should have trace, campaign or community'); }; const getAllVerfiedProjectdIds = async app => { From c9a87200471c862c16e9f9839c8761156b6dcc02 Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Sun, 20 Jun 2021 18:37:46 +0430 Subject: [PATCH 22/47] Revert "Revert "Fix exception if the project of donation didnt found"" --- .../verifiedPerojectsGiversReport.service.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js index b4f9c17a..4db1e07e 100644 --- a/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js +++ b/src/services/verifiedProjectsGiversReport/verifiedPerojectsGiversReport.service.js @@ -32,7 +32,8 @@ const extractProjectInfo = donation => { }; } logger.error('donation should have trace, campaign or community', donation); - throw new Error('donation should have trace, campaign or community'); + // If we throw exception we get error in UAT env, but in beta all donations have community, campaign or trace + return {}; }; const getAllVerfiedProjectdIds = async app => { From 532a585fc78453989fb164673aee9b09541fe723 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Mon, 28 Jun 2021 17:59:52 +0430 Subject: [PATCH 23/47] Format log data --- src/auditLog/feathersElasticSearch.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auditLog/feathersElasticSearch.js b/src/auditLog/feathersElasticSearch.js index f4a35864..a486893c 100644 --- a/src/auditLog/feathersElasticSearch.js +++ b/src/auditLog/feathersElasticSearch.js @@ -2,7 +2,7 @@ const { sendEventToElasticSearch } = require('./elatikSearchUtils'); const unifyData = ({ item, context, serviceName }) => { return { - entity: JSON.stringify(item), + entity: JSON.stringify(item, null, 4), entityType: serviceName, provider: (context && context.params && context.params.provider) || 'internal', user: context && context.params && context.params.user && context.params.user.address, From 255d05fac905e77a7815587581b0353f2d9d45af Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Tue, 29 Jun 2021 15:04:49 +0430 Subject: [PATCH 24/47] Cast verified field to boolean in GET requests related to Giveth/giveth-dapp#2315 --- src/app.hooks.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/app.hooks.js b/src/app.hooks.js index 501347a9..36e1916a 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -10,10 +10,21 @@ const authenticate = () => context => { return auth.hooks.authenticate('jwt')(context); }; +const convertVerfiedToBoolean = () => context => { + // verified field is boolean in Trace, Campaign and Community so for getting this filter + // in query string we should cast it to boolean here + if (context.params.query && context.params.query.verified === 'true') { + context.params.query.verified = true; + } else if (context.params.query && context.params.query.verified === 'false') { + context.params.query.verified = false; + } + return context; +}; + module.exports = { before: { all: [], - find: [], + find: [convertVerfiedToBoolean()], get: [], create: [authenticate()], update: [authenticate()], From 684e48803e664ac24ddee16b744d569b9ca9ddd2 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Thu, 1 Jul 2021 12:14:42 +0430 Subject: [PATCH 25/47] Add sentry for crash reporting related to Giveth/giveth-dapp#2303 https://github.com/feathersjs/feathers/discussions/2406 --- config/default.json | 3 +- package-lock.json | 108 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 + src/app.hooks.js | 22 +++++++-- src/app.js | 9 ++++ src/hooks/logger.js | 13 ------ 6 files changed, 140 insertions(+), 17 deletions(-) diff --git a/config/default.json b/config/default.json index eee48f80..30c89b11 100644 --- a/config/default.json +++ b/config/default.json @@ -175,5 +175,6 @@ "elasticSearchUrl": "http://localhost:9200/entities/_doc", "elasticSearchUsername": "elastic", "elasticSearchPassword": "changeme", - "enableAuditLog": false + "enableAuditLog": false, + "sentryDsn": "" } diff --git a/package-lock.json b/package-lock.json index 1c0c1f55..5cfab16f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2712,6 +2712,109 @@ "any-observable": "^0.3.0" } }, + "@sentry/core": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.8.0.tgz", + "integrity": "sha512-vJzWt/znEB+JqVwtwfjkRrAYRN+ep+l070Ti8GhJnvwU4IDtVlV3T/jVNrj6rl6UChcczaJQMxVxtG5x0crlAA==", + "requires": { + "@sentry/hub": "6.8.0", + "@sentry/minimal": "6.8.0", + "@sentry/types": "6.8.0", + "@sentry/utils": "6.8.0", + "tslib": "^1.9.3" + } + }, + "@sentry/hub": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@sentry/hub/-/hub-6.8.0.tgz", + "integrity": "sha512-hFrI2Ss1fTov7CH64FJpigqRxH7YvSnGeqxT9Jc1BL7nzW/vgCK+Oh2mOZbosTcrzoDv+lE8ViOnSN3w/fo+rg==", + "requires": { + "@sentry/types": "6.8.0", + "@sentry/utils": "6.8.0", + "tslib": "^1.9.3" + } + }, + "@sentry/minimal": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@sentry/minimal/-/minimal-6.8.0.tgz", + "integrity": "sha512-MRxUKXiiYwKjp8mOQMpTpEuIby1Jh3zRTU0cmGZtfsZ38BC1JOle8xlwC4FdtOH+VvjSYnPBMya5lgNHNPUJDQ==", + "requires": { + "@sentry/hub": "6.8.0", + "@sentry/types": "6.8.0", + "tslib": "^1.9.3" + } + }, + "@sentry/node": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-6.8.0.tgz", + "integrity": "sha512-DPUtDd1rRbDJys+aZdQTScKy2Xxo4m8iSQPxzfwFROsLmzE7XhDoriDwM+l1BpiZYIhxUU2TLxDyVzmdc/TMAw==", + "requires": { + "@sentry/core": "6.8.0", + "@sentry/hub": "6.8.0", + "@sentry/tracing": "6.8.0", + "@sentry/types": "6.8.0", + "@sentry/utils": "6.8.0", + "cookie": "^0.4.1", + "https-proxy-agent": "^5.0.0", + "lru_map": "^0.3.3", + "tslib": "^1.9.3" + }, + "dependencies": { + "cookie": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==" + }, + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, + "@sentry/tracing": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@sentry/tracing/-/tracing-6.8.0.tgz", + "integrity": "sha512-3gDkQnmOuOjHz5rY7BOatLEUksANU3efR8wuBa2ujsPQvoLSLFuyZpRjPPsxuUHQOqAYIbSNAoDloXECvQeHjw==", + "requires": { + "@sentry/hub": "6.8.0", + "@sentry/minimal": "6.8.0", + "@sentry/types": "6.8.0", + "@sentry/utils": "6.8.0", + "tslib": "^1.9.3" + } + }, + "@sentry/types": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@sentry/types/-/types-6.8.0.tgz", + "integrity": "sha512-PbSxqlh6Fd5thNU5f8EVYBVvX+G7XdPA+ThNb2QvSK8yv3rIf0McHTyF6sIebgJ38OYN7ZFK7vvhC/RgSAfYTA==" + }, + "@sentry/utils": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-6.8.0.tgz", + "integrity": "sha512-OYlI2JNrcWKMdvYbWNdQwR4QBVv2V0y5wK0U6f53nArv6RsyO5TzwRu5rMVSIZofUUqjoE5hl27jqnR+vpUrsA==", + "requires": { + "@sentry/types": "6.8.0", + "tslib": "^1.9.3" + } + }, "@sindresorhus/is": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz", @@ -15150,6 +15253,11 @@ "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz", "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI=" }, + "lru_map": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/lru_map/-/lru_map-0.3.3.tgz", + "integrity": "sha1-tcg1G5Rky9dQM1p5ZQoOwOVhGN0=" + }, "ltgt": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", diff --git a/package.json b/package.json index 3148d0da..49620267 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,8 @@ "@feathersjs/express": "^4.5.11", "@feathersjs/feathers": "^4.5.11", "@feathersjs/socketio": "^4.5.11", + "@sentry/node": "^6.8.0", + "@sentry/tracing": "^6.8.0", "async": "^3.2.0", "axios": "^0.21.1", "bignumber.js": "^8.1.1", diff --git a/src/app.hooks.js b/src/app.hooks.js index 501347a9..2c2f0452 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -1,7 +1,23 @@ // Application hooks that run for every service const auth = require('@feathersjs/authentication'); const { discard } = require('feathers-hooks-common'); -const logger = require('./hooks/logger'); +const Sentry = require('@sentry/node'); +const logger = require('winston'); +const loggerHook = require('./hooks/logger'); + +const errorHandlerHook = () => context => { + const e = context.error; + Sentry.captureException(e); + delete e.context; + + if (context.path === 'authentication') { + logger.debug(e); + } else if (context.error.name === 'NotFound') { + logger.info(`${context.path} - ${context.error.message}`); + } else { + logger.error('Hook error:', e); + } +}; const authenticate = () => context => { // socket connection is already authenticated @@ -22,7 +38,7 @@ module.exports = { }, after: { - all: [logger(), discard('__v')], + all: [loggerHook(), discard('__v')], find: [], get: [], create: [], @@ -32,7 +48,7 @@ module.exports = { }, error: { - all: [logger()], + all: [errorHandlerHook()], find: [], get: [], create: [], diff --git a/src/app.js b/src/app.js index 0ffcdb73..70356c44 100644 --- a/src/app.js +++ b/src/app.js @@ -8,6 +8,7 @@ const helmet = require('helmet'); const feathers = require('@feathersjs/feathers'); const express = require('@feathersjs/express'); const configuration = require('@feathersjs/configuration'); +const Sentry = require('@sentry/node'); const socketsConfig = require('./socketsConfig'); const configureLogger = require('./utils/configureLogger'); @@ -24,6 +25,14 @@ const channels = require('./channels'); const app = express(feathers()); +Sentry.init({ + dsn: config.sentryDsn, + // Set tracesSampleRate to 1.0 to capture 100% + // of transactions for performance monitoring. + // We recommend adjusting this value in production + tracesSampleRate: 1.0, +}); + function initFeatherApp() { // Load app configuration app.configure(configuration()); diff --git a/src/hooks/logger.js b/src/hooks/logger.js index 9dbd2b76..766984fd 100644 --- a/src/hooks/logger.js +++ b/src/hooks/logger.js @@ -22,18 +22,5 @@ module.exports = function loggerFactory() { if (hook.result) { logger.debug('hook.result', hook.result); } - - if (hook.error) { - const e = hook.error; - delete e.hook; - - if (hook.path === 'authentication') { - logger.debug(e); - } else if (hook.error.name === 'NotFound') { - logger.info(`${hook.path} - ${hook.error.message}`); - } else { - logger.error('Hook error:', e); - } - } }; }; From e2605b6181376d96dd6fa3a4baabf2e0eac96833 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Thu, 1 Jul 2021 15:25:07 +0430 Subject: [PATCH 26/47] Allow campaign coowner to edit trace related to #541 --- src/services/traces/getApprovedKeys.js | 48 +++++++++++------- src/services/traces/getApprovedKeys.test.js | 54 ++++++++++++++++++++- src/services/traces/traces.service.js | 2 +- 3 files changed, 84 insertions(+), 20 deletions(-) diff --git a/src/services/traces/getApprovedKeys.js b/src/services/traces/getApprovedKeys.js index 88dd7932..2b17d709 100644 --- a/src/services/traces/getApprovedKeys.js +++ b/src/services/traces/getApprovedKeys.js @@ -76,9 +76,11 @@ const getApprovedKeys = (trace, data, user) => { // Editing trace can be done by Trace or Campaign Manager if (data.status === TraceStatus.PROPOSED) { if ( - ![trace.ownerAddress, trace.campaign.ownerAddress, trace.campaign.coownerAddess].includes( - user.address, - ) + ![ + trace.ownerAddress, + trace.campaign.ownerAddress, + trace.campaign.coownerAddress, + ].includes(user.address) ) { throw new errors.Forbidden('Only the Trace or Campaign Manager can edit proposed trace'); } @@ -127,9 +129,11 @@ const getApprovedKeys = (trace, data, user) => { // Archive trace by Trace Manager or Campaign Manager if (data.status === TraceStatus.ARCHIVED) { if ( - ![trace.campaign.ownerAddress, trace.campaign.coownerAddess, trace.ownerAddress].includes( - user.address, - ) + ![ + trace.campaign.ownerAddress, + trace.campaign.coownerAddress, + trace.ownerAddress, + ].includes(user.address) ) { throw new errors.Forbidden( 'Only the Trace Manager or Campaign Manager can archive a trace', @@ -150,9 +154,11 @@ const getApprovedKeys = (trace, data, user) => { // Editing trace can be done by Campaign or Trace Manager if (data.status === TraceStatus.IN_PROGRESS) { if ( - ![trace.ownerAddress, trace.campaign.ownerAddress, trace.campaign.coownerAddess].includes( - user.address, - ) + ![ + trace.ownerAddress, + trace.campaign.ownerAddress, + trace.campaign.coownerAddress, + ].includes(user.address) ) { throw new errors.Forbidden('Only the Trace and Campaign Manager can edit trace'); } @@ -163,9 +169,11 @@ const getApprovedKeys = (trace, data, user) => { // Editing a proposed trace can be done by either manager and all usual properties can be changed since it's not on chain if (data.status === TraceStatus.PROPOSED) { if ( - ![trace.ownerAddress, trace.campaign.ownerAddress, trace.campaign.coownerAddess].includes( - user.address, - ) + ![ + trace.ownerAddress, + trace.campaign.ownerAddress, + trace.campaign.coownerAddress, + ].includes(user.address) ) { throw new errors.Forbidden('Only the Trace and Campaign Manager can edit proposed trace'); } @@ -212,9 +220,11 @@ const getApprovedKeys = (trace, data, user) => { // Editing trace can be done by Trace or Campaign Manager if (data.status === TraceStatus.NEEDS_REVIEW) { if ( - ![trace.ownerAddress, trace.campaign.ownerAddress, trace.campaign.coownerAddess].includes( - user.address, - ) + ![ + trace.ownerAddress, + trace.campaign.ownerAddress, + trace.campaign.coownerAddress, + ].includes(user.address) ) { throw new errors.Forbidden('Only the Trace and Campaign Manager can edit trace'); } @@ -240,9 +250,11 @@ const getApprovedKeys = (trace, data, user) => { if (data.status === TraceStatus.ARCHIVED) { if ( - ![trace.campaign.ownerAddress, trace.campaign.coownerAddess, trace.ownerAddress].includes( - user.address, - ) + ![ + trace.campaign.ownerAddress, + trace.campaign.coownerAddress, + trace.ownerAddress, + ].includes(user.address) ) { throw new errors.Forbidden( 'Only the Trace Manager or Campaign Manager can archive a trace', diff --git a/src/services/traces/getApprovedKeys.test.js b/src/services/traces/getApprovedKeys.test.js index 720e9379..8da0114d 100644 --- a/src/services/traces/getApprovedKeys.test.js +++ b/src/services/traces/getApprovedKeys.test.js @@ -1,6 +1,6 @@ const { assert, expect } = require('chai'); const getApprovedKeys = require('./getApprovedKeys'); -const { SAMPLE_DATA } = require('../../../test/testUtility'); +const { SAMPLE_DATA, generateRandomEtheriumAddress } = require('../../../test/testUtility'); function getApprovedKeysTestCases() { const trace = { @@ -301,6 +301,58 @@ function getApprovedKeysTestCases() { } }); + it('should not throw exception when campaign coowner trying to edit inProgress trace', () => { + const campaignCoownerAddress = generateRandomEtheriumAddress(); + + const goodFunc = () => { + getApprovedKeys( + { + ...trace, + status: SAMPLE_DATA.TRACE_STATUSES.IN_PROGRESS, + campaign: { + coownerAddress: campaignCoownerAddress, + }, + }, + { + status: SAMPLE_DATA.TRACE_STATUSES.IN_PROGRESS, + mined: false, + }, + { + address: campaignCoownerAddress, + }, + ); + }; + assert.doesNotThrow(goodFunc); + }); + + it( + 'should throw exception when someone except campaign coowner,' + + ' campaignOwner and traceOwner trying to edit trace', + () => { + const campaignCoownerAddress = generateRandomEtheriumAddress(); + + const badFunc = () => { + getApprovedKeys( + { + ...trace, + status: SAMPLE_DATA.TRACE_STATUSES.IN_PROGRESS, + campaign: { + coownerAddress: campaignCoownerAddress, + }, + }, + { + status: SAMPLE_DATA.TRACE_STATUSES.IN_PROGRESS, + mined: false, + }, + { + address: generateRandomEtheriumAddress(), + }, + ); + }; + assert.throw(badFunc, 'Only the Trace and Campaign Manager can edit trace'); + }, + ); + it('should throw exception, Only the Trace or Campaign Reviewer can approve trace has been completed', () => { const badFunc = () => { getApprovedKeys( diff --git a/src/services/traces/traces.service.js b/src/services/traces/traces.service.js index 66f02858..b4c4f98f 100644 --- a/src/services/traces/traces.service.js +++ b/src/services/traces/traces.service.js @@ -4,7 +4,7 @@ const { createModel } = require('../../models/traces.model'); const hooks = require('./traces.hooks'); const { defaultFeatherMongooseOptions } = require('../serviceCommons'); -module.exports = function milestones() { +module.exports = function traces() { const app = this; const Model = createModel(app); const paginate = app.get('paginate'); From b7747cb97408a0b4ce838902139a886d1bb961d5 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 3 Jul 2021 08:05:26 +0430 Subject: [PATCH 27/47] Put back the error hook in to responseLogger hook --- src/app.hooks.js | 22 +++------------------- src/hooks/logger.js | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/app.hooks.js b/src/app.hooks.js index 2c2f0452..2245eccf 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -1,23 +1,7 @@ // Application hooks that run for every service const auth = require('@feathersjs/authentication'); const { discard } = require('feathers-hooks-common'); -const Sentry = require('@sentry/node'); -const logger = require('winston'); -const loggerHook = require('./hooks/logger'); - -const errorHandlerHook = () => context => { - const e = context.error; - Sentry.captureException(e); - delete e.context; - - if (context.path === 'authentication') { - logger.debug(e); - } else if (context.error.name === 'NotFound') { - logger.info(`${context.path} - ${context.error.message}`); - } else { - logger.error('Hook error:', e); - } -}; +const responseLoggerHook = require('./hooks/logger'); const authenticate = () => context => { // socket connection is already authenticated @@ -38,7 +22,7 @@ module.exports = { }, after: { - all: [loggerHook(), discard('__v')], + all: [responseLoggerHook(), discard('__v')], find: [], get: [], create: [], @@ -48,7 +32,7 @@ module.exports = { }, error: { - all: [errorHandlerHook()], + all: [responseLoggerHook()], find: [], get: [], create: [], diff --git a/src/hooks/logger.js b/src/hooks/logger.js index 766984fd..c1fe023f 100644 --- a/src/hooks/logger.js +++ b/src/hooks/logger.js @@ -1,7 +1,8 @@ // A hook that logs service method before, after and error const logger = require('winston'); +const Sentry = require('@sentry/node'); -module.exports = function loggerFactory() { +module.exports = function responseLoggerHook() { return function log(hook) { let message = `${hook.type}: ${hook.path} - Method: ${hook.method}`; @@ -22,5 +23,19 @@ module.exports = function loggerFactory() { if (hook.result) { logger.debug('hook.result', hook.result); } + + if (hook.error) { + const e = hook.error; + Sentry.captureException(e); + delete e.hook; + + if (hook.path === 'authentication') { + logger.debug(e); + } else if (hook.error.name === 'NotFound') { + logger.info(`${hook.path} - ${hook.error.message}`); + } else { + logger.error('Hook error:', e); + } + } }; }; From f331447cd9fd528cfc5efb3863e080b1351adc70 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 3 Jul 2021 10:15:25 +0430 Subject: [PATCH 28/47] Add sentry monitoring to feathers hook related to Giveth/giveth-dapp#2301 --- config/default.json | 3 +- src/app.hooks.js | 5 +-- src/hooks/logger.js | 48 +++++++++++++++++++++-- src/services/whitelist/whitelist.hooks.js | 1 + 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/config/default.json b/config/default.json index 30c89b11..5cd804f8 100644 --- a/config/default.json +++ b/config/default.json @@ -176,5 +176,6 @@ "elasticSearchUsername": "elastic", "elasticSearchPassword": "changeme", "enableAuditLog": false, - "sentryDsn": "" + "enableMonitoring": true, + "sentryDsn": "https://17220e8c93224bc288f600551f416015@o218966.ingest.sentry.io/5840267" } diff --git a/src/app.hooks.js b/src/app.hooks.js index 2245eccf..a8b91fbd 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -1,18 +1,17 @@ // Application hooks that run for every service const auth = require('@feathersjs/authentication'); const { discard } = require('feathers-hooks-common'); -const responseLoggerHook = require('./hooks/logger'); +const { responseLoggerHook, startMonitoring } = require('./hooks/logger'); const authenticate = () => context => { // socket connection is already authenticated if (context.params.provider !== 'rest') return context; - return auth.hooks.authenticate('jwt')(context); }; module.exports = { before: { - all: [], + all: [startMonitoring()], find: [], get: [], create: [authenticate()], diff --git a/src/hooks/logger.js b/src/hooks/logger.js index c1fe023f..d6bc9e8a 100644 --- a/src/hooks/logger.js +++ b/src/hooks/logger.js @@ -1,11 +1,41 @@ // A hook that logs service method before, after and error const logger = require('winston'); const Sentry = require('@sentry/node'); +const config = require('config'); +const { isRequestInternal } = require('../utils/feathersUtils'); -module.exports = function responseLoggerHook() { +const startMonitoring = () => context => { + /** + * inpired by official sentry middleware for express + * @see{@link https://github.com/getsentry/sentry-javascript/blob/ab0bc9313a798403dbaeae1e3d867cdf7841d6e4/packages/node/src/handlers.ts#L62-L93} + */ + // Add monitoring for external requests + if ( + !config.enableMonitoring || + isRequestInternal(context) || + // internal calls that use the external context doesnt have headers + !context.params.headers + ) + return context; + const transaction = Sentry.startTransaction({ + name: `${context.path}-${context.method}`, + method: context.method, + op: context.params.provider, + }); + // const span = transaction.startChild({ + // data: { + // }, + // op: 'task', + // description: `processing shopping cart result`, + // }); + context.__sentry_transaction = transaction; + return context; +}; + +const responseLoggerHook = () => { return function log(hook) { let message = `${hook.type}: ${hook.path} - Method: ${hook.method}`; - + const sentryTransaction = hook.__sentry_transaction; if (hook.type === 'error') { message += ` - ${hook.error.message}`; } @@ -23,10 +53,20 @@ module.exports = function responseLoggerHook() { if (hook.result) { logger.debug('hook.result', hook.result); } + // I think when hook.params._populate is equal to 'skip` it means we have internal calls + // that use an extenral call context + if (sentryTransaction && !hook.params._populate) { + sentryTransaction.setHttpStatus(hook.statusCode); + sentryTransaction.finish(); + } if (hook.error) { const e = hook.error; - Sentry.captureException(e); + + // for making sure the feathers errors like unAuthorized wouldn't capture as exceptions + if (e.type !== 'FeathersError') { + Sentry.captureException(e); + } delete e.hook; if (hook.path === 'authentication') { @@ -39,3 +79,5 @@ module.exports = function responseLoggerHook() { } }; }; + +module.exports = { responseLoggerHook, startMonitoring }; diff --git a/src/services/whitelist/whitelist.hooks.js b/src/services/whitelist/whitelist.hooks.js index 0a052d7e..a48ccaae 100644 --- a/src/services/whitelist/whitelist.hooks.js +++ b/src/services/whitelist/whitelist.hooks.js @@ -3,6 +3,7 @@ const { getTokenBySymbol } = require('../../utils/tokenHelper'); const getWhitelist = () => context => { const { app } = context; + x=y; // fetch whitelisted addresses from default.json const reviewerWhitelistEnabled = !!app.get('useReviewerWhitelist'); const delegateWhitelistEnabled = !!app.get('useDelegateWhitelist'); From 82e586f019a1318f737060136f16d8099fa5fb9c Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 3 Jul 2021 10:17:02 +0430 Subject: [PATCH 29/47] Config Sentry to just capture non-featehrs error --- src/hooks/logger.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hooks/logger.js b/src/hooks/logger.js index c1fe023f..ec594843 100644 --- a/src/hooks/logger.js +++ b/src/hooks/logger.js @@ -26,7 +26,10 @@ module.exports = function responseLoggerHook() { if (hook.error) { const e = hook.error; - Sentry.captureException(e); + // for making sure the feathers errors like unAuthorized wouldn't capture as exceptions + if (e.type !== 'FeathersError'){ + Sentry.captureException(e); + } delete e.hook; if (hook.path === 'authentication') { From a4316149e90af73637f40e141c6d9db25011a42c Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 3 Jul 2021 10:20:13 +0430 Subject: [PATCH 30/47] Fix eslint errors --- src/hooks/logger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hooks/logger.js b/src/hooks/logger.js index ec594843..b48c029a 100644 --- a/src/hooks/logger.js +++ b/src/hooks/logger.js @@ -27,7 +27,7 @@ module.exports = function responseLoggerHook() { if (hook.error) { const e = hook.error; // for making sure the feathers errors like unAuthorized wouldn't capture as exceptions - if (e.type !== 'FeathersError'){ + if (e.type !== 'FeathersError') { Sentry.captureException(e); } delete e.hook; From 38554c9388abc99a8cc0ac3c7c6294ec5d8d7365 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 3 Jul 2021 10:38:06 +0430 Subject: [PATCH 31/47] Add environment and release to sentry when initializing --- src/app.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/app.js b/src/app.js index 70356c44..e7596b4f 100644 --- a/src/app.js +++ b/src/app.js @@ -24,9 +24,10 @@ const { configureAuditLog } = require('./auditLog/feathersElasticSearch'); const channels = require('./channels'); const app = express(feathers()); - Sentry.init({ dsn: config.sentryDsn, + environment: process.env.NODE_ENV, + release: `Giveth-Feathers@${process.env.npm_package_version}`, // Set tracesSampleRate to 1.0 to capture 100% // of transactions for performance monitoring. // We recommend adjusting this value in production From ffe7bdf45b1557f9494a3c2b28bf5f66a0f785c1 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sat, 3 Jul 2021 10:44:46 +0430 Subject: [PATCH 32/47] Set traceSampleRate for sentry --- src/app.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/app.js b/src/app.js index 70356c44..6df023c0 100644 --- a/src/app.js +++ b/src/app.js @@ -27,10 +27,15 @@ const app = express(feathers()); Sentry.init({ dsn: config.sentryDsn, - // Set tracesSampleRate to 1.0 to capture 100% - // of transactions for performance monitoring. - // We recommend adjusting this value in production - tracesSampleRate: 1.0, + // we want to capture 100% of errors + sampleRate: 1, + + /** + * @see{@link https://docs.sentry.io/platforms/node/configuration/sampling/#setting-a-uniform-sample-rate} + */ + // Set tracesSampleRate to 1.0 to capture 100% of transactions for performance monitoring. + // But we recommend adjusting this value in production + tracesSampleRate: 0.1, }); function initFeatherApp() { From 9361bb080161bc3bc92d304674db51e83dc9ff76 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 4 Jul 2021 15:00:48 +0430 Subject: [PATCH 33/47] Add test case --- src/services/traces/traces.service.test.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/services/traces/traces.service.test.js b/src/services/traces/traces.service.test.js index a53c5317..fd8a9ee4 100644 --- a/src/services/traces/traces.service.test.js +++ b/src/services/traces/traces.service.test.js @@ -1,7 +1,13 @@ const request = require('supertest'); const config = require('config'); const { assert, expect } = require('chai'); -const { getJwt, SAMPLE_DATA, generateRandomMongoId } = require('../../../test/testUtility'); +const { + getJwt, + SAMPLE_DATA, + generateRandomMongoId, + generateRandomEtheriumAddress, + assertThrowsAsync, +} = require('../../../test/testUtility'); const { getFeatherAppInstance } = require('../../app'); let app; @@ -102,6 +108,16 @@ function postMilestoneTestCases() { const trace = await createTrace(createMileStoneData); assert.isFalse(trace.verified); }); + + it('should throw exception with create trace with ownerAddress of another user', async () => { + const createMileStoneData = { ...SAMPLE_DATA.createTraceData(), verified: true }; + createMileStoneData.status = SAMPLE_DATA.TRACE_STATUSES.PROPOSED; + createMileStoneData.ownerAddress = generateRandomEtheriumAddress(); + const badFunc = async () => { + await createTrace(createMileStoneData); + }; + await assertThrowsAsync(badFunc, 'user can create trace with his/her address'); + }); } function patchMilestoneTestCases() { From 96d7826045a4e6825035567ab26a9dc45ecab17f Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 4 Jul 2021 15:08:44 +0430 Subject: [PATCH 34/47] Remove sentryDsn from config --- config/default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.json b/config/default.json index 5cd804f8..ad3163eb 100644 --- a/config/default.json +++ b/config/default.json @@ -177,5 +177,5 @@ "elasticSearchPassword": "changeme", "enableAuditLog": false, "enableMonitoring": true, - "sentryDsn": "https://17220e8c93224bc288f600551f416015@o218966.ingest.sentry.io/5840267" + "sentryDsn": "" } From 3466a60c5bab0bfb4b5313f5edd664e1b9b3afe7 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 4 Jul 2021 15:09:51 +0430 Subject: [PATCH 35/47] Remove unused codes --- dump.rdb | Bin 0 -> 92 bytes src/hooks/logger.js | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 dump.rdb diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..0e73e559a6b9361f90221e91a131ad0dc4247312 GIT binary patch literal 92 zcmWG?b@2=~Ffg$E#aWb^l3A=&fgPk8#j0suAxCC2~& literal 0 HcmV?d00001 diff --git a/src/hooks/logger.js b/src/hooks/logger.js index 63f286ba..8afd9809 100644 --- a/src/hooks/logger.js +++ b/src/hooks/logger.js @@ -6,7 +6,7 @@ const { isRequestInternal } = require('../utils/feathersUtils'); const startMonitoring = () => context => { /** - * inpired by official sentry middleware for express + * inspired by official sentry middleware for express * @see{@link https://github.com/getsentry/sentry-javascript/blob/ab0bc9313a798403dbaeae1e3d867cdf7841d6e4/packages/node/src/handlers.ts#L62-L93} */ // Add monitoring for external requests From f30893d5212eaae8fa60aa34aa434d9e0746ba30 Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Sun, 4 Jul 2021 15:52:15 +0430 Subject: [PATCH 36/47] Rename variable and its intial value --- config/default.json | 2 +- src/hooks/logger.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/default.json b/config/default.json index ad3163eb..96054384 100644 --- a/config/default.json +++ b/config/default.json @@ -176,6 +176,6 @@ "elasticSearchUsername": "elastic", "elasticSearchPassword": "changeme", "enableAuditLog": false, - "enableMonitoring": true, + "enableSentryMonitoring": false, "sentryDsn": "" } diff --git a/src/hooks/logger.js b/src/hooks/logger.js index 8afd9809..c552313f 100644 --- a/src/hooks/logger.js +++ b/src/hooks/logger.js @@ -11,7 +11,7 @@ const startMonitoring = () => context => { */ // Add monitoring for external requests if ( - !config.enableMonitoring || + !config.enableSentryMonitoring || isRequestInternal(context) || // internal calls that use the external context doesnt have headers !context.params.headers From e47911d0988f3d13dbfbfbed0325f2d7b9df742a Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 4 Jul 2021 22:18:29 +0430 Subject: [PATCH 37/47] Add aithentication for socketio requests related to #543 --- src/app.hooks.js | 23 +++++++++++++++++----- src/services/traces/traces.service.test.js | 9 +++------ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/app.hooks.js b/src/app.hooks.js index 36e1916a..80496629 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -1,16 +1,29 @@ // Application hooks that run for every service const auth = require('@feathersjs/authentication'); const { discard } = require('feathers-hooks-common'); +const { NotAuthenticated } = require('@feathersjs/errors'); const logger = require('./hooks/logger'); +const { isRequestInternal } = require('./utils/feathersUtils'); const authenticate = () => context => { - // socket connection is already authenticated - if (context.params.provider !== 'rest') return context; + // No need to authenticate internal calls + if (isRequestInternal(context)) return context; - return auth.hooks.authenticate('jwt')(context); + // socket connection is already authenticated, we just check if user has been set on context.params + if (context.params.provider === 'socketio' && context.params.user) { + return context; + } + // if the path is authentication that means user wants to login and get accessToken + if (context.params.provider === 'socketio' && context.path === 'authentication') { + return context; + } + if (context.params.provider === 'rest') { + return auth.hooks.authenticate('jwt')(context); + } + throw new NotAuthenticated(); }; -const convertVerfiedToBoolean = () => context => { +const convertVerifiedToBoolean = () => context => { // verified field is boolean in Trace, Campaign and Community so for getting this filter // in query string we should cast it to boolean here if (context.params.query && context.params.query.verified === 'true') { @@ -24,7 +37,7 @@ const convertVerfiedToBoolean = () => context => { module.exports = { before: { all: [], - find: [convertVerfiedToBoolean()], + find: [convertVerifiedToBoolean()], get: [], create: [authenticate()], update: [authenticate()], diff --git a/src/services/traces/traces.service.test.js b/src/services/traces/traces.service.test.js index fd8a9ee4..0befdfa9 100644 --- a/src/services/traces/traces.service.test.js +++ b/src/services/traces/traces.service.test.js @@ -6,7 +6,6 @@ const { SAMPLE_DATA, generateRandomMongoId, generateRandomEtheriumAddress, - assertThrowsAsync, } = require('../../../test/testUtility'); const { getFeatherAppInstance } = require('../../app'); @@ -109,14 +108,12 @@ function postMilestoneTestCases() { assert.isFalse(trace.verified); }); - it('should throw exception with create trace with ownerAddress of another user', async () => { + it('should set userAddress as ownerAddress of trace, doesnt matter what you send', async () => { const createMileStoneData = { ...SAMPLE_DATA.createTraceData(), verified: true }; createMileStoneData.status = SAMPLE_DATA.TRACE_STATUSES.PROPOSED; createMileStoneData.ownerAddress = generateRandomEtheriumAddress(); - const badFunc = async () => { - await createTrace(createMileStoneData); - }; - await assertThrowsAsync(badFunc, 'user can create trace with his/her address'); + const trace = await createTrace(createMileStoneData); + assert.equal(trace.ownerAddress, SAMPLE_DATA.USER_ADDRESS ) }); } From 0aae937db11af07b44f0b678447e067406101f14 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 4 Jul 2021 22:29:10 +0430 Subject: [PATCH 38/47] Fix eslint errors --- src/services/traces/traces.service.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/traces/traces.service.test.js b/src/services/traces/traces.service.test.js index 0befdfa9..c7a99fc2 100644 --- a/src/services/traces/traces.service.test.js +++ b/src/services/traces/traces.service.test.js @@ -113,7 +113,7 @@ function postMilestoneTestCases() { createMileStoneData.status = SAMPLE_DATA.TRACE_STATUSES.PROPOSED; createMileStoneData.ownerAddress = generateRandomEtheriumAddress(); const trace = await createTrace(createMileStoneData); - assert.equal(trace.ownerAddress, SAMPLE_DATA.USER_ADDRESS ) + assert.equal(trace.ownerAddress, SAMPLE_DATA.USER_ADDRESS); }); } From 4e58047c8764407fd24fb09d09429f4dccd76bda Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Tue, 6 Jul 2021 12:27:58 +0430 Subject: [PATCH 39/47] removed dump.rdb --- dump.rdb | Bin 92 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dump.rdb diff --git a/dump.rdb b/dump.rdb deleted file mode 100644 index 0e73e559a6b9361f90221e91a131ad0dc4247312..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 92 zcmWG?b@2=~Ffg$E#aWb^l3A=&fgPk8#j0suAxCC2~& From 81b5135f02487bf069ddb4242fd161f8688c2f0c Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Wed, 7 Jul 2021 18:53:23 +0430 Subject: [PATCH 40/47] Capture exception and sent to sentry when can not connect to mongo --- src/mongoose.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/mongoose.js b/src/mongoose.js index 03c6fc9c..5c059de7 100644 --- a/src/mongoose.js +++ b/src/mongoose.js @@ -2,6 +2,7 @@ const mongoose = require('mongoose'); require('mongoose-long')(mongoose); require('./models/mongoose-bn')(mongoose); const logger = require('winston'); +const Sentry = require('@sentry/node'); // mongoose query hook function that will // remove the key from the doc if the value is undefined @@ -36,7 +37,10 @@ module.exports = function mongooseFactory() { }); const db = mongoose.connection; - db.on('error', err => logger.error('Could not connect to Mongo', err)); + db.on('error', err => { + logger.error('Could not connect to Mongo', err); + Sentry.captureException(err); + }); db.once('open', () => logger.info('Connected to Mongo')); mongoose.plugin(schema => { From 0b613533e0d3306bc7e246b07c36032ef32c9078 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Thu, 8 Jul 2021 11:18:50 +0430 Subject: [PATCH 41/47] Dont check authentication for creating donations related to #551 --- src/app.hooks.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/app.hooks.js b/src/app.hooks.js index 60162ee9..e6151e46 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -17,6 +17,14 @@ const authenticate = () => context => { if (context.params.provider === 'socketio' && context.path === 'authentication') { return context; } + if ( + context.params.provider === 'socketio' && + context.path === 'donations' && + context.method === 'create' + ) { + // for creating donations it's not needed to be authenticated, anonymous users can donate + return context; + } if (context.params.provider === 'rest') { return auth.hooks.authenticate('jwt')(context); } From 121f0363298164d1b4b89477ebbef7732e6ae553 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Thu, 8 Jul 2021 18:46:28 +0430 Subject: [PATCH 42/47] Add analytics endpoint for giveth-dapp use related to #548 --- config/default.json | 3 +- package-lock.json | 54 +++++++++++-- package.json | 1 + src/app.hooks.js | 4 +- src/services/analytics/analytics.service.js | 12 +++ .../analytics/analytics.service.test.js | 50 ++++++++++++ src/services/index.js | 2 + src/utils/analyticsUtils.js | 78 +++++++++++++++++++ 8 files changed, 197 insertions(+), 7 deletions(-) create mode 100644 src/services/analytics/analytics.service.js create mode 100644 src/services/analytics/analytics.service.test.js create mode 100644 src/utils/analyticsUtils.js diff --git a/config/default.json b/config/default.json index 96054384..06c8d068 100644 --- a/config/default.json +++ b/config/default.json @@ -177,5 +177,6 @@ "elasticSearchPassword": "changeme", "enableAuditLog": false, "enableSentryMonitoring": false, - "sentryDsn": "" + "sentryDsn": "", + "segmentApiKey": "" } diff --git a/package-lock.json b/package-lock.json index 5cfab16f..b8d8e780 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2712,6 +2712,15 @@ "any-observable": "^0.3.0" } }, + "@segment/loosely-validate-event": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@segment/loosely-validate-event/-/loosely-validate-event-2.0.0.tgz", + "integrity": "sha512-ZMCSfztDBqwotkl848ODgVcAmN4OItEWDCkshcKz0/W6gGSQayuuCtWV/MlodFivAZD793d6UgANd6wCXUfrIw==", + "requires": { + "component-type": "^1.2.1", + "join-component": "^1.1.0" + } + }, "@sentry/core": { "version": "6.8.0", "resolved": "https://registry.npmjs.org/@sentry/core/-/core-6.8.0.tgz", @@ -3135,6 +3144,21 @@ "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", "optional": true }, + "analytics-node": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/analytics-node/-/analytics-node-4.0.1.tgz", + "integrity": "sha512-+zXOOTB+eTRW6R9+pfvPfk1dHraFJzhNnAyZiYJIDGOjHQgfk9qfqgoJX9MfR4qY0J/E1YJ3FBncrLGadTDW1A==", + "requires": { + "@segment/loosely-validate-event": "^2.0.0", + "axios": "^0.21.1", + "axios-retry": "^3.0.2", + "lodash.isstring": "^4.0.1", + "md5": "^2.2.1", + "ms": "^2.0.0", + "remove-trailing-slash": "^0.1.0", + "uuid": "^3.2.1" + } + }, "ansi-bgblack": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/ansi-bgblack/-/ansi-bgblack-0.1.1.tgz", @@ -3837,6 +3861,14 @@ } } }, + "axios-retry": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-3.1.9.tgz", + "integrity": "sha512-NFCoNIHq8lYkJa6ku4m+V1837TP6lCa7n79Iuf8/AqATAHYB0ISaAS1eyIenDOfHOLtym34W65Sjke2xjg2fsA==", + "requires": { + "is-retry-allowed": "^1.1.0" + } + }, "babel-code-frame": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", @@ -5502,8 +5534,7 @@ "charenc": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=", - "dev": true + "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" }, "check-error": { "version": "1.0.2", @@ -5980,6 +6011,11 @@ "resolved": "https://registry.npmjs.org/component-inherit/-/component-inherit-0.0.3.tgz", "integrity": "sha1-ZF/ErfWLcrZJ1crmUTVhnbJv8UM=" }, + "component-type": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/component-type/-/component-type-1.2.1.tgz", + "integrity": "sha1-ikeQFwAjjk/DIml3EjAibyS0Fak=" + }, "compressible": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", @@ -6612,8 +6648,7 @@ "crypt": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=", - "dev": true + "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" }, "crypto-browserify": { "version": "3.12.0", @@ -13213,6 +13248,11 @@ } } }, + "join-component": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/join-component/-/join-component-1.1.0.tgz", + "integrity": "sha1-uEF7dQZho5K+4sJTfGiyqdSXfNU=" + }, "js-sha3": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.6.1.tgz", @@ -15386,7 +15426,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", - "dev": true, "requires": { "charenc": "0.0.2", "crypt": "0.0.2", @@ -19258,6 +19297,11 @@ "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=" }, + "remove-trailing-slash": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/remove-trailing-slash/-/remove-trailing-slash-0.1.1.tgz", + "integrity": "sha512-o4S4Qh6L2jpnCy83ysZDau+VORNvnFw07CKSAymkd6ICNVEPisMyzlc00KlvvicsxKck94SEwhDnMNdICzO+tA==" + }, "repeat-element": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.3.tgz", diff --git a/package.json b/package.json index 49620267..f7bf3bcf 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "@feathersjs/socketio": "^4.5.11", "@sentry/node": "^6.8.0", "@sentry/tracing": "^6.8.0", + "analytics-node": "^4.0.1", "async": "^3.2.0", "axios": "^0.21.1", "bignumber.js": "^8.1.1", diff --git a/src/app.hooks.js b/src/app.hooks.js index e6151e46..cb20b075 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -8,7 +8,9 @@ const { responseLoggerHook, startMonitoring } = require('./hooks/logger'); const authenticate = () => context => { // No need to authenticate internal calls if (isRequestInternal(context)) return context; - + if (context.path === 'analytics') { + return context; + } // socket connection is already authenticated, we just check if user has been set on context.params if (context.params.provider === 'socketio' && context.params.user) { return context; diff --git a/src/services/analytics/analytics.service.js b/src/services/analytics/analytics.service.js new file mode 100644 index 00000000..e7599f53 --- /dev/null +++ b/src/services/analytics/analytics.service.js @@ -0,0 +1,12 @@ +const { sendAnalytics } = require('../../utils/analyticsUtils'); + +module.exports = function analytics() { + const app = this; + const analyticsService = { + async create(data, params) { + const result = sendAnalytics({ data, params }); + return result; + }, + }; + app.use('/analytics', analyticsService); +}; diff --git a/src/services/analytics/analytics.service.test.js b/src/services/analytics/analytics.service.test.js new file mode 100644 index 00000000..dc41fdb1 --- /dev/null +++ b/src/services/analytics/analytics.service.test.js @@ -0,0 +1,50 @@ +const request = require('supertest'); +const config = require('config'); +const { assert } = require('chai'); +const { getJwt } = require('../../../test/testUtility'); +const { getFeatherAppInstance } = require('../../app'); + +const app = getFeatherAppInstance(); +const baseUrl = config.get('givethFathersBaseUrl'); +const relativeUrl = '/analytics'; +const pageType = 'page'; +const trackingType = 'track'; +function postAnalyticsTestCases() { + it('should return successful when sending page', async () => { + const response = await request(baseUrl) + .post(relativeUrl) + .send({ + reportType: pageType, + }) + .set({ Authorization: getJwt() }); + assert.equal(response.statusCode, 201); + }); + + it('should return successful when sending tracking', async () => { + const response = await request(baseUrl) + .post(relativeUrl) + .send({ + reportType: trackingType, + }) + .set({ Authorization: getJwt() }); + assert.equal(response.statusCode, 201); + }); + + it('should get 400, invalid reportType', async () => { + const reportType = 'invalidReportType'; + const response = await request(baseUrl) + .post(relativeUrl) + .send({ + reportType, + }) + .set({ Authorization: getJwt() }); + assert.equal(response.statusCode, 400); + }); +} + +it('should analytics service registration be ok', () => { + const service = app.service('analytics'); + assert.ok(service, 'Registered the service'); +}); + +describe(`Test POST ${relativeUrl}`, postAnalyticsTestCases); diff --git a/src/services/index.js b/src/services/index.js index 6beb8fd1..d3e70eb0 100644 --- a/src/services/index.js +++ b/src/services/index.js @@ -4,6 +4,7 @@ const events = require('./events/events.service'); const homePaymentsTransactions = require('./homePaymentsTransactions/homePaymentsTransactions.service'); const emails = require('./emails/emails.service'); const subscription = require('./subscriptions/subscription.service'); +const analytics = require('./analytics/analytics.service'); const communities = require('./communities/communities.service.js'); const milestones = require('./traces/traces.service.js'); @@ -39,6 +40,7 @@ module.exports = function configure() { app.configure(homePaymentsTransactions); app.configure(emails); app.configure(subscription); + app.configure(analytics); app.configure(conversations); app.configure(campaigncsv); app.configure(givbackReportDonations); diff --git a/src/utils/analyticsUtils.js b/src/utils/analyticsUtils.js new file mode 100644 index 00000000..af362757 --- /dev/null +++ b/src/utils/analyticsUtils.js @@ -0,0 +1,78 @@ +const config = require('config'); +const Analytics = require('analytics-node'); +const logger = require('winston'); +const { BadRequest } = require('@feathersjs/errors'); + +let analytics; +if (config.segmentApiKey) { + analytics = new Analytics(config.segmentApiKey); +} else { + logger.info('You dont have segmentApiKey in your config, so analytics is disabled'); +} + +const track = data => { + if (!analytics) { + return; + } + try { + logger.error('send segment tracking', data); + analytics.track(data); + } catch (e) { + logger.error('send segment tracking error', { e, data }); + } +}; + +const page = data => { + if (!analytics) { + return; + } + try { + logger.debug('send segment page', data); + analytics.page(data); + } catch (e) { + logger.error('send segment page error', { e, data }); + } +}; + +const sendAnalytics = ({ data, params }) => { + const dataContext = { + ...data.context, + /** + * @see{@link https://atlassc.net/2020/02/25/feathersjs-client-real-ip} + */ + ip: params.headers['x-real-ip'], + userAgent: params.headers['user-agent'], + }; + const eventData = { + userId: data.userId, + context: dataContext, + userAgent: params.headers['user-agent'], + properties: data.properties, + }; + if (!eventData.userId){ + eventData.anonymousId = data.anonymousId; + } + if (data.reportType === 'track') { + track({ + ...eventData, + event: data.event, + }); + return { + message: 'success', + }; + } + if (data.reportType === 'page') { + page({ + ...eventData, + name: data.page, + }); + return { + message: 'success', + }; + } + throw new BadRequest('invalid reportType'); +}; + +module.exports = { + sendAnalytics, +}; From fc817a98604de1d1e8ba695ebb8d9b7aec876586 Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Thu, 8 Jul 2021 18:51:25 +0430 Subject: [PATCH 43/47] Fix eslint errors --- src/utils/analyticsUtils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/analyticsUtils.js b/src/utils/analyticsUtils.js index af362757..77257c2f 100644 --- a/src/utils/analyticsUtils.js +++ b/src/utils/analyticsUtils.js @@ -49,7 +49,7 @@ const sendAnalytics = ({ data, params }) => { userAgent: params.headers['user-agent'], properties: data.properties, }; - if (!eventData.userId){ + if (!eventData.userId) { eventData.anonymousId = data.anonymousId; } if (data.reportType === 'track') { From f75e5effc3e27f7171ed46215ae79aebe60dc24c Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 11 Jul 2021 10:38:13 +0430 Subject: [PATCH 44/47] Dont check authentication for update donations related Giveth/giveth-dapp#2358 --- src/app.hooks.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/app.hooks.js b/src/app.hooks.js index e6151e46..19aa70ce 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -17,12 +17,8 @@ const authenticate = () => context => { if (context.params.provider === 'socketio' && context.path === 'authentication') { return context; } - if ( - context.params.provider === 'socketio' && - context.path === 'donations' && - context.method === 'create' - ) { - // for creating donations it's not needed to be authenticated, anonymous users can donate + if ( context.params.provider === 'socketio' && context.path === 'donations') { + // for creating and updating donations it's not needed to be authenticated, anonymous users can donate return context; } if (context.params.provider === 'rest') { From 4dfd73fa9880f94f7473616f772ec8c34c6a55dd Mon Sep 17 00:00:00 2001 From: Mohammad Ranjbar Z Date: Sun, 11 Jul 2021 11:02:13 +0430 Subject: [PATCH 45/47] Fix eslint errors --- src/app.hooks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app.hooks.js b/src/app.hooks.js index 19aa70ce..d15ff8e9 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -17,7 +17,7 @@ const authenticate = () => context => { if (context.params.provider === 'socketio' && context.path === 'authentication') { return context; } - if ( context.params.provider === 'socketio' && context.path === 'donations') { + if (context.params.provider === 'socketio' && context.path === 'donations') { // for creating and updating donations it's not needed to be authenticated, anonymous users can donate return context; } From 7d99c17a8360129ecc870dcbc6f8d05b010348ad Mon Sep 17 00:00:00 2001 From: Amin Latifi Date: Sun, 11 Jul 2021 13:26:40 +0430 Subject: [PATCH 46/47] Revert "Dont check authentication for update donations" --- src/app.hooks.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/app.hooks.js b/src/app.hooks.js index d15ff8e9..e6151e46 100644 --- a/src/app.hooks.js +++ b/src/app.hooks.js @@ -17,8 +17,12 @@ const authenticate = () => context => { if (context.params.provider === 'socketio' && context.path === 'authentication') { return context; } - if (context.params.provider === 'socketio' && context.path === 'donations') { - // for creating and updating donations it's not needed to be authenticated, anonymous users can donate + if ( + context.params.provider === 'socketio' && + context.path === 'donations' && + context.method === 'create' + ) { + // for creating donations it's not needed to be authenticated, anonymous users can donate return context; } if (context.params.provider === 'rest') { From 85a951302cdf5cd495d9783338be5f1e6246c088 Mon Sep 17 00:00:00 2001 From: mohammadranjbarz Date: Sun, 11 Jul 2021 21:57:21 +0430 Subject: [PATCH 47/47] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 380ea355..023aa0aa 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ The configuration param `blockchain.nodeUrl` is used to establish a connection. ``` ipfs daemon ``` -5. Run db migration files +5. Run db migration files ( if this the first time you want to start application, it's not needed to run migrations) ``` ./node_modules/.bin/migrate-mongo up ```