Jatin get profile images from website

package.json

@@ -58,12 +58,14 @@
     "babel-plugin-module-resolver": "^5.0.0",
     "bcryptjs": "^2.4.3",
     "body-parser": "^1.18.3",
+    "cheerio": "^1.0.0-rc.10",
     "cors": "^2.8.4",
     "cron": "^1.8.2",
     "dotenv": "^5.0.1",
     "express": "^4.17.1",
     "express-validator": "^7.0.1",
     "googleapis": "^100.0.0",
+    "jsdom": "^25.0.0",
     "jsonwebtoken": "^9.0.0",
     "lodash": "^4.17.21",
     "moment": "^2.29.4",
@@ -74,6 +76,7 @@
     "node-cache": "^5.1.2",
     "node-datetime": "^2.0.3",
     "nodemailer": "^6.4.16",
+    "puppeteer": "^10.4.0",
     "redis": "^4.2.0",
     "sanitize-html": "^2.13.0",
     "supertest": "^6.3.4",

src/controllers/userProfileController.js

@@ -847,13 +847,11 @@ const userProfileController = function (UserProfile, Project) {
 
   const getUserById = function (req, res) {
     const userid = req.params.userId;
-
     // if (cache.getCache(`user-${userid}`)) {
     //   const getData = JSON.parse(cache.getCache(`user-${userid}`));
     //   res.status(200).send(getData);
     //   return;
     // }
-
     UserProfile.findById(userid, '-password -refreshTokens -lastModifiedDate -__v')
       .populate([
         {
@@ -1763,6 +1761,33 @@ const userProfileController = function (UserProfile, Project) {
     }
 
   };
+
+  const removeProfileImage = async (req,res) =>{
+    try{
+      var user_id=req.body.user_id
+      await UserProfile.updateOne({_id:user_id},{$unset:{profilePic:""}})
+      cache.removeCache(`user-${user_id}`);
+      return res.status(200).send({message:'Image Removed'})
+    }catch(err){
+      console.log(err)
+      return res.status(404).send({message:"Error Removing Image"})
+    }
+  }
+  const updateProfileImageFromWebsite = async (req,res) =>{
+    try{
+      var user=req.body
+      const result=await UserProfile.updateOne({_id:user.user_id},
+        {
+          $set: { profilePic : user.selectedImage.imgs[0].nitroLazySrc},
+          $unset: { suggestedProfilePics: "" }
+      })
+      cache.removeCache(`user-${user.user_id}`);
+      return res.status(200).send({message:"Profile Updated"})
+    }catch(err){
+      console.log(err)
+      return res.status(404).send({message:"Profile Update Failed"})
+    }
+  }
 
   return {
     postUserProfile,
@@ -1792,6 +1817,8 @@ const userProfileController = function (UserProfile, Project) {
     getProjectsByPerson,
     getAllTeamCode,
     getAllTeamCodeHelper,
+    removeProfileImage,
+    updateProfileImageFromWebsite
   };
 };
 

src/cronjobs/userProfileJobs.js

@@ -11,6 +11,7 @@ const userProfileJobs = () => {
     async () => {
       const SUNDAY = 0; // will change back to 0 after fix
       if (moment().tz('America/Los_Angeles').day() === SUNDAY) {
+        await userhelper.getProfileImagesFromWebsite();
         await userhelper.assignBlueSquareForTimeNotMet();
         await userhelper.applyMissedHourForCoreTeam();
         await userhelper.emailWeeklySummariesForAllUsers();
@@ -25,7 +26,7 @@ const userProfileJobs = () => {
     false,
     'America/Los_Angeles',
   );
-
+  
   allUserProfileJobs.start();
 };
 module.exports = userProfileJobs;

src/helpers/userHelper.js

@@ -28,6 +28,9 @@ const timeOffRequest = require('../models/timeOffRequest');
 const notificationService = require('../services/notificationService');
 const { NEW_USER_BLUE_SQUARE_NOTIFICATION_MESSAGE } = require('../constants/message');
 const timeUtils = require('../utilities/timeUtils');
+const puppeteer = require('puppeteer');
+const fs = require('fs');
+const cheerio = require('cheerio');
 
 const userHelper = function () {
   // Update format to "MMM-DD-YY" from "YYYY-MMM-DD" (Confirmed with Jae)
@@ -2221,6 +2224,185 @@ const userHelper = function () {
     }
   };
 
+  function extractDataFromHtml(html) {
+    const $ = cheerio.load(html);
+    const results = [];
+
+    // Find all <p> elements
+    $('p').each((i, p) => {
+        const pData = { imgs: [], strongTexts: [] };
+
+        // Extract <img> attributes
+        $(p).find('img').each((i, img) => {
+            const src = $(img).attr('src') || '';
+            const alt = $(img).attr('alt') || '';
+            const title = $(img).attr('title') || '';
+            const nitroLazySrc = $(img).attr('nitro-lazy-src') || '';
+            pData.imgs.push({ src, alt, title, nitroLazySrc });
+        });
+
+        // Extract text from <strong> tags
+        $(p).find('strong').each((i, strong) => {
+            const text = $(strong).text().trim();  // Extract text, not HTML
+            if (text) {  // Check if text is not empty
+                pData.strongTexts.push(text);
+            }
+        });
+
+        if (pData.imgs.length || pData.strongTexts.length) {
+            results.push(pData);
+        }
+    });
+
+    return results;
+  }
+
+  function searchForTermsInFields(data, term1, term2) {
+    const lowerCaseTerm1 = term1.toLowerCase();
+    const lowerCaseTerm2 = term2.toLowerCase();
+
+    let bothTermsMatches = [];
+    let term2Matches = [];
+
+    // Check if the current data is an array
+    if (Array.isArray(data)) {
+        data.forEach(item => {
+            const bothTermsFound = searchForBothTerms(item, lowerCaseTerm1, lowerCaseTerm2);
+            const term2OnlyFound = searchForTerm2(item, lowerCaseTerm2);
+
+            if (bothTermsFound) {
+                bothTermsMatches.push(item); // If both terms are found, store the item
+            } else if (term2OnlyFound) {
+                term2Matches.push(item); // If only term2 is found, store the item
+            }
+        });
+
+        // If matches for both terms are found, return them, else return term2 matches
+        if (bothTermsMatches.length > 0) {
+            return bothTermsMatches;
+        } else if (term2Matches.length > 0) {
+            return term2Matches;
+        } else {
+            return [];  // No match found, return empty array
+        }
+    }
+
+    // Recursion case for nested objects
+    if (typeof data === 'object' && data !== null) {
+        const result = Object.keys(data).some(key => {
+            if (typeof data[key] === 'object') {
+                return searchForTermsInFields(data[key], lowerCaseTerm1, lowerCaseTerm2);
+            }
+        });
+        return result ? data : null;
+    }
+
+    return [];
+}
+
+// Helper function to check if both terms are in the string
+function searchForBothTerms(data, term1, term2) {
+    if (typeof data === 'object' && data !== null) {
+        const fieldsToCheck = ['strongTexts', 'alt', 'title', 'nitroLazySrc'];
+        return Object.keys(data).some(key => {
+            if (fieldsToCheck.includes(key)) {
+                const stringValue = String(data[key]).toLowerCase();
+                return stringValue.includes(term1) && stringValue.includes(term2); // Check if both terms are in the string
+            }
+            return false;
+        });
+    }
+    return false;
+}
+
+// Helper function to check if only term2 is in the string
+function searchForTerm2(data, term2) {
+    if (typeof data === 'object' && data !== null) {
+        const fieldsToCheck = ['strongTexts', 'alt', 'title', 'nitroLazySrc'];
+        return Object.keys(data).some(key => {
+            if (fieldsToCheck.includes(key)) {
+                const stringValue = String(data[key]).toLowerCase();
+                return stringValue.includes(term2); // Check if only term2 is in the string
+            }
+            return false;
+        });
+    }
+    return false;
+}
+
+
+  const getProfileImagesFromWebsite= async () => {
+    try {
+        // Launch a new browser instance in headless mode
+        const browser = await puppeteer.launch({ headless: true });
+        const page = await browser.newPage();
+        // Navigate to the website
+        await page.goto('https://www.onecommunityglobal.org/team', { waitUntil: 'networkidle2' });
+        // XPath for the specific element you want to target
+        const specificElementXPath = '//*[@id="page-119101"]';
+        // Wait for the specific element to be present on the page
+        await page.waitForXPath(specificElementXPath);
+        // Get the specific element using XPath
+        const [elementHandle] = await page.$x(specificElementXPath);
+
+        if (!elementHandle) {
+            console.error('Element not found');
+            await browser.close();
+            return;
+        }
+        // Extract HTML of the specific element
+        const html = await page.evaluate(el => el.innerHTML, elementHandle);
+        // Extract data from the HTML string
+        var data = extractDataFromHtml(html);
+        var newData = data.map(item => {
+            return {
+              ...item,
+              strongTexts: (() => {
+                const filteredTexts = Array.from(new Set(
+                  item.strongTexts
+                    .map(text => text.replace(/[-‐‑—–:]/g, ''))
+                    .filter(text => text.split(' ').length <= 2)
+                    .filter(text => text !== '')
+                ));
+                return filteredTexts.length === 1 ? [filteredTexts[0]] : filteredTexts.length === 2 ? [filteredTexts.join(' ')] : filteredTexts;
+              })()
+            };
+          });
+
+          newData.map((e)=>{
+                if(e.strongTexts.length==0){
+                    if(e.imgs.length!==0){
+                        if(e.imgs[0].title!==""){
+                            e.strongTexts.push(e.imgs[0].title.split(' ').slice(0, 2).join(' '));
+                        }else if(e.imgs.alt!==""){
+                            e.strongTexts.push(e.imgs[0].alt.split(' ').slice(0, 2).join(' '));
+                        }
+                    }
+                }
+            })
+        newData=newData.filter(item=>item.imgs.length!==0 && item.strongTexts.length!==0)
+        await browser.close();
+        var users=await userProfile.find({'isActive':true},"firstName lastName email profilePic suggestedProfilePics")    
+        users.map(async(u)=>{
+          if(u.profilePic==undefined || u.profilePic==null || u.profilePic==""){
+            let result=searchForTermsInFields(newData,u.firstName,u.lastName)
+              try {
+                if(result.length==1){
+                  await userProfile.updateOne({_id:u._id},{$set:{"profilePic":result[0].imgs[0].nitroLazySrc}});
+                }else if(result.length>1){
+                  await userProfile.updateOne({_id:u._id},{$set:{"suggestedProfilePics":result}});
+                }
+              } catch (error) {
+                console.log(error);
+              }
+
+          }
+        })  
+    } catch (error) {
+        console.error('An error occurred:', error);
+    }
+  }
+
   return {
     changeBadgeCount,
     getUserName,
@@ -2241,6 +2423,7 @@ const userHelper = function () {
     getTangibleHoursReportedThisWeekByUserId,
     deleteExpiredTokens,
     deleteOldTimeOffRequests,
+    getProfileImagesFromWebsite
   };
 };
 

src/models/userProfile.js

@@ -101,6 +101,10 @@ const userProfileSchema = new Schema({
     },
   ],
   profilePic: { type: String },
+  suggestedProfilePics:{
+    type:[mongoose.Schema.Types.Mixed],
+    default:[]
+  },
   infringements: [
     {
       date: { type: String, required: true },

src/routes/userProfileRouter.js

@@ -112,7 +112,10 @@ const routes = function (userProfile, project) {
   userProfileRouter.route('/userProfile/projects/:name').get(controller.getProjectsByPerson);
 
   userProfileRouter.route('/userProfile/teamCode/list').get(controller.getAllTeamCode);
-
+
+  userProfileRouter.route('/userProfile/profileImage/remove').put(controller.removeProfileImage);
+  userProfileRouter.route('/userProfile/profileImage/imagefromwebsite').put(controller.updateProfileImageFromWebsite);
+
   return userProfileRouter;
 };
 

src/server.js

@@ -1,9 +1,7 @@
 /* eslint-disable quotes */
 require('dotenv').load();
-
 const { app, logger } = require('./app');
 const websockets = require('./websockets').default;
-
 require('./startup/db')();
 require('./cronjobs/userProfileJobs')();