From cd9536fdc8663f80a6bbfb0ae217af5216cc6ec7 Mon Sep 17 00:00:00 2001 From: Lucas Dunker Date: Mon, 27 Nov 2023 18:39:13 -0500 Subject: [PATCH 1/2] Update scraped course data with new CRN link (#209) * Change url for course and term in scrapers * Add new URL to section parser * Update snapshot tests --- scrapers/classes/parsersxe/classParser.ts | 7 +++---- scrapers/classes/parsersxe/sectionParser.ts | 4 ++-- .../tests/__snapshots__/classParser.test.js.snap | 8 ++++---- .../tests/__snapshots__/sectionParser.test.js.snap | 6 +++--- scrapers/request.ts | 2 +- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/scrapers/classes/parsersxe/classParser.ts b/scrapers/classes/parsersxe/classParser.ts index 4a982b5a..10bd01be 100644 --- a/scrapers/classes/parsersxe/classParser.ts +++ b/scrapers/classes/parsersxe/classParser.ts @@ -107,12 +107,12 @@ class ClassParser { nupath: this.nupath(attributes), desc: he.decode(description), prettyUrl: - "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_course_detail?" + + "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?" + `cat_term_in=${termId}&subj_code_in=${subjectCode}&crse_numb_in=${courseNumber}`, name: he.decode(SR.courseTitle), url: - "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_listcrse?" + - `term_in=${termId}&subj_in=${subjectCode}&crse_in=${courseNumber}&schd_in=%`, + "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?" + + `cat_term_in=${termId}&subj_code_in=${subjectCode}&crse_numb_in=${courseNumber}`, lastUpdateTime: Date.now(), maxCredits: SR.creditHourHigh || SR.creditHourLow, minCredits: SR.creditHourLow, @@ -120,7 +120,6 @@ class ClassParser { feeAmount, feeDescription, }; - if (prereqs) { classDetails.prereqs = prereqs; } diff --git a/scrapers/classes/parsersxe/sectionParser.ts b/scrapers/classes/parsersxe/sectionParser.ts index 55afa592..6c7e8ed7 100644 --- a/scrapers/classes/parsersxe/sectionParser.ts +++ b/scrapers/classes/parsersxe/sectionParser.ts @@ -63,8 +63,8 @@ class SectionParser { return a.description === "Honors"; }), url: - "https://wl11gp.neu.edu/udcprod8/bwckschd.p_disp_detail_sched" + - `?term_in=${SR.term}&crn_in=${SR.courseReferenceNumber}`, + "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?" + + `cat_term_in=${SR.term}&subj_code_in=${SR.subject}&crse_numb_in=${SR.courseNumber}`, profs: SR.faculty.map(MeetingParser.profName), meetings: MeetingParser.parseMeetings(SR.meetingsFaculty), }; diff --git a/scrapers/classes/parsersxe/tests/__snapshots__/classParser.test.js.snap b/scrapers/classes/parsersxe/tests/__snapshots__/classParser.test.js.snap index da359f52..49d0e2ff 100644 --- a/scrapers/classes/parsersxe/tests/__snapshots__/classParser.test.js.snap +++ b/scrapers/classes/parsersxe/tests/__snapshots__/classParser.test.js.snap @@ -35,10 +35,10 @@ Object { }, ], }, - "prettyUrl": "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311", + "prettyUrl": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311", "subject": "CHEM", "termId": "202010", - "url": "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_listcrse?term_in=202010&subj_in=CHEM&crse_in=2311&schd_in=%", + "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311", } `; @@ -77,9 +77,9 @@ Object { }, ], }, - "prettyUrl": "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CS&crse_numb_in=2500", + "prettyUrl": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CS&crse_numb_in=2500", "subject": "CS", "termId": "202010", - "url": "https://wl11gp.neu.edu/udcprod8/bwckctlg.p_disp_listcrse?term_in=202010&subj_in=CS&crse_in=2500&schd_in=%", + "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CS&crse_numb_in=2500", } `; diff --git a/scrapers/classes/parsersxe/tests/__snapshots__/sectionParser.test.js.snap b/scrapers/classes/parsersxe/tests/__snapshots__/sectionParser.test.js.snap index a424428a..5e26ebb7 100644 --- a/scrapers/classes/parsersxe/tests/__snapshots__/sectionParser.test.js.snap +++ b/scrapers/classes/parsersxe/tests/__snapshots__/sectionParser.test.js.snap @@ -51,7 +51,7 @@ Object { "seatsRemaining": 36, "subject": "CHEM", "termId": "202010", - "url": "https://wl11gp.neu.edu/udcprod8/bwckschd.p_disp_detail_sched?term_in=202010&crn_in=10259", + "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=202010&subj_code_in=CHEM&crse_numb_in=2311", "waitCapacity": 0, "waitRemaining": 0, } @@ -101,8 +101,8 @@ Object { "seatsRemaining": 4, "subject": "CS", "termId": "201930", - "url": "https://wl11gp.neu.edu/udcprod8/bwckschd.p_disp_detail_sched?term_in=201930&crn_in=30340", + "url": "https://bnrordsp.neu.edu/ssb-prod/bwckctlg.p_disp_course_detail?cat_term_in=201930&subj_code_in=CS&crse_numb_in=2500", "waitCapacity": 0, "waitRemaining": 0, } -`; \ No newline at end of file +`; diff --git a/scrapers/request.ts b/scrapers/request.ts index 43e4b58e..ba54a26d 100644 --- a/scrapers/request.ts +++ b/scrapers/request.ts @@ -75,7 +75,7 @@ const separateReqPools: Record = { // Took 1hr and 15 min with 500 sockets and RETRY_DELAY set to 20000 and delta set to 15000. // Usually takes just under 1 hr at 1k sockets and the same timeouts. // Took around 20 min with timeouts set to 100ms and 150ms and 100 sockets. - "wl11gp.neu.edu": { + "https://bnrordsp.neu.edu": { options: { maxSockets: 100, keepAlive: true, maxFreeSockets: 100 }, agents: false, }, From d81126a7589b0f78ee4941c40b627eff85c49874 Mon Sep 17 00:00:00 2001 From: Lucas Dunker Date: Mon, 27 Nov 2023 18:40:08 -0500 Subject: [PATCH 2/2] Add new course abbreviations (#210) --- scrapers/classes/classMapping.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapers/classes/classMapping.json b/scrapers/classes/classMapping.json index 61ca297c..7529ee75 100755 --- a/scrapers/classes/classMapping.json +++ b/scrapers/classes/classMapping.json @@ -85,7 +85,11 @@ "AI => Artificial Intelligence", "NLP => Natural Language Processing", "orgo => Organic Chemistry", - "swd => Software Development" + "swd => Software Development", + "choir => Chorus", + "syssec => Systems Security", + "netsec => Network Security", + "id => Interaction Design" ] } }