Skip to content

Commit

Permalink
refactor: add sources to filtered url
Browse files Browse the repository at this point in the history 
No expected changes for end users
  • Loading branch information
@brainwo
brainwo committed May 16, 2024
1 parent 4d427a9 commit c4a2706
Show file tree
Hide file tree
Showing 7 changed files with 1,420 additions and 64 deletions.
43 changes: 30 additions & 13 deletions filter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,35 @@ const blockedUrls = [
"http://cleancoder.com",
];

/**
* @typedef {Object} FilteredUrl
* @property {string} source
* @property {string} url
*/

/**
* Return urls of article and what video it's from
* @param {string} feed
* @return {FilteredUrl[]}
*/
export function getFilteredUrlFromFeed(feed) {
return extractUrls(
JSON.parse(toJson(feed), { sanitize: false })
["feed"]["entry"].map(
(entry) => entry["media:group"]["media:description"],
)
.toString(),
).filter(
(url) =>
!blockedUrls.some((e) => url.startsWith(e)) ||
url.startsWith("https://www.youtube.com/watch") ||
(url.includes("status") &&
(url.includes("twitter.com") || url.includes("x.com"))),
);
return JSON.parse(toJson(feed), { sanitize: false })
["feed"]["entry"].map((entry) => ({
video: entry["media:group"]["media:content"]["url"],
urls: extractUrls(`${entry["media:group"]["media:description"]}`),
}))
.flat()
.reduce((acc, curr) => {
if (typeof curr["urls"] === "undefined") return acc;
curr["urls"]
.filter(
(url) =>
!blockedUrls.some((e) => url.startsWith(e)) ||
url.startsWith("https://www.youtube.com/watch") ||
(url.includes("status") &&
(url.includes("twitter.com") || url.includes("x.com")))
)
.forEach((url) => acc.push({ source: curr.video, url: url }));
return acc;
}, new Array());
}
218 changes: 177 additions & 41 deletions filter.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,53 +2,189 @@
// Use of this source code is governed by an MIT-style license that can be
// found in the LICENSE file.

import { feed as theoFeed } from "./test_examples/theo";
import { feed as primeFeed } from "./test_examples/prime";
import fs from "node:fs";

import { getFilteredUrlFromFeed } from "./filter.js";

const theoFeed = fs.readFileSync("test_examples/theo.xml", "utf8").toString();
const primeFeed = fs.readFileSync("test_examples/prime.xml", "utf8").toString();

test("Filter Theo feed", () =>
expect(getFilteredUrlFromFeed(theoFeed)).toStrictEqual([
"https://planetscale.com/blog/the-problem-with-using-a-uuid-primary-key-in-mysql",
"https://www.youtube.com/watch?v=aWfYxg-Ypm4",
"https://twitter.com/sebastienlorber/status/1783114423894233235",
"https://github.com/facebook/react/pull/28896",
"https://nextjs.org/docs/app/api-reference/functions/unstable_cache",
"https://twitter.com/edgedatabase/status/1782485844814647402",
"https://www.edgedb.com/blog/we-can-do-better-than-sql#null-a-bag-of-surprises",
"https://twitter.com/ajassy/status/1785293612835823716",
"https://twitter.com/leeerob/status/1785498474462560309",
"https://github.blog/2024-04-29-github-copilot-workspace/",
"https://supermaven.com/",
"https://ielm.io/blog/turbo",
"https://adrianroselli.com/2024/03/the-ultimate-ideal-bestest-base-font-size-that-everyone-is-keeping-a-secret-especially-chet.html",
"https://nodejs.org/en/blog/release/v22.0.0",
"https://openjsf.org/blog/nodejs-22-available",
"https://nodejs.org/en/blog/announcements/v22-release-announce",
"https://github.com/apple/corenet",
"https://huggingface.co/collections/apple/openelm-pretrained-models-6619ac6ca12a10bd0d0df89e",
"https://www.ftc.gov/news-events/news/press-releases/2024/04/ftc-announces-rule-banning-noncompetes",
"https://www.hashicorp.com/blog/hashicorp-joins-ibm",
"https://opentofu.org/blog/our-response-to-hashicorps-cease-and-desist/",
"https://newsroom.ibm.com/2024-04-24-IBM-to-Acquire-HashiCorp-Inc-Creating-a-Comprehensive-End-to-End-Hybrid-Cloud-Platform",
"https://react.dev/blog/2024/04/25/react-19",
"https://lisyarus.github.io/blog/programming/2023/02/21/exponential-smoothing.html",
"https://mrbruh.com/chattr/",
"https://env.fail/posts/firewreck-1/",
"https://kibty.town/blog/chattr/",
{
source: "https://www.youtube.com/v/DhfeXfF_W4w?version=3",
url: "https://playground.react.dev",
},
{
source: "https://www.youtube.com/v/DhfeXfF_W4w?version=3",
url: "https://react.dev/learn/react-compiler",
},
{
source: "https://www.youtube.com/v/DhfeXfF_W4w?version=3",
url: "https://www.youtube.com/watch?v=PYHBHK37xlE",
},
{
source: "https://www.youtube.com/v/DhfeXfF_W4w?version=3",
url: "https://github.com/facebook/react/pull/29061",
},
{
source: "https://www.youtube.com/v/FFxhkWk3I7U?version=3",
url: "https://developer.mozilla.org/en-US/docs/Web/API/Popover_API",
},
{
source: "https://www.youtube.com/v/FFxhkWk3I7U?version=3",
url: "https://twitter.com/chancethedev/status/1789037040161747361",
},
{
source: "https://www.youtube.com/v/FFxhkWk3I7U?version=3",
url: "https://web.dev/blog/popover-api",
},
{
source: "https://www.youtube.com/v/FFxhkWk3I7U?version=3",
url: "https://codepen.io/web-dot-dev/pen/RwOvjrv",
},
{
source: "https://www.youtube.com/v/FFxhkWk3I7U?version=3",
url: "https://developer.chrome.com/blog/anchor-positioning-api",
},
{
source: "https://www.youtube.com/v/aOk67eT3fpg?version=3",
url: "https://phoenixframework.org/blog/phoenix-liveview-1.0-released",
},
{
source: "https://www.youtube.com/v/ZV7S337wJ18?version=3",
url: "https://uploadthing.com",
},
{
source: "https://www.youtube.com/v/ZV7S337wJ18?version=3",
url: "https://medium.com/@maciej.pocwierz/how-an-empty-s3-bucket-can-make-your-aws-bill-explode-934a383cb8b1",
},
{
source: "https://www.youtube.com/v/ZV7S337wJ18?version=3",
url: "https://twitter.com/jeffbarr/status/1787844682216792163",
},
{
source: "https://www.youtube.com/v/KuPu4OTIaT8?version=3",
url: "https://twitter.com/StackOverflow/status/1787467736097939562",
},
{
source: "https://www.youtube.com/v/KuPu4OTIaT8?version=3",
url: "https://twitter.com/t3dotgg/status/1787596365008380306",
},
{
source: "https://www.youtube.com/v/KuPu4OTIaT8?version=3",
url: "https://observablehq.com/@ayhanfuat/the-fall-of-stack-overflow",
},
{
source: "https://www.youtube.com/v/1gZmkpsVGkk?version=3",
url: "https://danielnagy.me/posts/Post_tsr8q6sx37pl",
},
{
source: "https://www.youtube.com/v/PAf6NjyCnrI?version=3",
url: "https://twitter.com/acdlite/status/1785691330988986587",
},
{
source: "https://www.youtube.com/v/a-K2C3sf1_Q?version=3",
url: "https://planetscale.com/blog/the-problem-with-using-a-uuid-primary-key-in-mysql",
},
{
source: "https://www.youtube.com/v/AKNH7mXciEM?version=3",
url: "https://twitter.com/sebastienlorber/status/1783114423894233235",
},
{
source: "https://www.youtube.com/v/AKNH7mXciEM?version=3",
url: "https://github.com/facebook/react/pull/28896",
},
{
source: "https://www.youtube.com/v/AKNH7mXciEM?version=3",
url: "https://nextjs.org/docs/app/api-reference/functions/unstable_cache",
},
{
source: "https://www.youtube.com/v/b23x_IWIb4c?version=3",
url: "https://twitter.com/edgedatabase/status/1782485844814647402",
},
{
source: "https://www.youtube.com/v/b23x_IWIb4c?version=3",
url: "https://www.edgedb.com/blog/we-can-do-better-than-sql#null-a-bag-of-surprises",
},
{
source: "https://www.youtube.com/v/Toar450Gk5Y?version=3",
url: "https://twitter.com/ajassy/status/1785293612835823716",
},
{
source: "https://www.youtube.com/v/Toar450Gk5Y?version=3",
url: "https://twitter.com/leeerob/status/1785498474462560309",
},
{
source: "https://www.youtube.com/v/75Hv0RUFIrQ?version=3",
url: "https://github.blog/2024-04-29-github-copilot-workspace/",
},
{
source: "https://www.youtube.com/v/75Hv0RUFIrQ?version=3",
url: "https://supermaven.com/",
},
{
source: "https://www.youtube.com/v/MS1W7UqrX3I?version=3",
url: "https://ielm.io/blog/turbo",
},
{
source: "https://www.youtube.com/v/rg3zgQ3xBRc?version=3",
url: "https://adrianroselli.com/2024/03/the-ultimate-ideal-bestest-base-font-size-that-everyone-is-keeping-a-secret-especially-chet.html",
},
]));

test("Filter ThePrimeTime feed", () =>
expect(getFilteredUrlFromFeed(primeFeed)).toStrictEqual([
"https://www.youtube.com/watch?v=pnnx1bkFXng",
"https://medium.com/@maciej.pocwierz/how-an-empty-s3-bucket-can-make-your-aws-bill-explode-934a383cb8b1",
"https://breckyunits.com/ckmeans.html",
"https://twitter.com/hyprturing/status/1784426156659453977",
"https://www.theverge.com/2024/4/30/24145838/rabbit-r1-android-app-pixel-6a",
"https://env.fail/posts/firewreck-1/",
"https://www.youtube.com/watch?v=aWfYxg-Ypm4",
"https://www.youtube.com/watch?v=pkotufZchjE&t=11s",
"https://faultlore.com/blah/c-isnt-a-language/",
"https://twitter.com/iavins/status/1774464622067679738",
"https://www.youtube.com/watch?v=bP5uH7TWiUs",
"https://www.freecodecamp.org/news/solid-principles-explained-in-plain-english/",
{
source: "https://www.youtube.com/v/cXw4mZZXJNU?version=3",
url: "https://openai.com/index/hello-gpt-4o/",
},
{
source: "https://www.youtube.com/v/T7WBIPSZ87g?version=3",
url: "https://www.youtube.com/watch?v=0uQ3bkiW5SE",
},
{
source: "https://www.youtube.com/v/fPYSh680ZBY?version=3",
url: "https://www.youtube.com/watch?v=3obM5OaOpHQ",
},
{
source: "https://www.youtube.com/v/ik-jAjYFVz0?version=3",
url: "https://www.sophiajt.com/search-for-easier-safe-systems-programming/",
},
{
source: "https://www.youtube.com/v/tvrZ5XraB7s?version=3",
url: "https://sqlite.org/draft/whybytecode.html",
},
{
source: "https://www.youtube.com/v/DucriSA8ukw?version=3",
url: "https://www.youtube.com/watch?v=pnnx1bkFXng",
},
{
source: "https://www.youtube.com/v/OWggTcVgiNg?version=3",
url: "https://medium.com/@maciej.pocwierz/how-an-empty-s3-bucket-can-make-your-aws-bill-explode-934a383cb8b1",
},
{
source: "https://www.youtube.com/v/KRD5jiYUOhk?version=3",
url: "https://breckyunits.com/ckmeans.html",
},
{
source: "https://www.youtube.com/v/Qatdg5bJCGE?version=3",
url: "https://twitter.com/hyprturing/status/1784426156659453977",
},
{
source: "https://www.youtube.com/v/ModLKm6nomk?version=3",
url: "https://www.theverge.com/2024/4/30/24145838/rabbit-r1-android-app-pixel-6a",
},
{
source: "https://www.youtube.com/v/P2gJdbb3Pec?version=3",
url: "https://env.fail/posts/firewreck-1/",
},
{
source: "https://www.youtube.com/v/P2gJdbb3Pec?version=3",
url: "https://env.fail/about",
},
{
source: "https://www.youtube.com/v/6SNQlSRAqAY?version=3",
url: "https://www.youtube.com/watch?v=aWfYxg-Ypm4",
},
]));
14 changes: 8 additions & 6 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,15 @@ Promise.all(getNewsUrl(feedList))
.then((source) => source.flat())
.then((flattenData) =>
Promise.all(
[...new Set(flattenData)].map(async (link) => {
try {
return await extract(link);
} catch (err) {
return null;
[...flattenData.reduce((acc, curr) => acc.add(curr), new Set())].map(
async (link) => {
try {
return await extract(link["url"]);
} catch (err) {
return null;
}
}
})
)
)
.then((rssItem) => rssItem.filter((e) => e !== null))
.then((rssItem) => writeToFile(rssItem))
Expand Down
2 changes: 0 additions & 2 deletions test_examples/prime.js
View file

This file was deleted.

Loading

0 comments on commit c4a2706

Please sign in to comment.