From 7fb6654f1b098f6c2f20770facd128f46fdbbdde Mon Sep 17 00:00:00 2001 From: Damien Le Berrigaud Date: Thu, 15 Feb 2024 12:45:35 -0700 Subject: [PATCH] Parse and display media from Mastodon RSS --- Damo.Io.Server/src/Article.fs | 11 +++++++++- Damo.Io.Server/src/ArticleListTemplate.fs | 3 +-- Damo.Io.Server/src/ArticleTemplate.fs | 20 ++++++++++++++----- Damo.Io.Server/src/ArticlesRepository.fs | 7 ++++--- Damo.Io.Server/src/FeedsProcessor.fs | 5 ++--- Damo.Io.Server/www/styles/app.css | 20 ++++++++++++++++--- FeedsProcessing.Tests/src/XmlTests.fs | 18 ++++++++++++++++- .../resources/samples/rss.sample.xml | 11 +++++++++- FeedsProcessing/src/Article.fs | 7 ++++++- FeedsProcessing/src/Xml.fs | 16 +++++++++++++-- 10 files changed, 96 insertions(+), 22 deletions(-) diff --git a/Damo.Io.Server/src/Article.fs b/Damo.Io.Server/src/Article.fs index 2b59985..af24886 100644 --- a/Damo.Io.Server/src/Article.fs +++ b/Damo.Io.Server/src/Article.fs @@ -1,11 +1,20 @@ module DamoIoServer.Article +open FeedsProcessing.Article open Time open DamoIoServer.Source -type Article = +type MediaRecord = { Url: string; Description: string } + +module MediaRecord = + let ofMedia (media: Media) = + { Url = media.Url + Description = media.Description } + +type ArticleRecord = { Title: string option Link: string option Content: string + Media: MediaRecord option Date: Posix option Source: Source } diff --git a/Damo.Io.Server/src/ArticleListTemplate.fs b/Damo.Io.Server/src/ArticleListTemplate.fs index 943f877..7cb37e0 100644 --- a/Damo.Io.Server/src/ArticleListTemplate.fs +++ b/Damo.Io.Server/src/ArticleListTemplate.fs @@ -41,10 +41,9 @@ let private sourceLink selectedSources source = li [] [ a attrs [ str (Source.toString source) ] ] -let render (articles: Article list) (sources: Source list) : XmlNode = +let render (articles: ArticleRecord list) (sources: Source list) : XmlNode = let sourceLinks = Source.all |> List.map (sourceLink sources) let articleList = articles |> List.map ArticleTemplate.render - let logo = h1 [] [ str "damo.io" ] let menu = ul [ _class "main-menu" ] sourceLinks diff --git a/Damo.Io.Server/src/ArticleTemplate.fs b/Damo.Io.Server/src/ArticleTemplate.fs index 35283e2..f4fdfc0 100644 --- a/Damo.Io.Server/src/ArticleTemplate.fs +++ b/Damo.Io.Server/src/ArticleTemplate.fs @@ -31,7 +31,7 @@ let private dateToString posix = open Giraffe.ViewEngine -let private articleTitle (article: Article) title = +let private articleTitle (article: ArticleRecord) title = match article.Link with | Some link -> h1 [] [ a [ _href link ] [ str title ] ] | None -> h1 [] [ str title ] @@ -39,7 +39,7 @@ let private articleTitle (article: Article) title = let private articleDate (date: Posix) = h2 [ _class "date" ] [ str (dateToString date) ] -let private articleHeader (article: Article) = +let private articleHeader (article: ArticleRecord) = header [] (List.choose @@ -47,12 +47,18 @@ let private articleHeader (article: Article) = [ Option.map (articleTitle article) article.Title Option.map articleDate article.Date ]) -let private trySourceLink (article: Article) : XmlNode option = +let private trySourceLink (article: ArticleRecord) : XmlNode option = match (article.Title, article.Link) with | None, Some url -> Some(nav [] [ a [ _href url; _target "_blank" ] [ str "Source" ] ]) | _, _ -> None -let render (article: Article) : XmlNode = +let private renderMedia (media: MediaRecord) : XmlNode = + figure + [] + [ img [ _src media.Url; _alt media.Description ] + figcaption [] [ str media.Description ] ] + +let render (article: ArticleRecord) : XmlNode = let articleHeader = articleHeader article let articleContent = section [] [ rawText article.Content ] let maybeSourceLink = trySourceLink article @@ -62,6 +68,10 @@ let render (article: Article) : XmlNode = [ yield articleHeader yield articleContent + match article.Media with + | Some media -> yield renderMedia media + | None -> () + match maybeSourceLink with | Some sourceLink -> yield sourceLink - | _ -> () ] + | None -> () ] diff --git a/Damo.Io.Server/src/ArticlesRepository.fs b/Damo.Io.Server/src/ArticlesRepository.fs index 432014d..c95e5c0 100644 --- a/Damo.Io.Server/src/ArticlesRepository.fs +++ b/Damo.Io.Server/src/ArticlesRepository.fs @@ -40,18 +40,19 @@ let private aboutContent =

""" -let private about: Article = +let private about: ArticleRecord = { Title = Some "About" Link = None Content = aboutContent + Media = None Date = None Source = About } -let mutable private allRecords: Article list = [] +let mutable private allRecords: ArticleRecord list = [] let findAll () = about :: allRecords -type FindAllBySources = Source list -> Article list +type FindAllBySources = Source list -> ArticleRecord list let findAllBySources: FindAllBySources = fun sources -> findAll () |> List.filter (fun r -> List.contains r.Source sources) diff --git a/Damo.Io.Server/src/FeedsProcessor.fs b/Damo.Io.Server/src/FeedsProcessor.fs index 6552a6a..ffbe7c6 100644 --- a/Damo.Io.Server/src/FeedsProcessor.fs +++ b/Damo.Io.Server/src/FeedsProcessor.fs @@ -1,7 +1,7 @@ module DamoIoServer.FeedsProcessor +open DamoIoServer.Article open FSharp.Control -open FeedsProcessing open FeedsProcessing.Article open FeedsProcessing.DataGateway open FeedsProcessing.Feeds @@ -9,12 +9,11 @@ open FeedsProcessing.ProcessingResult open FeedsProcessing.Xml open Microsoft.Extensions.Logging -type ArticleRecord = DamoIoServer.Article.Article - let private articleToRecord sourceType (article: Article) : ArticleRecord = { Title = Article.title article Link = Article.link article Content = Article.content article + Media = Article.media article |> Option.map MediaRecord.ofMedia Date = Article.date article Source = sourceType } diff --git a/Damo.Io.Server/www/styles/app.css b/Damo.Io.Server/www/styles/app.css index 9403994..fe0e4cc 100644 --- a/Damo.Io.Server/www/styles/app.css +++ b/Damo.Io.Server/www/styles/app.css @@ -156,7 +156,7 @@ pre { background-color: var(--code-bg-color); padding: 2.4rem; margin-bottom: 2.4rem; - border-radius: 1rem; + border-radius: .5rem; font-family: var(--monospace); font-size: 1.4rem; @@ -188,12 +188,26 @@ p { em, strong { font-weight: 700; } + +figure { + margin: 0; + + img { + border-radius: .5rem; + } + + figcaption { + font-size: 1.4rem; + text-align: center; + font-style: italic; + } +} article { background-color: var(--card-color); padding: 2.4rem 4.8rem; margin: 0 0 2.4rem 0; - border-radius: 3rem; + border-radius: 1rem; box-shadow: var(--card-shadow); } @@ -230,7 +244,7 @@ article section { article.Social { background-image: url('data:image/svg+xml;utf8,%3Csvg%20height%3D%222500%22%20width%3D%222331%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%22-0.41%200.22%20747.62%20801.4499999999999%22%3E%3Cpath%20d%3D%22M729.94%20479.5c-10.96%2056.4-98.17%20118.12-198.34%20130.08-52.23%206.23-103.66%2011.96-158.49%209.44-89.68-4.1-160.45-21.4-160.45-21.4%200%208.73.54%2017.04%201.62%2024.81%2011.66%2088.52%2087.76%2093.82%20159.84%2096.29%2072.76%202.49%20137.55-17.94%20137.55-17.94l2.99%2065.79s-50.89%2027.32-141.55%2032.35c-50%202.75-112.07-1.26-184.37-20.39C31.94%20737.02%204.97%20569.86.85%20400.26-.41%20349.9.37%20302.42.37%20262.7.37%2089.27%20113.99%2038.44%20113.99%2038.44%20171.28%2012.12%20269.59%201.06%20371.79.22h2.52c102.19.84%20200.57%2011.9%20257.86%2038.22%200%200%20113.62%2050.83%20113.62%20224.26%200%200%201.42%20127.96-15.85%20216.8%22%20fill%3D%22%23888%22%2F%3E%3Cpath%20d%3D%22M611.77%20276.16v209.99h-83.2V282.33c0-42.97-18.07-64.77-54.23-64.77-39.98%200-60.01%2025.86-60.01%2077.02v111.57h-82.71V294.58c0-51.16-20.04-77.02-60.01-77.02-36.16%200-54.24%2021.8-54.24%2064.77v203.82h-83.19V276.16c0-42.92%2010.93-77.03%2032.88-102.26%2022.63-25.23%2052.27-38.17%2089.07-38.17%2042.57%200%2074.81%2016.37%2096.12%2049.1l20.72%2034.74%2020.73-34.74c21.31-32.73%2053.55-49.1%2096.12-49.1%2036.79%200%2066.44%2012.94%2089.07%2038.17%2021.95%2025.23%2032.88%2059.34%2032.88%20102.26z%22%20fill%3D%22%23fff%22%2F%3E%3C%2Fsvg%3E'); background-repeat: no-repeat; - background-position: 1.6rem 1.6rem; + background-position: 4.8rem 2.4rem; background-size: 4.8rem 4.8rem; } diff --git a/FeedsProcessing.Tests/src/XmlTests.fs b/FeedsProcessing.Tests/src/XmlTests.fs index 59ebb6c..96dd998 100644 --- a/FeedsProcessing.Tests/src/XmlTests.fs +++ b/FeedsProcessing.Tests/src/XmlTests.fs @@ -116,8 +116,24 @@ let ``with mastodon RSS`` () = match result with | Error _ -> Assert.Fail "Expected success" - | Ok records -> List.length records |> should equal 19 + | Ok records -> + List.length records |> should equal 19 + + let mediaAt index = + records |> List.item index |> Article.media + + let expectedMediaNoDescription = + { Url = "https://mastodon.kleph.eu/system/media_attachments/files/000/055/839/original/4874168bf454bddb.jpg" + Description = "" } + + mediaAt 17 |> should equal (Some expectedMediaNoDescription) + + let expectedMediaAndDescription = + { Url = + "https://mastodon.kleph.eu/system/media_attachments/files/108/207/041/262/751/249/original/1e8a0ccac5f59165.jpg" + Description = "Mountain landscape at sunset with bicycle on the foreground." } + mediaAt 13 |> should equal (Some expectedMediaAndDescription) [] let ``processFeed with slashdot RDF XML`` () = diff --git a/FeedsProcessing/resources/samples/rss.sample.xml b/FeedsProcessing/resources/samples/rss.sample.xml index 0931b3b..bd5b86b 100644 --- a/FeedsProcessing/resources/samples/rss.sample.xml +++ b/FeedsProcessing/resources/samples/rss.sample.xml @@ -1,5 +1,7 @@ - <![CDATA[Stories by Damien Le Berrigaud on Medium]]> @@ -23,6 +25,10 @@ Tue, 20 Sep 2016 12:54:44 GMT 2016-09-20T12:54:44.561Z This is the content in encoded tag

]]> + + nonadult + Mountain landscape at sunset with bicycle on the foreground. + <![CDATA[Second title!]]> @@ -32,6 +38,9 @@ Tue, 20 Sep 2016 12:54:44 GMT 2016-09-20T12:54:44.561Z This is the content in description tag

]]> + + nonadult + diff --git a/FeedsProcessing/src/Article.fs b/FeedsProcessing/src/Article.fs index 0dd17bc..829091c 100644 --- a/FeedsProcessing/src/Article.fs +++ b/FeedsProcessing/src/Article.fs @@ -13,18 +13,23 @@ and private Fields = { Title: string option Link: string option Content: string + Media: Media option Date: Posix option } +and Media = { Url: string; Description: string } + [] module Article = let title (Article fields) = fields.Title let link (Article fields) = fields.Link let content (Article fields) = fields.Content + let media (Article fields) = fields.Media let date (Article fields) = fields.Date - let create title link content date = + let create title link content media date = Article { Title = title |> Option.bind stringToOption Link = stringToOption link Content = content |> Option.bind stringToOption |> Option.defaultValue "" |> Html.sanitize + Media = media Date = Option.map Posix.fromDateTimeOffset date } diff --git a/FeedsProcessing/src/Xml.fs b/FeedsProcessing/src/Xml.fs index e24e558..5f553b3 100644 --- a/FeedsProcessing/src/Xml.fs +++ b/FeedsProcessing/src/Xml.fs @@ -39,8 +39,19 @@ module private Rss = type private RssProvider = XmlProvider<"../FeedsProcessing/resources/samples/rss.sample.xml"> + let private descriptionToString (description: RssProvider.Description) = description.Value + + let private contentToMedia (content: RssProvider.Content) = + { Url = content.Url + Description = content.Description |> Option.map descriptionToString |> Option.defaultValue "" } + let private itemToArticle (item: RssProvider.Item) = - Article.create item.Title item.Link (item.Encoded |> Option.orElse item.Description) (Some item.PubDate) + Article.create + item.Title + item.Link + (item.Encoded |> Option.orElse item.Description) + (item.Content |> Option.map contentToMedia) + (Some item.PubDate) let private toArticles (rss: RssProvider.Rss) = Try.value "Rss to articles" (fun _ -> rss.Channel.Items |> Seq.map itemToArticle |> Seq.toList) @@ -66,6 +77,7 @@ module private Atom = (Some entry.Title.Value) (entry.Links |> Array.head |> (fun l -> l.Href)) (Some entry.Content.Value) + None (Some entry.Published) let private toArticles (atom: AtomProvider.Feed) = @@ -94,7 +106,7 @@ module private Rdf = type private RdfProvider = XmlProvider<"../FeedsProcessing/resources/samples/rdf.sample.xml"> let private itemToArticle (item: RdfProvider.Item) = - Article.create (Some item.Title) item.Link (Some item.Description) (Some item.Date) + Article.create (Some item.Title) item.Link (Some item.Description) None (Some item.Date) let private toArticles (rdf: RdfProvider.Rdf) : Result
= Try.result