diff --git a/Damo.Io.Blog/src/BlogGenerator/Html.fs b/Damo.Io.Blog/src/BlogGenerator/Html.fs index 0ec9007..7fdf5f1 100644 --- a/Damo.Io.Blog/src/BlogGenerator/Html.fs +++ b/Damo.Io.Blog/src/BlogGenerator/Html.fs @@ -1,6 +1,7 @@ module BlogGenerator.Html open System + open BlogGenerator.Posts open BlogGenerator.Config diff --git a/Damo.Io.Blog/src/BlogGenerator/Posts.fs b/Damo.Io.Blog/src/BlogGenerator/Posts.fs index fa39e29..42b7e7b 100644 --- a/Damo.Io.Blog/src/BlogGenerator/Posts.fs +++ b/Damo.Io.Blog/src/BlogGenerator/Posts.fs @@ -2,9 +2,10 @@ open System open System.IO -open Metadata -open Markdown -open Config + +open BlogGenerator.Metadata +open BlogGenerator.Markdown +open BlogGenerator.Config type Post = { Title: string diff --git a/Damo.Io.Blog/src/BlogGenerator/Rss.fs b/Damo.Io.Blog/src/BlogGenerator/Rss.fs index a85b025..f9b8d97 100644 --- a/Damo.Io.Blog/src/BlogGenerator/Rss.fs +++ b/Damo.Io.Blog/src/BlogGenerator/Rss.fs @@ -2,6 +2,7 @@ open System open System.Xml.Linq + open BlogGenerator.Posts open BlogGenerator.Config diff --git a/Damo.Io.Blog/src/Program.fs b/Damo.Io.Blog/src/Program.fs index d3088df..f9ec4cd 100644 --- a/Damo.Io.Blog/src/Program.fs +++ b/Damo.Io.Blog/src/Program.fs @@ -1,6 +1,6 @@ open BlogGenerator.Build [] -let main argv = +let main _ = Build.run () 0 diff --git a/Damo.Io.Server/src/Article.fs b/Damo.Io.Server/src/Article.fs index e4ecf47..93f2a46 100644 --- a/Damo.Io.Server/src/Article.fs +++ b/Damo.Io.Server/src/Article.fs @@ -1,7 +1,8 @@ module DamoIoServer.Article -open FeedsProcessing.Article open Time + +open FeedsProcessing.Article open DamoIoServer.Source type MediaRecord = { Url: string; Description: string } diff --git a/Damo.Io.Server/src/ArticleListTemplate.fs b/Damo.Io.Server/src/ArticleListTemplate.fs index 7cb37e0..c775219 100644 --- a/Damo.Io.Server/src/ArticleListTemplate.fs +++ b/Damo.Io.Server/src/ArticleListTemplate.fs @@ -1,8 +1,8 @@ -[] module DamoIoServer.ArticleListTemplate open DamoIoServer.Source open DamoIoServer.Article +open DamoIoServer.ArticleTemplate let private sourceToggleHref selectedSources source = Source.all @@ -41,10 +41,12 @@ let private sourceLink selectedSources source = li [] [ a attrs [ str (Source.toString source) ] ] -let render (articles: ArticleRecord list) (sources: Source list) : XmlNode = - let sourceLinks = Source.all |> List.map (sourceLink sources) - let articleList = articles |> List.map ArticleTemplate.render - let logo = h1 [] [ str "damo.io" ] - let menu = ul [ _class "main-menu" ] sourceLinks - - div [ _id "template" ] [ aside [] [ logo; menu ]; main [] articleList ] +[] +module ArticleListTemplate = + let render (articles: ArticleRecord list) (sources: Source list) : XmlNode = + let sourceLinks = Source.all |> List.map (sourceLink sources) + let articleList = articles |> List.map ArticleTemplate.render + let logo = h1 [] [ str "damo.io" ] + let menu = ul [ _class "main-menu" ] sourceLinks + + div [ _id "template" ] [ aside [] [ logo; menu ]; main [] articleList ] diff --git a/Damo.Io.Server/src/ArticleTemplate.fs b/Damo.Io.Server/src/ArticleTemplate.fs index 4eeb395..be1cec1 100644 --- a/Damo.Io.Server/src/ArticleTemplate.fs +++ b/Damo.Io.Server/src/ArticleTemplate.fs @@ -1,7 +1,7 @@ -[] module DamoIoServer.ArticleTemplate open Time + open DamoIoServer.Article let private monthDisplayName month = @@ -58,24 +58,26 @@ let private renderMedia (media: MediaRecord) : XmlNode = [ img [ _src media.Url; _alt media.Description ] figcaption [] [ str media.Description ] ] -let render (article: ArticleRecord) : XmlNode = - let articleHeader = articleHeader article - let articleContent = section [] [ rawText article.Content ] - let maybeSourceLink = trySourceLink article +[] +module ArticleTemplate = + let render (article: ArticleRecord) : XmlNode = + let articleHeader = articleHeader article + let articleContent = section [] [ rawText article.Content ] + let maybeSourceLink = trySourceLink article - let sourceNameClass = article.SourceName.Replace(" ", "") - let sourceTypeClass = article.SourceType.ToString() - let cssClasses = $"{sourceNameClass} {sourceTypeClass}" + let sourceNameClass = article.SourceName.Replace(" ", "") + let sourceTypeClass = article.SourceType.ToString() + let cssClasses = $"{sourceNameClass} {sourceTypeClass}" - HtmlElements.article - [ _class cssClasses ] - [ yield articleHeader - yield articleContent + HtmlElements.article + [ _class cssClasses ] + [ yield articleHeader + yield articleContent - match article.Media with - | Some media -> yield renderMedia media - | None -> () + match article.Media with + | Some media -> yield renderMedia media + | None -> () - match maybeSourceLink with - | Some sourceLink -> yield sourceLink - | None -> () ] + match maybeSourceLink with + | Some sourceLink -> yield sourceLink + | None -> () ] diff --git a/Damo.Io.Server/src/ArticlesHandler.fs b/Damo.Io.Server/src/ArticlesHandler.fs index 77ec50c..da704a4 100644 --- a/Damo.Io.Server/src/ArticlesHandler.fs +++ b/Damo.Io.Server/src/ArticlesHandler.fs @@ -2,9 +2,12 @@ module DamoIoServer.ArticlesHandler open System open Time -open DamoIoServer.Article + open DamoIoServer.Source +open DamoIoServer.LayoutTemplate +open DamoIoServer.Article open DamoIoServer.ArticlesRepository +open DamoIoServer.ArticleListTemplate let private sourcesFromPath (path: string) = path.Split(",") |> Array.toList |> List.choose Source.tryFromString diff --git a/Damo.Io.Server/src/AssetHashBuilder.fs b/Damo.Io.Server/src/AssetHashBuilder.fs index c137353..c0050ca 100644 --- a/Damo.Io.Server/src/AssetHashBuilder.fs +++ b/Damo.Io.Server/src/AssetHashBuilder.fs @@ -2,7 +2,6 @@ module DamoIoServer.AssetHashBuilder open System open System.Security.Cryptography - open System.Collections.Concurrent open Microsoft.AspNetCore.Http open WebOptimizer diff --git a/Damo.Io.Server/src/BackgroundProcessor.fs b/Damo.Io.Server/src/BackgroundProcessor.fs index ce061a4..69b1b93 100644 --- a/Damo.Io.Server/src/BackgroundProcessor.fs +++ b/Damo.Io.Server/src/BackgroundProcessor.fs @@ -5,6 +5,7 @@ open Microsoft.Extensions.Logging open DamoIoServer.SourcesRepository open DamoIoServer.ArticlesRepository +open DamoIoServer.FeedsProcessor [] type BackgroundProcessor(logger: ILogger, updateArticles: ArticlesRepository.UpdateArticles) = diff --git a/Damo.Io.Server/src/FeedsProcessor.fs b/Damo.Io.Server/src/FeedsProcessor.fs index 1adacdb..c14a76e 100644 --- a/Damo.Io.Server/src/FeedsProcessor.fs +++ b/Damo.Io.Server/src/FeedsProcessor.fs @@ -1,14 +1,15 @@ module DamoIoServer.FeedsProcessor -open DamoIoServer.Article -open DamoIoServer.SourcesRepository open FSharp.Control +open Microsoft.Extensions.Logging + open FeedsProcessing.Article open FeedsProcessing.DataGateway open FeedsProcessing.Feeds open FeedsProcessing.ProcessingResult open FeedsProcessing.Xml -open Microsoft.Extensions.Logging +open DamoIoServer.Article +open DamoIoServer.SourcesRepository let private articleToRecord (sourceFeed: SourceFeed) (article: Article) : ArticleRecord = { Title = Article.title article @@ -26,11 +27,11 @@ let private downloadAndProcessFeed (logger: ILogger) (sourceFeed: SourceFeed) : match sourceFeed.Feed with | Xml(url) -> async { - let! download = downloadContent url + let! download = DataGateway.download url return download - |> Result.bind processFeed + |> Result.bind Xml.processFeed |> Result.onOk (fun articles -> let count = List.length articles logger.LogInformation($"Parsed feed %A{url}, found %d{count} article(s)") @@ -43,12 +44,14 @@ let private downloadAndProcessFeed (logger: ILogger) (sourceFeed: SourceFeed) : ) } -let processFeeds (logger: ILogger) (sources: SourcesRepository.SourceFeed list) : AsyncSeq = - asyncSeq { - for sourceFeed in sources do - let! processingResult = downloadAndProcessFeed logger sourceFeed - let articles = processingResult |> resultToList sourceFeed +[] +module FeedsProcessor = + let processFeeds (logger: ILogger) (sources: SourcesRepository.SourceFeed list) : AsyncSeq = + asyncSeq { + for sourceFeed in sources do + let! processingResult = downloadAndProcessFeed logger sourceFeed + let articles = processingResult |> resultToList sourceFeed - for a in articles do - yield a - } + for a in articles do + yield a + } diff --git a/Damo.Io.Server/src/LayoutTemplate.fs b/Damo.Io.Server/src/LayoutTemplate.fs index e471983..79cac25 100644 --- a/Damo.Io.Server/src/LayoutTemplate.fs +++ b/Damo.Io.Server/src/LayoutTemplate.fs @@ -1,22 +1,24 @@ -[] module DamoIoServer.LayoutTemplate -open DamoIoServer.AssetHashBuilder open Giraffe open Giraffe.ViewEngine open Microsoft.AspNetCore.Http -let render (ctx: HttpContext) innerTemplate = - let hashBuilder = ctx.GetService() - let _assetHref = hashBuilder.Path ctx >> _href +open DamoIoServer.AssetHashBuilder + +[] +module LayoutTemplate = + let render (ctx: HttpContext) innerTemplate = + let hashBuilder = ctx.GetService() + let _assetHref = hashBuilder.Path ctx >> _href - html - [ _lang "en" ] - [ head - [] - [ meta [ _charset "utf-8" ] - meta [ _name "viewport"; _content "width=device-width" ] - title [] [ str "damo.io - Damien Le Berrigaud's feeds" ] - link [ _rel "stylesheet"; _type "text/css"; _assetHref "/styles/app.min.css" ] - link [ _rel "icon"; _type "image/svg+xml"; _sizes "any"; _href "/favicon.svg" ] ] - body [] [ innerTemplate; script [ _src "/javascript/htmx-1.8.5.min.js" ] [] ] ] + html + [ _lang "en" ] + [ head + [] + [ meta [ _charset "utf-8" ] + meta [ _name "viewport"; _content "width=device-width" ] + title [] [ str "damo.io - Damien Le Berrigaud's feeds" ] + link [ _rel "stylesheet"; _type "text/css"; _assetHref "/styles/app.min.css" ] + link [ _rel "icon"; _type "image/svg+xml"; _sizes "any"; _href "/favicon.svg" ] ] + body [] [ innerTemplate; script [ _src "/javascript/htmx-1.8.5.min.js" ] [] ] ] diff --git a/Damo.Io.Server/src/SourcesRepository.fs b/Damo.Io.Server/src/SourcesRepository.fs index 3b08797..34ec3d6 100644 --- a/Damo.Io.Server/src/SourcesRepository.fs +++ b/Damo.Io.Server/src/SourcesRepository.fs @@ -1,8 +1,8 @@ module DamoIoServer.SourcesRepository -open DamoIoServer.Source open FeedsProcessing.Download open FeedsProcessing.Feeds +open DamoIoServer.Source type SourceFeed = { Type: Source diff --git a/FeedsProcessing.Tests/src/XmlTests.fs b/FeedsProcessing.Tests/src/XmlTests.fs index 96dd998..13a2fe1 100644 --- a/FeedsProcessing.Tests/src/XmlTests.fs +++ b/FeedsProcessing.Tests/src/XmlTests.fs @@ -1,19 +1,20 @@ module ``Xml Processor Tests`` -open FeedsProcessing.Article -open FeedsProcessing.Xml -open FeedsProcessingTests.DownloadSupport open FsUnit open FsUnitTyped open NUnit.Framework open System open Time +open FeedsProcessing.Article +open FeedsProcessing.Xml +open FeedsProcessingTests.DownloadSupport + [] let ``with unsupported XML`` () = let download = Download.fromContent "Not quite expected xml content" - let result = processFeed download + let result = Xml.processFeed download match result with | Ok _ -> Assert.Fail "Expected failure" @@ -28,7 +29,7 @@ let ``with github Atom XML`` () = "../../../../FeedsProcessing.Tests/resources/test-samples/github.xml" |> Download.fromFilePath - let result = processFeed download + let result = Xml.processFeed download match result with | Error _ -> Assert.Fail "Expected success" @@ -58,7 +59,7 @@ let ``with Dualshock Atom XML`` () = "../../../../FeedsProcessing.Tests/resources/test-samples/dualshock.xml" |> Download.fromFilePath - let result = processFeed downloaded + let result = Xml.processFeed downloaded match result with | Error _ -> Assert.Fail "Expected success" @@ -71,7 +72,7 @@ let ``with RSS XML`` () = "../../../../FeedsProcessing/resources/samples/rss.sample.xml" |> Download.fromFilePath - let result = processFeed downloadedFeed + let result = Xml.processFeed downloadedFeed match result with | Error _ -> Assert.Fail "Expected success" @@ -112,7 +113,7 @@ let ``with mastodon RSS`` () = "../../../../FeedsProcessing.Tests/resources/test-samples/mastodon.xml" |> Download.fromFilePath - let result = processFeed downloadedFeed + let result = Xml.processFeed downloadedFeed match result with | Error _ -> Assert.Fail "Expected success" @@ -141,7 +142,7 @@ let ``processFeed with slashdot RDF XML`` () = "../../../../FeedsProcessing.Tests/resources/test-samples/slashdot.xml" |> Download.fromFilePath - let result = processFeed downloadedFeed + let result = Xml.processFeed downloadedFeed match result with | Error _ -> Assert.Fail "Expected success" diff --git a/FeedsProcessing/src/Article.fs b/FeedsProcessing/src/Article.fs index 829091c..ab67cca 100644 --- a/FeedsProcessing/src/Article.fs +++ b/FeedsProcessing/src/Article.fs @@ -1,9 +1,10 @@ module FeedsProcessing.Article -open FeedsProcessing open System open Time +open FeedsProcessing.Html + let private stringToOption text = if String.IsNullOrWhiteSpace text then None else Some text diff --git a/FeedsProcessing/src/DataGateway.fs b/FeedsProcessing/src/DataGateway.fs index a999b84..7f14fe5 100644 --- a/FeedsProcessing/src/DataGateway.fs +++ b/FeedsProcessing/src/DataGateway.fs @@ -2,10 +2,11 @@ module FeedsProcessing.DataGateway open FSharp.Data open FSharp.Data.HttpRequestHeaders -open FeedsProcessing.Download open System open System.Web +open FeedsProcessing.Download + type private BasicAuthHeader = BasicAuthHeader of string type private BearerToken = BearerToken of string @@ -56,19 +57,21 @@ let private requestToken (BasicAuthHeader authHeader) = parseToken responseString ) -let downloadContent (Url url) : DownloadResult = - async { - return - Try.value - "Download content" - (fun _ -> - let content = - Http.RequestString( - url, - headers = [ "User-Agent", "somanyfeeds.com" ], - responseEncodingOverride = "utf-8" - ) +[] +module DataGateway = + let download (Url url) : DownloadResult = + async { + return + Try.value + "Download content" + (fun _ -> + let content = + Http.RequestString( + url, + headers = [ "User-Agent", "somanyfeeds.com" ], + responseEncodingOverride = "utf-8" + ) - { Url = (Url url); Content = content } - ) - } + { Url = (Url url); Content = content } + ) + } diff --git a/FeedsProcessing/src/Download.fs b/FeedsProcessing/src/Download.fs index b0381ca..df15a4c 100644 --- a/FeedsProcessing/src/Download.fs +++ b/FeedsProcessing/src/Download.fs @@ -1,6 +1,5 @@ module FeedsProcessing.Download - type Url = Url of string [] diff --git a/FeedsProcessing/src/Feeds.fs b/FeedsProcessing/src/Feeds.fs index f3d80c2..356992f 100644 --- a/FeedsProcessing/src/Feeds.fs +++ b/FeedsProcessing/src/Feeds.fs @@ -2,5 +2,4 @@ module FeedsProcessing.Feeds open FeedsProcessing.Download - type Feed = Xml of Url diff --git a/FeedsProcessing/src/Html.fs b/FeedsProcessing/src/Html.fs index 118e9a3..c4ac85c 100644 --- a/FeedsProcessing/src/Html.fs +++ b/FeedsProcessing/src/Html.fs @@ -28,4 +28,6 @@ let private sanitizer = |> disallow "style" |> removeUrlsContaining "/tracking" -let sanitize = sanitizer.Sanitize +[] +module Html = + let sanitize = sanitizer.Sanitize diff --git a/FeedsProcessing/src/ProcessingResult.fs b/FeedsProcessing/src/ProcessingResult.fs index fa34e73..bafe227 100644 --- a/FeedsProcessing/src/ProcessingResult.fs +++ b/FeedsProcessing/src/ProcessingResult.fs @@ -2,5 +2,4 @@ module FeedsProcessing.ProcessingResult open FeedsProcessing.Article - type ProcessingResult = Result
diff --git a/FeedsProcessing/src/Search.fs b/FeedsProcessing/src/Search.fs index ffe490d..65cdb86 100644 --- a/FeedsProcessing/src/Search.fs +++ b/FeedsProcessing/src/Search.fs @@ -1,7 +1,7 @@ module FeedsProcessing.Search open FSharp.Data -open FeedsProcessing + open FeedsProcessing.Download open FeedsProcessing.Xml diff --git a/FeedsProcessing/src/Xml.fs b/FeedsProcessing/src/Xml.fs index 5f553b3..3bf798c 100644 --- a/FeedsProcessing/src/Xml.fs +++ b/FeedsProcessing/src/Xml.fs @@ -1,10 +1,11 @@ module FeedsProcessing.Xml open FSharp.Data +open System + open FeedsProcessing.Article open FeedsProcessing.Download open FeedsProcessing.ProcessingResult -open System let private stringToOption text = if String.IsNullOrWhiteSpace text then None else Some text @@ -141,10 +142,12 @@ let private tryProcessor downloaded (previousState: ProcessingResult) (processor let private processors: Processor list = [ Rss.processor; Atom.processor; Rdf.processor ] -let processFeed (download: Download) : ProcessingResult = - (Error.ofMessage "", processors) - ||> List.fold (tryProcessor download) - |> Result.mapError (Explanation.wrapMessage (sprintf "Failed all the parsers: %s")) - -let tryGetMetadata (download: Download) : FeedMetadata option = - processors |> List.choose (fun p -> p.TryGetMetadata download) |> List.tryHead +[] +module Xml = + let processFeed (download: Download) : ProcessingResult = + (Error.ofMessage "", processors) + ||> List.fold (tryProcessor download) + |> Result.mapError (Explanation.wrapMessage (sprintf "Failed all the parsers: %s")) + + let tryGetMetadata (download: Download) : FeedMetadata option = + processors |> List.choose (fun p -> p.TryGetMetadata download) |> List.tryHead diff --git a/somanyfeeds.sln.DotSettings b/somanyfeeds.sln.DotSettings index 0324e76..498e734 100644 --- a/somanyfeeds.sln.DotSettings +++ b/somanyfeeds.sln.DotSettings @@ -1,5 +1,5 @@  - True + True True True True