From 9524b9e65cf115f70d8b8268aca3005f9d2e63af Mon Sep 17 00:00:00 2001 From: Anthony Scotti Date: Sun, 10 Mar 2024 18:12:02 -0400 Subject: [PATCH] Updates to remove Mutex --- .gitignore | 3 +- Dockerfile | 4 +- LICENSE | 4 +- README.md | 10 ++-- go.mod | 14 ++--- go.sum | 23 ++++---- hackernews/api.go | 123 +++++++++++++++++++++++-------------------- hackernews/models.go | 1 + main.go | 2 +- 9 files changed, 101 insertions(+), 83 deletions(-) diff --git a/.gitignore b/.gitignore index adace10..a22c027 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -hacker_news \ No newline at end of file +hacker_news +.DS_Store diff --git a/Dockerfile b/Dockerfile index d7d9991..258521f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20 AS builder +FROM golang:1.22 AS builder WORKDIR /app @@ -17,4 +17,4 @@ WORKDIR / COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /hacker_news /hacker_news -ENTRYPOINT ["/hacker_news"] \ No newline at end of file +ENTRYPOINT ["/hacker_news"] diff --git a/LICENSE b/LICENSE index eeabf0f..c4be274 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Anthony Scotti +Copyright (c) 2024 Anthony Scotti Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/README.md b/README.md index 2223c1e..adcc11c 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,14 @@ A command line tool that shows the top stories from [Hacker News](https://news.ycombinator.com/) using the [Hacker News API from Firebase](https://github.com/HackerNews/API). -This is a port of an dotnet version, [hn-top](https://github.com/amscotti/hn-top) which I used to learn about Go and understand how to keep the order of the stories when using Goroutines with help of Mutex and WaitGroup. +This is a port of an dotnet version, [hn-top](https://github.com/amscotti/hn-top) which I used to learn about Go and understand how to keep the order of the stories when using Goroutines. ![hacker_news Output](hacker_news.png) ## Building and Running ### With Go + To build and run the application, follow these steps: 1. Clone the repository. @@ -17,6 +18,7 @@ To build and run the application, follow these steps: 4. Run the binary with `./hacker_news`. ### With Docker + To build and run the application with Docker, follow these steps: 1. Clone the repository. @@ -25,17 +27,19 @@ To build and run the application with Docker, follow these steps: 4. Run the Docker container with `docker run hacker_news`. ### With pre-built Docker image from [ghcr.io](https://github.com/amscotti/hacker_news/pkgs/container/hacker_news) + To download and run the pre-built Docker image, follow these steps: 1. Install Docker. 2. Run the command `docker run ghcr.io/amscotti/hacker_news:main`. ### Command Line Arguments + The following command line arguments are available: ``` Usage of ./hacker_news: -n int - Specify the number of top stories to display (default: 5). (default 5) + Specify the number of top stories to display. (default: 30) -u Include the source URLs of the stories in the output. -``` \ No newline at end of file +``` diff --git a/go.mod b/go.mod index 5f0de8d..0376f29 100644 --- a/go.mod +++ b/go.mod @@ -1,17 +1,17 @@ module github.com/amscotti/hacker_news -go 1.20 +go 1.22 -require github.com/charmbracelet/lipgloss v0.7.1 +require github.com/charmbracelet/lipgloss v0.10.0 require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.15.1 // indirect - github.com/rivo/uniseg v0.2.0 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/valyala/fastjson v1.6.4 - golang.org/x/sys v0.6.0 // indirect + golang.org/x/sys v0.12.0 // indirect ) diff --git a/go.sum b/go.sum index 2bae8e2..3860c99 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,24 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= -github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= -github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= +github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s= +github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= -github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/hackernews/api.go b/hackernews/api.go index da8e791..2b20b91 100644 --- a/hackernews/api.go +++ b/hackernews/api.go @@ -16,19 +16,29 @@ const ( ITEM_URL_BASE = "https://hacker-news.firebaseio.com/v0/item" ) -func createClient() *http.Client { - transport := &http.Transport{ - MaxIdleConns: 100, - IdleConnTimeout: 30 * time.Second, - } - return &http.Client{ - Transport: transport, - Timeout: time.Duration(10 * time.Second), - } +type StoryIdPair struct { + PlacementId int + StoryId int +} + +var clientPool = sync.Pool{ + New: func() interface{} { + transport := &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 30 * time.Second, + } + return &http.Client{ + Transport: transport, + Timeout: time.Duration(10 * time.Second), + } + }, } func fetchJSON(url string) (*fastjson.Value, error) { - client := createClient() + client := clientPool.Get().(*http.Client) + defer clientPool.Put(client) + resp, err := client.Get(url) if err != nil { return nil, fmt.Errorf("failed to fetch %s: %w", url, err) @@ -48,36 +58,36 @@ func fetchJSON(url string) (*fastjson.Value, error) { return value, nil } -func downloadStory(id int, wg *sync.WaitGroup, m *sync.Mutex, stories map[int]Story) { - defer wg.Done() - - url := fmt.Sprintf("%s/%d.json", ITEM_URL_BASE, id) - - value, err := fetchJSON(url) - if err != nil { - log.Printf("Error fetching story with id %d: %v", id, err) - return +func downloadStory(storyIdChan <-chan StoryIdPair, storyChan chan<- Story) { + for storyId := range storyIdChan { + url := fmt.Sprintf("%s/%d.json", ITEM_URL_BASE, storyId.StoryId) + + value, err := fetchJSON(url) + if err != nil { + log.Printf("Error fetching story with id %d: %v", storyId.StoryId, err) + storyChan <- Story{} + return + } + + story := Story{ + Index: storyId.PlacementId, + Title: string(value.GetStringBytes("title")), + By: string(value.GetStringBytes("by")), + Descendants: int(value.GetInt("descendants")), + Id: int(value.GetInt("id")), + Score: int(value.GetInt("score")), + Time: int(value.GetInt("time")), + Type: string(value.GetStringBytes("type")), + URL: string(value.GetStringBytes("url")), + } + + kids := value.GetArray("kids") + for i := range kids { + story.Kids = append(story.Kids, int(kids[i].GetInt())) + } + + storyChan <- story } - - story := Story{ - Title: string(value.GetStringBytes("title")), - By: string(value.GetStringBytes("by")), - Descendants: int(value.GetInt("descendants")), - Id: int(value.GetInt("id")), - Score: int(value.GetInt("score")), - Time: int(value.GetInt("time")), - Type: string(value.GetStringBytes("type")), - URL: string(value.GetStringBytes("url")), - } - - kids := value.GetArray("kids") - for i := range kids { - story.Kids = append(story.Kids, int(kids[i].GetInt())) - } - - m.Lock() - stories[id] = story - m.Unlock() } func FetchTopStories(number int, showSourceUrls bool) { @@ -87,32 +97,33 @@ func FetchTopStories(number int, showSourceUrls bool) { return } - var ids []int valueArray := value.GetArray() - for i := 0; i < len(valueArray); i++ { - id := int(valueArray[i].GetInt()) - ids = append(ids, id) - } + count := min(len(valueArray), number) - if len(ids) > number { - ids = ids[:number] - } + stories := make([]Story, count) - storyDetails := make(map[int]Story) + storyIdChan := make(chan StoryIdPair, count) + storyChan := make(chan Story, count) - var m sync.Mutex - var wg sync.WaitGroup + for range count { + go downloadStory(storyIdChan, storyChan) + } - wg.Add(len(ids)) + go func() { + for index, id := range valueArray[:number] { + storyIdChan <- StoryIdPair{PlacementId: index, StoryId: id.GetInt()} + } + close(storyIdChan) + }() - for _, id := range ids { - go downloadStory(id, &wg, &m, storyDetails) + for range count { + story := <-storyChan + stories[story.Index] = story } - wg.Wait() + close(storyChan) - for _, id := range ids { - story := storyDetails[id] + for _, story := range stories { fmt.Print(story.PrintStyling(showSourceUrls)) } } diff --git a/hackernews/models.go b/hackernews/models.go index 2be8fce..aaf3b46 100644 --- a/hackernews/models.go +++ b/hackernews/models.go @@ -11,6 +11,7 @@ const ( ) type Story struct { + Index int Title string By string Descendants int diff --git a/main.go b/main.go index 07badb4..83bccb1 100644 --- a/main.go +++ b/main.go @@ -12,7 +12,7 @@ func main() { var number int var showSourceUrls bool - flag.IntVar(&number, "n", 5, "Specify the number of top stories to display (default: 5).") + flag.IntVar(&number, "n", 30, "Specify the number of top stories to display.") flag.BoolVar(&showSourceUrls, "u", false, "Include the source URLs of the stories in the output.") flag.Parse()