diff --git a/go.mod b/go.mod index 57a77a9..54eae61 100644 --- a/go.mod +++ b/go.mod @@ -68,6 +68,7 @@ require ( github.com/lestrrat-go/jwx/v2 v2.0.18 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-encoding v0.0.2 github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.19 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index eb5aaf2..4d5eb0f 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9t github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-encoding v0.0.2 h1:OC1L+QXLJge9n7yIE3R5Os/UNasUeFvK3Sa4NjbDi6c= +github.com/mattn/go-encoding v0.0.2/go.mod h1:WUNsdPQLK4JYRzkn8IAdmYKFYGGJ4/9YPxdPoMumPgY= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= diff --git a/timeline.go b/timeline.go index 888a033..1815aca 100644 --- a/timeline.go +++ b/timeline.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "bytes" "context" "encoding/json" @@ -28,9 +29,11 @@ import ( "github.com/bluesky-social/indigo/xrpc" "github.com/fatih/color" cid "github.com/ipfs/go-cid" + "golang.org/x/net/html/charset" "github.com/PuerkitoBio/goquery" "github.com/gorilla/websocket" + encoding "github.com/mattn/go-encoding" "github.com/urfave/cli/v2" ) @@ -181,53 +184,73 @@ func doDelete(cCtx *cli.Context) error { } func addLink(xrpcc *xrpc.Client, post *bsky.FeedPost, link string) { - doc, err := goquery.NewDocument(link) - var title string - var description string - var imgURL string - - if err == nil { - title = doc.Find(`title`).Text() - description, _ = doc.Find(`meta[property="description"]`).Attr("content") - imgURL, _ = doc.Find(`meta[property="og:image"]`).Attr("content") - if title == "" { - title, _ = doc.Find(`meta[property="og:title"]`).Attr("content") - if title == "" { - title = link + res, _ := http.Get(link) + if res != nil { + defer res.Body.Close() + + br := bufio.NewReader(res.Body) + var reader io.Reader = br + + data, err2 := br.Peek(1024) + if err2 == nil { + enc, name, _ := charset.DetermineEncoding(data, res.Header.Get("content-type")) + if enc != nil { + reader = enc.NewDecoder().Reader(br) + } else if len(name) > 0 { + enc := encoding.GetEncoding(name) + if enc != nil { + reader = enc.NewDecoder().Reader(br) + } } } - if description == "" { - description, _ = doc.Find(`meta[property="og:description"]`).Attr("content") + + var title string + var description string + var imgURL string + doc, err := goquery.NewDocumentFromReader(reader) + if err == nil { + title = doc.Find(`title`).Text() + description, _ = doc.Find(`meta[property="description"]`).Attr("content") + imgURL, _ = doc.Find(`meta[property="og:image"]`).Attr("content") + if title == "" { + title, _ = doc.Find(`meta[property="og:title"]`).Attr("content") + if title == "" { + title = link + } + } if description == "" { - description = link + description, _ = doc.Find(`meta[property="og:description"]`).Attr("content") + if description == "" { + description = link + } + } + post.Embed.EmbedExternal = &bsky.EmbedExternal{ + External: &bsky.EmbedExternal_External{ + Description: description, + Title: title, + Uri: link, + }, + } + } else { + post.Embed.EmbedExternal = &bsky.EmbedExternal{ + External: &bsky.EmbedExternal_External{ + Uri: link, + }, } } - post.Embed.EmbedExternal = &bsky.EmbedExternal{ - External: &bsky.EmbedExternal_External{ - Description: description, - Title: title, - Uri: link, - }, - } - } else { - post.Embed.EmbedExternal = &bsky.EmbedExternal{ - External: &bsky.EmbedExternal_External{ - Uri: link, - }, - } - } - if imgURL != "" && post.Embed.EmbedExternal != nil { - resp, err := http.Get(imgURL) - if err == nil && resp.StatusCode == http.StatusOK { - defer resp.Body.Close() - b, err := io.ReadAll(resp.Body) - if err == nil { - resp, err := comatproto.RepoUploadBlob(context.TODO(), xrpcc, bytes.NewReader(b)) + if imgURL != "" && post.Embed.EmbedExternal != nil { + resp, err := http.Get(imgURL) + if err == nil && resp.StatusCode == http.StatusOK { + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) if err == nil { - post.Embed.EmbedExternal.External.Thumb = &lexutil.LexBlob{ - Ref: resp.Blob.Ref, - MimeType: http.DetectContentType(b), - Size: resp.Blob.Size, + resp, err := comatproto.RepoUploadBlob(context.TODO(), xrpcc, bytes.NewReader(b)) + if err == nil { + post.Embed.EmbedExternal.External.Thumb = &lexutil.LexBlob{ + Ref: resp.Blob.Ref, + MimeType: http.DetectContentType(b), + Size: resp.Blob.Size, + } } } }