Merge pull request #25 from mkyutani/linkcardwithcharset

Recognize charset of document for link card

go.mod

@@ -68,6 +68,7 @@ require (
 	github.com/lestrrat-go/jwx/v2 v2.0.18 // indirect
 	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-encoding v0.0.2
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-sqlite3 v1.14.19 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect

go.sum

@@ -221,6 +221,8 @@ github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9t
 github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-encoding v0.0.2 h1:OC1L+QXLJge9n7yIE3R5Os/UNasUeFvK3Sa4NjbDi6c=
+github.com/mattn/go-encoding v0.0.2/go.mod h1:WUNsdPQLK4JYRzkn8IAdmYKFYGGJ4/9YPxdPoMumPgY=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=

timeline.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -28,9 +29,11 @@ import (
 	"github.com/bluesky-social/indigo/xrpc"
 	"github.com/fatih/color"
 	cid "github.com/ipfs/go-cid"
+	"golang.org/x/net/html/charset"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/gorilla/websocket"
+	encoding "github.com/mattn/go-encoding"
 	"github.com/urfave/cli/v2"
 )
 
@@ -181,53 +184,73 @@ func doDelete(cCtx *cli.Context) error {
 }
 
 func addLink(xrpcc *xrpc.Client, post *bsky.FeedPost, link string) {
-	doc, err := goquery.NewDocument(link)
-	var title string
-	var description string
-	var imgURL string
-
-	if err == nil {
-		title = doc.Find(`title`).Text()
-		description, _ = doc.Find(`meta[property="description"]`).Attr("content")
-		imgURL, _ = doc.Find(`meta[property="og:image"]`).Attr("content")
-		if title == "" {
-			title, _ = doc.Find(`meta[property="og:title"]`).Attr("content")
-			if title == "" {
-				title = link
+	res, _ := http.Get(link)
+	if res != nil {
+		defer res.Body.Close()
+
+		br := bufio.NewReader(res.Body)
+		var reader io.Reader = br
+
+		data, err2 := br.Peek(1024)
+		if err2 == nil {
+			enc, name, _ := charset.DetermineEncoding(data, res.Header.Get("content-type"))
+			if enc != nil {
+				reader = enc.NewDecoder().Reader(br)
+			} else if len(name) > 0 {
+				enc := encoding.GetEncoding(name)
+				if enc != nil {
+					reader = enc.NewDecoder().Reader(br)
+				}
 			}
 		}
-		if description == "" {
-			description, _ = doc.Find(`meta[property="og:description"]`).Attr("content")
+
+		var title string
+		var description string
+		var imgURL string
+		doc, err := goquery.NewDocumentFromReader(reader)
+		if err == nil {
+			title = doc.Find(`title`).Text()
+			description, _ = doc.Find(`meta[property="description"]`).Attr("content")
+			imgURL, _ = doc.Find(`meta[property="og:image"]`).Attr("content")
+			if title == "" {
+				title, _ = doc.Find(`meta[property="og:title"]`).Attr("content")
+				if title == "" {
+					title = link
+				}
+			}
 			if description == "" {
-				description = link
+				description, _ = doc.Find(`meta[property="og:description"]`).Attr("content")
+				if description == "" {
+					description = link
+				}
+			}
+			post.Embed.EmbedExternal = &bsky.EmbedExternal{
+				External: &bsky.EmbedExternal_External{
+					Description: description,
+					Title:       title,
+					Uri:         link,
+				},
+			}
+		} else {
+			post.Embed.EmbedExternal = &bsky.EmbedExternal{
+				External: &bsky.EmbedExternal_External{
+					Uri: link,
+				},
 			}
 		}
-		post.Embed.EmbedExternal = &bsky.EmbedExternal{
-			External: &bsky.EmbedExternal_External{
-				Description: description,
-				Title:       title,
-				Uri:         link,
-			},
-		}
-	} else {
-		post.Embed.EmbedExternal = &bsky.EmbedExternal{
-			External: &bsky.EmbedExternal_External{
-				Uri: link,
-			},
-		}
-	}
-	if imgURL != "" && post.Embed.EmbedExternal != nil {
-		resp, err := http.Get(imgURL)
-		if err == nil && resp.StatusCode == http.StatusOK {
-			defer resp.Body.Close()
-			b, err := io.ReadAll(resp.Body)
-			if err == nil {
-				resp, err := comatproto.RepoUploadBlob(context.TODO(), xrpcc, bytes.NewReader(b))
+		if imgURL != "" && post.Embed.EmbedExternal != nil {
+			resp, err := http.Get(imgURL)
+			if err == nil && resp.StatusCode == http.StatusOK {
+				defer resp.Body.Close()
+				b, err := io.ReadAll(resp.Body)
 				if err == nil {
-					post.Embed.EmbedExternal.External.Thumb = &lexutil.LexBlob{
-						Ref:      resp.Blob.Ref,
-						MimeType: http.DetectContentType(b),
-						Size:     resp.Blob.Size,
+					resp, err := comatproto.RepoUploadBlob(context.TODO(), xrpcc, bytes.NewReader(b))
+					if err == nil {
+						post.Embed.EmbedExternal.External.Thumb = &lexutil.LexBlob{
+							Ref:      resp.Blob.Ref,
+							MimeType: http.DetectContentType(b),
+							Size:     resp.Blob.Size,
+						}
 					}
 				}
 			}