Skip to content

Commit

Permalink
Give priority to the canonical URL
Browse files Browse the repository at this point in the history
  • Loading branch information
@spiegel-im-spiegel
spiegel-im-spiegel committed Jun 20, 2021
1 parent c5f79c1 commit 4d995d3
Show file tree
Hide file tree
Showing 6 changed files with 36 additions and 14 deletions.
2 changes: 1 addition & 1 deletion Taskfile.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ tasks:
cmds:
- go mod verify
- go test ./...
- docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.40.1 golangci-lint run --enable gosec --timeout 3m0s ./...
- docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.41.1 golangci-lint run --enable gosec --timeout 3m0s ./...
sources:
- ./go.mod
- '**/*.go'
Expand Down
Binary file modified dependency.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ module github.com/spiegel-im-spiegel/ml
go 1.16

require (
github.com/PuerkitoBio/goquery v1.6.1
github.com/PuerkitoBio/goquery v1.7.0
github.com/atotto/clipboard v0.1.4
github.com/mattn/go-encoding v0.0.2
github.com/spf13/cobra v1.1.3
github.com/spiegel-im-spiegel/errs v1.0.2
github.com/spiegel-im-spiegel/fetch v0.2.3
github.com/spiegel-im-spiegel/gocli v0.10.4
github.com/zetamatta/go-readline-ny v0.4.12
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f
golang.org/x/net v0.0.0-20210614182718-04defd469f4e
)
8 changes: 4 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk=
github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/PuerkitoBio/goquery v1.7.0 h1:O5SP3b9JWqMSVMG69zMfj577zwkSNpxrFf7ybS74eiw=
github.com/PuerkitoBio/goquery v1.7.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
Expand Down Expand Up @@ -232,8 +232,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f h1:Si4U+UcgJzya9kpiEUJKQvjr512OLli+gL4poHrz93U=
golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down
30 changes: 26 additions & 4 deletions makelink/makelink.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
type Link struct {
URL string `json:"url,omitempty"`
Location string `json:"location,omitempty"`
Canonical string `json:"canonical,omitempty"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`
}
Expand Down Expand Up @@ -71,6 +72,13 @@ func New(ctx context.Context, urlStr string) (*Link, error) {
}
}
})
s.Find("link[rel='canonical']").Each(func(_ int, s *goquery.Selection) {
if v, ok := s.Attr("href"); ok {
if len(v) > 0 {
link.Canonical = trimString(v)
}
}
})
})
return link, nil
}
Expand All @@ -96,6 +104,20 @@ func (lnk *Link) TitleName() string {
return lnk.URL
}

//CanonicalURL returns the canonical URL.
func (lnk *Link) CanonicalURL() string {
if lnk == nil {
return ""
}
if len(lnk.Canonical) > 0 {
return lnk.Canonical
}
if len(lnk.Location) > 0 {
return lnk.Location
}
return lnk.URL
}

//Encode returns string (io.Reader) with other style
func (lnk *Link) Encode(t Style) io.Reader {
if lnk == nil {
Expand All @@ -104,13 +126,13 @@ func (lnk *Link) Encode(t Style) io.Reader {
buf := &bytes.Buffer{}
switch t {
case StyleMarkdown:
fmt.Fprintf(buf, "[%s](%s)", lnk.TitleName(), lnk.Location)
fmt.Fprintf(buf, "[%s](%s)", lnk.TitleName(), lnk.CanonicalURL())
case StyleWiki:
fmt.Fprintf(buf, "[%s %s]", lnk.Location, lnk.TitleName())
fmt.Fprintf(buf, "[%s %s]", lnk.CanonicalURL(), lnk.TitleName())
case StyleHTML:
fmt.Fprintf(buf, "<a href=\"%s\">%s</a>", lnk.Location, lnk.TitleName())
fmt.Fprintf(buf, "<a href=\"%s\">%s</a>", lnk.CanonicalURL(), lnk.TitleName())
case StyleCSV:
fmt.Fprintf(buf, "\"%s\",\"%s\",\"%s\",\"%s\"", escapeQuoteCsv(lnk.URL), escapeQuoteCsv(lnk.Location), escapeQuoteCsv(lnk.Title), escapeQuoteCsv(lnk.Description))
fmt.Fprintf(buf, "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"", escapeQuoteCsv(lnk.URL), escapeQuoteCsv(lnk.Location), escapeQuoteCsv(lnk.Canonical), escapeQuoteCsv(lnk.Title), escapeQuoteCsv(lnk.Description))
case StyleJSON:
_ = json.NewEncoder(buf).Encode(lnk)
}
Expand Down
6 changes: 3 additions & 3 deletions makelink/makelink_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ var typesTests2 = []typesTestCase{
{"[GitHub - spiegel-im-spiegel/ml: Make Link with Markdown Format](https://github.com/spiegel-im-spiegel/ml)", StyleMarkdown},
{"[https://github.com/spiegel-im-spiegel/ml GitHub - spiegel-im-spiegel/ml: Make Link with Markdown Format]", StyleWiki},
{"<a href=\"https://github.com/spiegel-im-spiegel/ml\">GitHub - spiegel-im-spiegel/ml: Make Link with Markdown Format</a>", StyleHTML},
{"\"https://git.io/vFR5M\",\"https://github.com/spiegel-im-spiegel/ml\",\"GitHub - spiegel-im-spiegel/ml: Make Link with Markdown Format\",\"ml - Make Link with Markdown Format\"", StyleCSV},
{"\"https://git.io/vFR5M\",\"https://github.com/spiegel-im-spiegel/ml\",\"\",\"GitHub - spiegel-im-spiegel/ml: Make Link with Markdown Format\",\"ml - Make Link with Markdown Format\"", StyleCSV},
{"", StyleUnknown},
}

Expand All @@ -36,12 +36,12 @@ var typesTests3 = []typesTestCase{
{"[https://git.io/vFR5M](https://github.com/spiegel-im-spiegel/ml)", StyleMarkdown},
{"[https://github.com/spiegel-im-spiegel/ml https://git.io/vFR5M]", StyleWiki},
{"<a href=\"https://github.com/spiegel-im-spiegel/ml\">https://git.io/vFR5M</a>", StyleHTML},
{"\"https://git.io/vFR5M\",\"https://github.com/spiegel-im-spiegel/ml\",\"\",\"\"", StyleCSV},
{"\"https://git.io/vFR5M\",\"https://github.com/spiegel-im-spiegel/ml\",\"https://github.com/spiegel-im-spiegel/ml\",\"\",\"\"", StyleCSV},
{"", StyleUnknown},
}

func TestEncodeNoUTF8(t *testing.T) {
lnk := &Link{URL: "https://git.io/vFR5M", Location: "https://github.com/spiegel-im-spiegel/ml", Title: "", Description: ""}
lnk := &Link{URL: "https://git.io/vFR5M", Location: "https://github.com/spiegel-im-spiegel/ml", Canonical: "https://github.com/spiegel-im-spiegel/ml", Title: "", Description: ""}
for _, tst := range typesTests3 {
r := lnk.Encode(tst.t)
buf := new(bytes.Buffer)
Expand Down

0 comments on commit 4d995d3

Please sign in to comment.