Add support for SDK 0.35

Implement thumbnails uploader
Fetch more data from youtube for additional metadata

go.mod

@@ -1,30 +1,47 @@
 module github.com/lbryio/ytsync
 
 require (
+	cloud.google.com/go v0.37.4 // indirect
+	github.com/ChannelMeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61
 	github.com/PuerkitoBio/goquery v1.5.0 // indirect
 	github.com/aws/aws-sdk-go v1.17.3
-	github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 // indirect
+	github.com/btcsuite/btcd v0.0.0-20190410025418-9bfb2ca0346b // indirect
+	github.com/btcsuite/btcutil v0.0.0-20190316010144-3ac1210f4b38 // indirect
+	github.com/channelmeter/iso8601duration v0.0.0-20150204201828-8da3af7a2a61 // indirect
 	github.com/go-ini/ini v1.42.0 // indirect
+	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect
 	github.com/gorilla/websocket v1.4.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pretty v0.1.0 // indirect
-	github.com/lbryio/lbry.go v0.0.0-20190321133538-94802df513b0
+	github.com/lbryio/errors.go v0.0.0-20180223142025-ad03d3cc6a5c
+	github.com/lbryio/lbry.go v0.0.0-20190419005332-80b25b225e18
+	github.com/lbryio/types v0.0.0-20190415181811-35ddf1afe731 // indirect
+	github.com/lusis/slack-test v0.0.0-20190408224659-6cf59653add2 // indirect
 	github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/nikooo777/ytdl v0.0.0-20190215151411-9c7832eaf457
+	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/prometheus/common v0.3.0
 	github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
-	github.com/sirupsen/logrus v1.3.0
-	github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac // indirect
-	github.com/smartystreets/goconvey v0.0.0-20190222165329-3881c169ceb8 // indirect
+	github.com/sirupsen/logrus v1.4.1
+	github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
+	github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect
 	github.com/spf13/cobra v0.0.0-20190109003409-7547e83b2d85
 	github.com/spf13/pflag v1.0.3 // indirect
 	github.com/ybbus/jsonrpc v2.1.2+incompatible // indirect
-	golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2 // indirect
-	golang.org/x/net v0.0.0-20190213061140-3a22650c66bd // indirect
-	golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect
-	golang.org/x/sys v0.0.0-20190222171317-cd391775e71e // indirect
-	golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 // indirect
-	google.golang.org/api v0.1.0
+	go.opencensus.io v0.20.2 // indirect
+	golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480 // indirect
+	golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 // indirect
+	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
+	golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be // indirect
+	google.golang.org/api v0.3.2
+	google.golang.org/appengine v1.5.0 // indirect
+	google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
+	google.golang.org/grpc v1.20.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/ini.v1 v1.42.0 // indirect
 	gopkg.in/yaml.v2 v2.2.2 // indirect

main.go

@@ -115,7 +115,7 @@ func ytSync(cmd *cobra.Command, args []string) {
 	awsS3Region := os.Getenv("AWS_S3_REGION")
 	awsS3Bucket := os.Getenv("AWS_S3_BUCKET")
 	if apiURL == "" {
-		log.Errorln("An API URL was not defined. Please set the environment variable LBRY_API")
+		log.Errorln("An API URL was not defined. Please set the environment variable LBRY_WEB_API")
 		return
 	}
 	if apiToken == "" {

manager/setup.go

@@ -2,16 +2,23 @@ package manager
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 	"strconv"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/lbryio/lbry.go/extras/errors"
 	"github.com/lbryio/lbry.go/extras/jsonrpc"
+	"github.com/lbryio/lbry.go/extras/util"
 	"github.com/lbryio/lbry.go/lbrycrd"
+	"github.com/lbryio/ytsync/thumbs"
 
 	"github.com/shopspring/decimal"
 	log "github.com/sirupsen/logrus"
+	"google.golang.org/api/googleapi/transport"
+	"google.golang.org/api/youtube/v3"
 )
 
 func (s *Sync) walletSetup() error {
@@ -274,12 +281,65 @@ func (s *Sync) ensureChannelOwnership() error {
 			return err
 		}
 	}
+	client := &http.Client{
+		Transport: &transport.APIKey{Key: s.APIConfig.YoutubeAPIKey},
+	}
+
+	service, err := youtube.New(client)
+	if err != nil {
+		return errors.Prefix("error creating YouTube service", err)
+	}
+
+	response, err := service.Channels.List("snippet").Id(s.YoutubeChannelID).Do()
+	if err != nil {
+		return errors.Prefix("error getting channel details", err)
+	}
 
-	c, err := s.daemon.ChannelNew(s.LbryChannelName, channelBidAmount, nil)
+	if len(response.Items) < 1 {
+		return errors.Err("youtube channel not found")
+	}
+
+	channelInfo := response.Items[0].Snippet
+
+	thumbnail := channelInfo.Thumbnails.Default
+	if channelInfo.Thumbnails.Maxres != nil {
+		thumbnail = channelInfo.Thumbnails.Maxres
+	} else if channelInfo.Thumbnails.High != nil {
+		thumbnail = channelInfo.Thumbnails.High
+	} else if channelInfo.Thumbnails.Medium != nil {
+		thumbnail = channelInfo.Thumbnails.Medium
+	} else if channelInfo.Thumbnails.Standard != nil {
+		thumbnail = channelInfo.Thumbnails.Standard
+	}
+	thumbnailURL, err := thumbs.MirrorThumbnail(thumbnail.Url, s.YoutubeChannelID, aws.Config{
+		Credentials: credentials.NewStaticCredentials(s.AwsS3ID, s.AwsS3Secret, ""),
+		Region:      &s.AwsS3Region,
+	})
+	if err != nil {
+		return err
+	}
+	var languages []string = nil
+	if channelInfo.DefaultLanguage != "" {
+		languages = []string{channelInfo.DefaultLanguage}
+	}
+	var locations []jsonrpc.Location = nil
+	if channelInfo.Country != "" {
+		locations = []jsonrpc.Location{{Country: util.PtrToString(channelInfo.Country)}}
+	}
+	c, err := s.daemon.ChannelCreate(s.LbryChannelName, channelBidAmount, jsonrpc.ChannelCreateOptions{
+		ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
+			Title:        channelInfo.Title,
+			Description:  channelInfo.Description,
+			Tags:         nil,
+			Languages:    languages,
+			Locations:    locations,
+			ThumbnailURL: &thumbnailURL,
+		},
+	})
 	if err != nil {
 		return err
 	}
-	s.lbryChannelID = c.ClaimID
+	s.lbryChannelID = c.Outputs[0].ClaimID
 	return s.Manager.apiConfig.SetChannelClaimID(s.YoutubeChannelID, s.lbryChannelID)
 }
 

manager/ytsync.go

@@ -388,16 +388,22 @@ func logShutdownError(shutdownErr error) {
 	SendErrorToSlack("WALLET HAS NOT BEEN MOVED TO THE WALLET BACKUP DIR")
 }
 
+var thumbnailHosts = []string{
+	"berk.ninja/thumbnails/",
+	"https://thumbnails.lbry.com/",
+}
+
 func isYtsyncClaim(c jsonrpc.Claim) bool {
-	if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.Stream == nil {
+
+	if !util.InSlice(c.Type, []string{"claim", "update"}) || c.Value.GetStream() == nil {
 		return false
 	}
-	if c.Value.Stream.Metadata == nil || c.Value.Stream.Metadata.Thumbnail == nil {
+	if c.Value.GetStream().GetThumbnailUrl() == "" {
 		//most likely a claim created outside of ytsync, ignore!
 		return false
 	}
 
-	return strings.Contains(*c.Value.Stream.Metadata.Thumbnail, "https://berk.ninja/thumbnails/")
+	return util.InSlice(c.Value.GetStream().GetThumbnailUrl(), thumbnailHosts)
 }
 
 // fixDupes abandons duplicate claims
@@ -408,7 +414,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
 		if !isYtsyncClaim(c) {
 			continue
 		}
-		tn := *c.Value.Stream.Metadata.Thumbnail
+		tn := c.Value.GetStream().GetThumbnailUrl()
 		videoID := tn[strings.LastIndex(tn, "/")+1:]
 
 		log.Infof("claimid: %s, claimName: %s, videoID: %s", c.ClaimID, c.Name, videoID)
@@ -425,7 +431,7 @@ func (s *Sync) fixDupes(claims []jsonrpc.Claim) (bool, error) {
 			videoIDs[videoID] = c
 		}
 		log.Debugf("abandoning %+v", claimToAbandon)
-		_, err := s.daemon.ClaimAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
+		_, err := s.daemon.StreamAbandon(claimToAbandon.Txid, claimToAbandon.Nout, nil, false)
 		if err != nil {
 			return true, err
 		}
@@ -444,7 +450,7 @@ func (s *Sync) updateRemoteDB(claims []jsonrpc.Claim) (total int, fixed int, err
 		}
 		count++
 		//check if claimID is in remote db
-		tn := *c.Value.Stream.Metadata.Thumbnail
+		tn := c.Value.GetStream().GetThumbnailUrl()
 		videoID := tn[strings.LastIndex(tn, "/")+1:]
 		pv, ok := s.syncedVideos[videoID]
 		if !ok || pv.ClaimName != c.Name {
@@ -464,7 +470,7 @@ func (s *Sync) getClaims() ([]jsonrpc.Claim, error) {
 	totalPages := uint64(1)
 	var allClaims []jsonrpc.Claim
 	for page := uint64(1); page <= totalPages; page++ {
-		claims, err := s.daemon.ClaimListMine(nil, page, 50)
+		claims, err := s.daemon.ClaimList(nil, page, 50)
 		if err != nil {
 			return nil, errors.Prefix("cannot list claims", err)
 		}
@@ -699,11 +705,22 @@ func (s *Sync) enqueueYoutubeVideos() error {
 			}
 			return errors.Err("playlist items not found")
 		}
-
-		for _, item := range playlistResponse.Items {
+		playlistMap := make(map[string]*youtube.PlaylistItemSnippet, 50)
+		videoIDs := make([]string, 50)
+		for i, item := range playlistResponse.Items {
 			// normally we'd send the video into the channel here, but youtube api doesn't have sorting
 			// so we have to get ALL the videos, then sort them, then send them in
-			videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item.Snippet))
+			playlistMap[item.Snippet.ResourceId.VideoId] = item.Snippet
+			videoIDs[i] = item.Snippet.ResourceId.VideoId
+		}
+		req2 := service.Videos.List("snippet,contentDetails").Id(strings.Join(videoIDs[:], ","))
+
+		videosListResponse, err := req2.Do()
+		if err != nil {
+			return errors.Prefix("error getting videos info", err)
+		}
+		for _, item := range videosListResponse.Items {
+			videos = append(videos, sources.NewYoutubeVideo(s.videoDirectory, item, playlistMap[item.Id].Position))
 		}
 
 		log.Infof("Got info for %d videos from youtube API", len(videos))

sources/shared.go

@@ -12,16 +12,17 @@ type SyncSummary struct {
 	ClaimName string
 }
 
-func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.PublishOptions, namer *namer.Namer) (*SyncSummary, error) {
+func publishAndRetryExistingNames(daemon *jsonrpc.Client, title, filename string, amount float64, options jsonrpc.StreamCreateOptions, namer *namer.Namer) (*SyncSummary, error) {
 	for {
 		name := namer.GetNextName(title)
-		response, err := daemon.Publish(name, filename, amount, options)
+		response, err := daemon.StreamCreate(name, filename, amount, options)
 		if err != nil {
 			if strings.Contains(err.Error(), "failed: Multiple claims (") {
 				continue
 			}
 			return nil, err
 		}
-		return &SyncSummary{ClaimID: response.ClaimID, ClaimName: name}, nil
+		PublishedClaim := response.Outputs[0]
+		return &SyncSummary{ClaimID: PublishedClaim.ClaimID, ClaimName: name}, nil
 	}
 }

sources/ucbVideo.go

@@ -175,19 +175,19 @@ func (v *ucbVideo) saveThumbnail() error {
 }
 
 func (v *ucbVideo) publish(daemon *jsonrpc.Client, claimAddress string, amount float64, channelID string, namer *namer.Namer) (*SyncSummary, error) {
-	options := jsonrpc.PublishOptions{
-		Metadata: &jsonrpc.Metadata{
-			Title:       v.title,
-			Description: v.getAbbrevDescription(),
-			Author:      "UC Berkeley",
-			Language:    "en",
-			License:     "see description",
-			Thumbnail:   strPtr("https://berk.ninja/thumbnails/" + v.id),
-			NSFW:        false,
+	options := jsonrpc.StreamCreateOptions{
+		ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
+			Title:        v.title,
+			Description:  v.getAbbrevDescription(),
+			ClaimAddress: &claimAddress,
+			Languages:    []string{"en"},
+			ThumbnailURL: strPtr("https://berk.ninja/thumbnails/" + v.id),
+			Tags:         []string{},
 		},
-		ChannelID:     &channelID,
-		ClaimAddress:  &claimAddress,
-		ChangeAddress: &claimAddress,
+		Author:     strPtr("UC Berkeley"),
+		License:    strPtr("see description"),
+		StreamType: &jsonrpc.StreamTypeVideo,
+		ChannelID:  &channelID,
 	}
 	return publishAndRetryExistingNames(daemon, v.title, v.getFilename(), amount, options, namer)
 }

sources/youtubeVideo.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"io/ioutil"
+	"math"
 	"net/http"
 	"os"
 	"regexp"
@@ -13,8 +14,10 @@ import (
 
 	"github.com/lbryio/lbry.go/extras/errors"
 	"github.com/lbryio/lbry.go/extras/jsonrpc"
+	"github.com/lbryio/lbry.go/extras/util"
 	"github.com/lbryio/ytsync/namer"
 
+	"github.com/ChannelMeter/iso8601duration"
 	"github.com/nikooo777/ytdl"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/api/youtube/v3"
@@ -31,18 +34,21 @@ type YoutubeVideo struct {
 	maxVideoLength   float64
 	publishedAt      time.Time
 	dir              string
+	youtubeInfo      *youtube.Video
+	tags             []string
 }
 
-func NewYoutubeVideo(directory string, snippet *youtube.PlaylistItemSnippet) *YoutubeVideo {
-	publishedAt, _ := time.Parse(time.RFC3339Nano, snippet.PublishedAt) // ignore parse errors
+func NewYoutubeVideo(directory string, videoData *youtube.Video, playlistPosition int64) *YoutubeVideo {
+	publishedAt, _ := time.Parse(time.RFC3339Nano, videoData.Snippet.PublishedAt) // ignore parse errors
 	return &YoutubeVideo{
-		id:               snippet.ResourceId.VideoId,
-		title:            snippet.Title,
-		description:      snippet.Description,
-		channelTitle:     snippet.ChannelTitle,
-		playlistPosition: snippet.Position,
+		id:               videoData.Id,
+		title:            videoData.Snippet.Title,
+		description:      videoData.Snippet.Description,
+		channelTitle:     videoData.Snippet.ChannelTitle,
+		playlistPosition: playlistPosition,
 		publishedAt:      publishedAt,
 		dir:              directory,
+		youtubeInfo:      videoData,
 	}
 }
 
@@ -245,20 +251,33 @@ func (v *YoutubeVideo) publish(daemon *jsonrpc.Client, claimAddress string, amou
 	if channelID == khanAcademyClaimID {
 		additionalDescription = additionalDescription + "\nNote: All Khan Academy content is available for free at (www.khanacademy.org)"
 	}
-	options := jsonrpc.PublishOptions{
-		Metadata: &jsonrpc.Metadata{
-			Title:       v.title,
-			Description: v.getAbbrevDescription() + additionalDescription,
-			Author:      v.channelTitle,
-			Language:    "en",
-			License:     "Copyrighted (contact author)",
-			Thumbnail:   strPtr("https://berk.ninja/thumbnails/" + v.id),
-			NSFW:        false,
+	var languages []string = nil
+	if v.youtubeInfo.Snippet.DefaultLanguage != "" {
+		languages = []string{v.youtubeInfo.Snippet.DefaultLanguage}
+	}
+
+	videoDuration, err := duration.FromString(v.youtubeInfo.ContentDetails.Duration)
+
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+	options := jsonrpc.StreamCreateOptions{
+		ClaimCreateOptions: jsonrpc.ClaimCreateOptions{
+			Title:        v.title,
+			Description:  v.getAbbrevDescription() + additionalDescription,
+			ClaimAddress: &claimAddress,
+			Languages:    languages,
+			ThumbnailURL: strPtr("https://thumbnails.lbry.com/" + v.id),
+			Tags:         v.youtubeInfo.Snippet.Tags,
 		},
+		Author:        strPtr(v.channelTitle),
+		License:       strPtr("Copyrighted (contact author)"),
+		StreamType:    &jsonrpc.StreamTypeVideo,
+		ReleaseTime:   util.PtrToInt64(v.publishedAt.Unix()),
+		VideoDuration: util.PtrToUint64(uint64(math.Ceil(videoDuration.ToDuration().Seconds()))),
 		ChannelID:     &channelID,
-		ClaimAddress:  &claimAddress,
-		ChangeAddress: &claimAddress,
 	}
+
 	return publishAndRetryExistingNames(daemon, v.title, v.getFullPath(), amount, options, namer)
 }