From e5be3d852e6a65c35ba57ba1505d73ce9f85ad1d Mon Sep 17 00:00:00 2001 From: Sean <34941871+g8rswimmer@users.noreply.github.com> Date: Mon, 28 Dec 2020 19:19:46 -0700 Subject: [PATCH] fixed the tweet lookup expansions (#34) --- tweet.go | 173 +++++++++++++++++------- tweet_test.go | 363 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 486 insertions(+), 50 deletions(-) diff --git a/tweet.go b/tweet.go index c1b3572..751498f 100644 --- a/tweet.go +++ b/tweet.go @@ -25,86 +25,159 @@ const ( // TweetLookups is .a map of tweet lookups type TweetLookups map[string]TweetLookup -func (t TweetLookups) lookup(decoder *json.Decoder) error { - type include struct { - Media []*MediaObj `json:"medias"` - Place []*PlaceObj `json:"places"` - Poll []*PollObj `json:"polls"` - User []*UserObj `json:"users"` - } - type body struct { - Data TweetObj `json:"data"` - Include include `json:"includes"` +type tweetLookupIncludes struct { + Media []*MediaObj `json:"medias"` + Place []*PlaceObj `json:"places"` + Poll []*PollObj `json:"polls"` + User []*UserObj `json:"users"` + Tweet []*TweetObj `json:"tweets"` +} + +type tweetLookupMaps struct { + usersByID map[string]*UserObj + usersByName map[string]*UserObj + placeByID map[string]*PlaceObj + pollByID map[string]*PollObj + mediaByKey map[string]*MediaObj + tweetsByID map[string]*TweetObj +} + +func lookupMaps(include tweetLookupIncludes) tweetLookupMaps { + maps := tweetLookupMaps{} + + maps.usersByID = map[string]*UserObj{} + maps.usersByName = map[string]*UserObj{} + for _, user := range include.User { + maps.usersByID[user.ID] = user + maps.usersByName[user.UserName] = user } - b := &body{} - if err := decoder.Decode(b); err != nil { - return fmt.Errorf("tweet lookup decode error %w", err) + + maps.placeByID = map[string]*PlaceObj{} + for _, place := range include.Place { + maps.placeByID[place.ID] = place } - tl := TweetLookup{ - Tweet: b.Data, + maps.pollByID = map[string]*PollObj{} + for _, poll := range include.Poll { + maps.pollByID[poll.ID] = poll } - if len(b.Include.Media) > 0 { - tl.Media = b.Include.Media[0] + + maps.mediaByKey = map[string]*MediaObj{} + for _, media := range include.Media { + maps.mediaByKey[media.Key] = media } - if len(b.Include.Place) > 0 { - tl.Place = b.Include.Place[0] + + maps.tweetsByID = map[string]*TweetObj{} + for _, tweet := range include.Tweet { + maps.tweetsByID[tweet.ID] = tweet } - if len(b.Include.Poll) > 0 { - tl.Poll = b.Include.Poll[0] + + return maps +} + +func (t TweetLookups) lookup(decoder *json.Decoder) error { + type body struct { + Data TweetObj `json:"data"` + Include tweetLookupIncludes `json:"includes"` } - if len(b.Include.User) > 0 { - tl.User = b.Include.User[0] + b := &body{} + if err := decoder.Decode(b); err != nil { + return fmt.Errorf("tweet lookup decode error %w", err) } + + maps := lookupMaps(b.Include) + + tl := createTweetLookup(b.Data, maps) + t[b.Data.ID] = tl return nil } func (t TweetLookups) lookups(decoder *json.Decoder) error { - type include struct { - Media []*MediaObj `json:"medias"` - Place []*PlaceObj `json:"places"` - Poll []*PollObj `json:"polls"` - User []*UserObj `json:"users"` - } type body struct { - Data []TweetObj `json:"data"` - Include include `json:"includes"` + Data []TweetObj `json:"data"` + Include tweetLookupIncludes `json:"includes"` } b := &body{} if err := decoder.Decode(b); err != nil { return fmt.Errorf("tweet lookup decode error %w", err) } - for i, tweet := range b.Data { - tl := TweetLookup{ - Tweet: tweet, - } - if i < len(b.Include.Media) { - tl.Media = b.Include.Media[i] + maps := lookupMaps(b.Include) + + for _, tweet := range b.Data { + tl := createTweetLookup(tweet, maps) + + t[tweet.ID] = tl + } + return nil +} + +func createTweetLookup(tweet TweetObj, maps tweetLookupMaps) TweetLookup { + tl := TweetLookup{ + Tweet: tweet, + } + + if author, has := maps.usersByID[tweet.AuthorID]; has { + tl.User = author + } + + if inReply, has := maps.usersByID[tweet.InReplyToUserID]; has { + tl.InReplyUser = inReply + } + + if place, has := maps.placeByID[tweet.Geo.PlaceID]; has { + tl.Place = place + } + + mentions := []*UserObj{} + for _, entity := range tweet.Entities.Mentions { + if user, has := maps.usersByName[entity.UserName]; has { + mentions = append(mentions, user) } - if i < len(b.Include.Place) { - tl.Place = b.Include.Place[i] + } + tl.Mentions = mentions + + attachmentPolls := []*PollObj{} + for _, id := range tweet.Attachments.PollIDs { + if poll, has := maps.pollByID[id]; has { + attachmentPolls = append(attachmentPolls, poll) } - if i < len(b.Include.Poll) { - tl.Poll = b.Include.Poll[i] + } + tl.AttachmentPolls = attachmentPolls + + attachmentMedia := []*MediaObj{} + for _, key := range tweet.Attachments.MediaKeys { + if media, has := maps.mediaByKey[key]; has { + attachmentMedia = append(attachmentMedia, media) } - if i < len(b.Include.User) { - tl.User = b.Include.User[i] + } + tl.AttachmentMedia = attachmentMedia + + tweetReferences := []TweetLookup{} + for _, rt := range tweet.ReferencedTweets { + if t, has := maps.tweetsByID[rt.ID]; has { + tweetReferences = append(tweetReferences, createTweetLookup(*t, maps)) } - t[tweet.ID] = tl } - return nil + tl.ReferencedTweets = tweetReferences + + return tl } // TweetLookup is a complete tweet objects type TweetLookup struct { - Tweet TweetObj - Media *MediaObj - Place *PlaceObj - Poll *PollObj - User *UserObj + Tweet TweetObj + Media *MediaObj + Place *PlaceObj + Poll *PollObj + User *UserObj + InReplyUser *UserObj + Mentions []*UserObj + AttachmentPolls []*PollObj + AttachmentMedia []*MediaObj + ReferencedTweets []TweetLookup } // TweetRecentSearchMeta is the media data returned from the recent search diff --git a/tweet_test.go b/tweet_test.go index 01faeac..e45729d 100644 --- a/tweet_test.go +++ b/tweet_test.go @@ -85,6 +85,10 @@ func TestTweet_Lookup(t *testing.T) { ID: "2244994945", Name: "Twitter Dev", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, }, wantErr: false, @@ -159,6 +163,10 @@ func TestTweet_Lookup(t *testing.T) { ID: "2244994945", Name: "Twitter Dev", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1278347468690915330": TweetLookup{ Tweet: TweetObj{ @@ -173,6 +181,10 @@ func TestTweet_Lookup(t *testing.T) { ID: "783214", Name: "Twitter", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, }, wantErr: false, @@ -341,60 +353,100 @@ func TestTweet_RecentSearch(t *testing.T) { ID: "1279990139888918528", Text: "Python now online for you !!\n\nWith the advent and acceptance of AI, Robotics, Python has become an inevitable factor in software development industry and most looked out skill both Nationally and Internationally. \n\nCoupon code: GVUP9\nCall: 9482303905/9482163905 https://t.co/ZFXCDJedAh", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990133463429120": TweetLookup{ Tweet: TweetObj{ ID: "1279990133463429120", Text: "RT @McQubit: Building Neural Networks with Python Code and Math in Detail — II https://t.co/l6PKTTFGkv #machine_learning #programming #math…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990118355476480": TweetLookup{ Tweet: TweetObj{ ID: "1279990118355476480", Text: "RT @SunnyVaram: Top 10 Natural Language Processing Online Courses https://t.co/oAGqkHdS8H via @https://twitter.com/analyticsinme \n#DataScie…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990114584875009": TweetLookup{ Tweet: TweetObj{ ID: "1279990114584875009", Text: "RT @mohitnihalani7: LINK IN BIO......\n\n#programming #coding #programmer #developer #python #code #technology #coder #javascript #java #comp…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990108968665088": TweetLookup{ Tweet: TweetObj{ ID: "1279990108968665088", Text: "RT @mohitnihalani7: LINK IN BIO......\n\n#programming #coding #programmer #developer #python #code #technology #coder #javascript #java #comp…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990090828320769": TweetLookup{ Tweet: TweetObj{ ID: "1279990090828320769", Text: "RT @SunnyVaram: Top 10 Natural Language Processing Online Courses https://t.co/oAGqkHdS8H via @https://twitter.com/analyticsinme \n#DataScie…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990084398387201": TweetLookup{ Tweet: TweetObj{ ID: "1279990084398387201", Text: "RT @mohitnihalani7: LINK IN BIO......\n\n#programming #coding #programmer #developer #python #code #technology #coder #javascript #java #comp…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990076748038145": TweetLookup{ Tweet: TweetObj{ ID: "1279990076748038145", Text: "RT @gp_pulipaka: Best Machine Learning and Data Science #Books 2020. #BigData #Analytics #DataScience #IoT #IIoT #PyTorch #Python #RStats #…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990069105917952": TweetLookup{ Tweet: TweetObj{ ID: "1279990069105917952", Text: "RT @SunnyVaram: Top 10 Natural Language Processing Online Courses https://t.co/oAGqkHdS8H via @https://twitter.com/analyticsinme \n#DataScie…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, "1279990063888076800": TweetLookup{ Tweet: TweetObj{ ID: "1279990063888076800", Text: "RT @mohitnihalani7: LINK IN BIO......\n\n#programming #coding #programmer #developer #python #code #technology #coder #javascript #java #comp…", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, }, Meta: TweetRecentSearchMeta{ @@ -896,6 +948,10 @@ func TestTweet_SearchStream(t *testing.T) { Text: "Just getting started with Twitter APIs? Find out what you need in order to build an app. Watch this video! https://t.co/Hg8nkfoizN", ID: "1067094924124872705", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, }, wantErr: false, @@ -1011,6 +1067,10 @@ func TestTweet_SampledStream(t *testing.T) { Text: "Just getting started with Twitter APIs? Find out what you need in order to build an app. Watch this video! https://t.co/Hg8nkfoizN", ID: "1067094924124872705", }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, }, }, wantErr: false, @@ -1222,3 +1282,306 @@ func TestTweet_Hide(t *testing.T) { }) } } + +func Test_createTweetLookup(t *testing.T) { + type args struct { + tweet TweetObj + maps tweetLookupMaps + } + tests := []struct { + name string + args args + want TweetLookup + }{ + { + name: "success", + args: args{ + tweet: TweetObj{ + ID: "1261326399320715264", + Text: "Tune in to the @MongoDB @Twitch stream featuring our very own @suhemparack to learn about Twitter Developer Labs - starting now! https://t.co/fAWpYi3o5O", + Attachments: TweetAttachmentsObj{ + PollIDs: []string{"1199786642468413448"}, + MediaKeys: []string{"13_1263145212760805376"}, + }, + Geo: TweetGeoObj{ + PlaceID: "01a9a39529b27f36", + }, + AuthorID: "2244994945", + InReplyToUserID: "783214", + Entities: EntitiesObj{ + Mentions: []EntityMentionObj{ + { + EntityObj: EntityObj{ + Start: 15, + End: 23, + }, + UserName: "MongoDB", + }, + { + EntityObj: EntityObj{ + Start: 24, + End: 31, + }, + UserName: "Twitch", + }, + { + EntityObj: EntityObj{ + Start: 62, + End: 74, + }, + UserName: "suhemparack", + }, + }, + }, + ReferencedTweets: []TweetReferencedTweetObj{ + { + Type: "quoted", + ID: "1261091720801980419", + }, + }, + }, + maps: func() tweetLookupMaps { + includes := tweetLookupIncludes{ + User: []*UserObj{ + { + ID: "2244994945", + Name: "Twitter Dev", + UserName: "TwitterDev", + }, + { + Name: "Twitter", + ID: "783214", + UserName: "Twitter", + }, + { + Name: "MongoDB", + ID: "18080585", + UserName: "MongoDB", + }, + { + Name: "Twitch", + ID: "309366491", + UserName: "Twitch", + }, + { + Name: "Suhem Parack", + ID: "857699969263964161", + UserName: "suhemparack", + }, + }, + Poll: []*PollObj{ + { + ID: "1199786642468413448", + VotingStatus: "closed", + DurationMinutes: 1440, + Options: []PollOptionObj{ + { + Position: 1, + Label: "C Sharp", + Votes: 795, + }, + { + Position: 2, + Label: "C Hashtag", + Votes: 156, + }, + }, + EndDateTime: "2019-11-28T20:26:41.000Z", + }, + }, + Media: []*MediaObj{ + { + DurationMS: 46947, + Type: "video", + Height: 1080, + Key: "13_1263145212760805376", + PublicMetrics: MediaMetricsObj{ + Views: 6909260, + }, + PreviewImageURL: "https://pbs.twimg.com/media/EYeX7akWsAIP1_1.jpg", + Width: 1920, + }, + }, + Place: []*PlaceObj{ + { + Geo: PlaceGeoObj{ + Type: "Feature", + BBox: []float64{ + -74.026675, + 40.683935, + -73.910408, + 40.877483, + }, + Properties: map[string]interface{}{}, + }, + CountryCode: "US", + Name: "Manhattan", + ID: "01a9a39529b27f36", + PlaceType: "city", + Country: "United States", + FullName: "Manhattan, NY", + }, + }, + Tweet: []*TweetObj{ + { + ID: "1261091720801980419", + AuthorID: "18080585", + Text: "Tomorrow (May 15) at 12pm EST (9am PST, 6pm CET), join us for a Twitch stream with @KukicAdo from MongoDB and @suhemparack from @TwitterDev! \n\nLearn about the new Twitter Developer Labs and how to get the most out of the new API with MongoDB: https://t.co/YbrbVNJrPe https://t.co/Oe4bMVpPmh", + }, + }, + } + return lookupMaps(includes) + }(), + }, + want: TweetLookup{ + Tweet: TweetObj{ + ID: "1261326399320715264", + Text: "Tune in to the @MongoDB @Twitch stream featuring our very own @suhemparack to learn about Twitter Developer Labs - starting now! https://t.co/fAWpYi3o5O", + AuthorID: "2244994945", + InReplyToUserID: "783214", + Geo: TweetGeoObj{ + PlaceID: "01a9a39529b27f36", + }, + Entities: EntitiesObj{ + Mentions: []EntityMentionObj{ + { + EntityObj: EntityObj{ + Start: 15, + End: 23, + }, + UserName: "MongoDB", + }, + { + EntityObj: EntityObj{ + Start: 24, + End: 31, + }, + UserName: "Twitch", + }, + { + EntityObj: EntityObj{ + Start: 62, + End: 74, + }, + UserName: "suhemparack", + }, + }, + }, + Attachments: TweetAttachmentsObj{ + PollIDs: []string{"1199786642468413448"}, + MediaKeys: []string{"13_1263145212760805376"}, + }, + ReferencedTweets: []TweetReferencedTweetObj{ + { + Type: "quoted", + ID: "1261091720801980419", + }, + }, + }, + User: &UserObj{ + ID: "2244994945", + Name: "Twitter Dev", + UserName: "TwitterDev", + }, + InReplyUser: &UserObj{ + Name: "Twitter", + ID: "783214", + UserName: "Twitter", + }, + Place: &PlaceObj{ + Geo: PlaceGeoObj{ + Type: "Feature", + BBox: []float64{ + -74.026675, + 40.683935, + -73.910408, + 40.877483, + }, + Properties: map[string]interface{}{}, + }, + CountryCode: "US", + Name: "Manhattan", + ID: "01a9a39529b27f36", + PlaceType: "city", + Country: "United States", + FullName: "Manhattan, NY", + }, + Mentions: []*UserObj{ + { + Name: "MongoDB", + ID: "18080585", + UserName: "MongoDB", + }, + { + Name: "Twitch", + ID: "309366491", + UserName: "Twitch", + }, + { + Name: "Suhem Parack", + ID: "857699969263964161", + UserName: "suhemparack", + }, + }, + AttachmentPolls: []*PollObj{ + { + ID: "1199786642468413448", + VotingStatus: "closed", + DurationMinutes: 1440, + Options: []PollOptionObj{ + { + Position: 1, + Label: "C Sharp", + Votes: 795, + }, + { + Position: 2, + Label: "C Hashtag", + Votes: 156, + }, + }, + EndDateTime: "2019-11-28T20:26:41.000Z", + }, + }, + AttachmentMedia: []*MediaObj{ + { + DurationMS: 46947, + Type: "video", + Height: 1080, + Key: "13_1263145212760805376", + PublicMetrics: MediaMetricsObj{ + Views: 6909260, + }, + PreviewImageURL: "https://pbs.twimg.com/media/EYeX7akWsAIP1_1.jpg", + Width: 1920, + }, + }, + ReferencedTweets: []TweetLookup{ + { + Tweet: TweetObj{ + ID: "1261091720801980419", + AuthorID: "18080585", + Text: "Tomorrow (May 15) at 12pm EST (9am PST, 6pm CET), join us for a Twitch stream with @KukicAdo from MongoDB and @suhemparack from @TwitterDev! \n\nLearn about the new Twitter Developer Labs and how to get the most out of the new API with MongoDB: https://t.co/YbrbVNJrPe https://t.co/Oe4bMVpPmh", + }, + User: &UserObj{ + Name: "MongoDB", + ID: "18080585", + UserName: "MongoDB", + }, + Mentions: []*UserObj{}, + AttachmentPolls: []*PollObj{}, + AttachmentMedia: []*MediaObj{}, + ReferencedTweets: []TweetLookup{}, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := createTweetLookup(tt.args.tweet, tt.args.maps); !reflect.DeepEqual(got, tt.want) { + t.Errorf("createTweetLookup() = %v, want %v", got, tt.want) + } + }) + } +}