Fix issue #130 Tumblr Search Offline

- If the privacy consent for Tumblr Search was missing, the redirect didn't work properly. It showed the blog as offline and didn't crawl. - The redirect code is corrected to work with absolute and relative redirects. If the privacy consent is missing an error bar is shown.
TumblThreeApp · Mar 19, 2021 · a766571 · a766571
1 parent d782997
commit a766571
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 11 deletions.
diff --git a/src/TumblThree/TumblThree.Applications/Controllers/CrawlerController.cs b/src/TumblThree/TumblThree.Applications/Controllers/CrawlerController.cs
@@ -206,20 +206,24 @@ private async Task RunCrawlerTasksAsync(PauseToken pt, CancellationToken ct)
                         }
                         IBlog blog = nextQueueItem.Blog;
 
+                        var privacyConsentNeeded = false;
                         ICrawler crawler = _crawlerFactory.GetCrawler(blog, new Progress<DownloadProgress>(), pt, ct);
-                        crawler.IsBlogOnlineAsync().Wait(4000);
-                        crawler.Dispose();
-
-                        if (_crawlerService.ActiveItems.Any(item =>
-                            item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
-                            item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
+                        try
                         {
-                            QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
-                            Monitor.Exit(_lockObject);
-                            continue;
+                            crawler.IsBlogOnlineAsync().Wait(4000);
+                        }
+                        catch (AggregateException ex)
+                        {
+                            if (ex.InnerExceptions.Any(x => x.Message == "Acceptance of privacy consent needed!"))
+                                privacyConsentNeeded = true;
                         }
+                        crawler.Dispose();
 
-                        if (!nextQueueItem.Blog.Online)
+                        if (privacyConsentNeeded
+                            || (_crawlerService.ActiveItems.Any(item =>
+                                item.Blog.Name.Equals(nextQueueItem.Blog.Name) &&
+                                item.Blog.BlogType.Equals(nextQueueItem.Blog.BlogType)))
+                            || (!nextQueueItem.Blog.Online))
                         {
                             QueueOnDispatcher.CheckBeginInvokeOnUI(() => QueueManager.RemoveItem(nextQueueItem));
                             Monitor.Exit(_lockObject);

diff --git a/src/TumblThree/TumblThree.Applications/Crawler/AbstractCrawler.cs b/src/TumblThree/TumblThree.Applications/Crawler/AbstractCrawler.cs
@@ -128,7 +128,17 @@ protected async Task<string> RequestDataAsync(string url, Dictionary<string, str
                     requestRegistration = Ct.Register(() => request.Abort());
                     responseDetails = await WebRequestFactory.ReadRequestToEnd2Async(request);
                     if (responseDetails.HttpStatusCode == HttpStatusCode.Found)
-                        url = request.RequestUri.GetLeftPart(UriPartial.Authority) + responseDetails.RedirectUrl;
+                    {
+                        url = responseDetails.RedirectUrl;
+                        if (url.Contains("privacy/consent"))
+                        {
+                            var ex = new Exception("Acceptance of privacy consent needed!");
+                            ShellService.ShowError(ex, Resources.ConfirmationTumblrPrivacyConsentNeeded);
+                            throw ex;
+                        }
+                        if (!url.StartsWith("http", StringComparison.InvariantCultureIgnoreCase))
+                            url = request.RequestUri.GetLeftPart(UriPartial.Authority) + url;
+                    }
 
                 } while (responseDetails.HttpStatusCode == HttpStatusCode.Found && redirects++ < 5);
 

diff --git a/src/TumblThree/TumblThree.Applications/Properties/Resources.Designer.cs b/src/TumblThree/TumblThree.Applications/Properties/Resources.Designer.cs
diff --git a/src/TumblThree/TumblThree.Applications/Properties/Resources.resx b/src/TumblThree/TumblThree.Applications/Properties/Resources.resx
@@ -383,4 +383,7 @@
   <data name="DownloadNewVersionTitle" xml:space="preserve">
     <value>New version available</value>
   </data>
+  <data name="ConfirmationTumblrPrivacyConsentNeeded" xml:space="preserve">
+    <value>Tumblr Privacy Consent needs to be accepted in the settings!</value>
+  </data>
 </root>