Merge branch 'dev'

Docs/Changelogs.md

@@ -1,3 +1,18 @@
+# 1.8.0
+
+**General:**
+
+- **(Issue #11)** FF.net extractor ought to be able to download pages once more. f-dl now uses [cloudscraper](https://pypi.org/project/cloudscraper/) to bypass bot protection provided by Cloudflare.
+
+**Extractors:**
+
+- Added authentication support to the AO3 extractor.
+- Reworked authentication using the XenForo extractor.
+
+**Stuff:**
+
+- Provided username and password are now remembered during the session by all the extractors using this method of authentication.
+
 # 1.7.0
 
 **General:**

README.md

@@ -1,4 +1,4 @@
-# ![Logo](/Stuff/Logo%20(64).png?raw=true) [fiction-dl](https://github.com/DreamCobbler/fiction-dl) (*1.7.0*)
+# ![Logo](/Stuff/Logo%20(64).png?raw=true) [fiction-dl](https://github.com/DreamCobbler/fiction-dl) (*1.8.0*)
 
 ![Screenshot (Windows 10)](/Stuff/Screenshot%20(Windows%2010).png?raw=true)
 ![Screenshot (Linux Mint 20)](/Stuff/Screenshot%20(Linux%20Mint%2020).png?raw=true)
@@ -25,7 +25,7 @@ In a select few sites, it allows you to log-in to the site using the command-lin
 
 | \#   | Category         | Site                                                                | Authentication support | Channel support                |
 |------|------------------|---------------------------------------------------------------------|------------------------|--------------------------------|
-| *1*  | fanfiction       | [Archive of Our Own](https://archiveofourown.org/)                  | ✗ no                   | ✓ **yes <sup>[1]</sup>**       |
+| *1*  | fanfiction       | [Archive of Our Own](https://archiveofourown.org/)                  | ✓ **yes**              | ✓ **yes <sup>[1]</sup>**       |
 | *2*  | fanfiction       | [FanFiction.net](https://www.fanfiction.net/)                       | ✗ no                   | ✓ **yes <sup>[2]</sup>**       |
 | *3*  | fanfiction       | [FicWad.com](https://ficwad.com/)                                   | ✗ no                   | ✓ **yes**                      |
 | *4*  | fanfiction       | [SpaceBattles.com](https://forums.spacebattles.com/)                | ✓ **yes**              | ✗ no                           |

Tests/Integration Test Dataset 2.txt

@@ -188,10 +188,6 @@ https://www.nifty.org/nifty/lesbian/adult-friends/sofie
 #
 ##
 
-# A single-chapter story.
-
-https://www.quotev.com/story/13241547/Rain-on-fire
-
 # A multi-chapter story.
 
 https://www.quotev.com/story/13160599/Bloodlines

Tests/Integration Test.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

fiction_dl/Concepts/Chapter.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

fiction_dl/Concepts/Extractor.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -30,17 +30,22 @@
 
 from fiction_dl.Concepts.Chapter import Chapter
 from fiction_dl.Concepts.Story import Story
+import fiction_dl.Configuration as Configuration
 
 # Standard packages.
 
+from enum import Enum
 import logging
 import requests
 from typing import List, Optional
 
 # Non-standard packages.
 
 from bs4 import BeautifulSoup
-from dreamy_utilities.Web import DownloadSoup, GetHostname
+from dreamy_utilities.Interface import Interface
+from dreamy_utilities.Web import GetHostname
+from dreamy_utilities.WebSession import WebSession
+from fake_useragent import UserAgent
 
 #
 #
@@ -59,6 +64,17 @@
 
 class Extractor:
 
+    ##
+    #
+    # Represents authentication result.
+    #
+    ##
+
+    class AuthenticationResult(Enum):
+        SUCCESS   = 1
+        FAILURE   = 2
+        ABANDONED = 3
+
     def __init__(self) -> None:
 
         ##
@@ -69,7 +85,7 @@ def __init__(self) -> None:
 
         self.Story = None
 
-        self._session = requests.session()
+        self._webSession = WebSession(UserAgent().random)
         self._chapterURLs = []
 
         self._downloadStorySoupWhenScanning = True
@@ -113,20 +129,19 @@ def SupportsAuthentication(self) -> bool:
 
         return False
 
-    def Authenticate(self) -> bool:
+    def Authenticate(self, interface: Interface) -> AuthenticationResult:
 
         ##
         #
         # Logs the user in, interactively.
         #
-        # @param username The username.
-        # @param password The password.
+        # @param interface The user interface to be used.
         #
-        # @return **True** if the user has been authenticated correctly, **False** otherwise.
+        # @return The result of the authentication attempt.
         #
         ##
 
-        return False
+        return AuthenticationResult.FAILURE
 
     def Initialize(self, URL: str) -> bool:
 
@@ -180,9 +195,10 @@ def ScanStory(self) -> bool:
 
         if self._downloadStorySoupWhenScanning:
 
-            soup = DownloadSoup(normalizedURL, self._session)
+            soup = self._webSession.GetSoup(normalizedURL)
+
             if not soup:
-                logging.error(f'Failed to download page: "{normalizedURL}".')
+                logging.error(f'Failed to download tag soup: "{normalizedURL}".')
                 return False
 
         return self._InternallyScanStory(normalizedURL, soup)
@@ -213,9 +229,9 @@ def ExtractChapter(self, index: int) -> Optional[Chapter]:
 
         if self._downloadChapterSoupWhenExtracting:
 
-            soup = DownloadSoup(chapterURL, self._session, parser = self._chapterParserName)
+            soup = self._webSession.GetSoup(chapterURL, self._chapterParserName)
             if not soup:
-                logging.error(f'Failed to download page: "{chapterURL}".')
+                logging.error(f'Failed to download tag soup: "{chapterURL}".')
                 return None
 
         return self._InternallyExtractChapter(chapterURL, soup)
@@ -235,11 +251,7 @@ def ExtractMedia(self, URL: str) -> Optional[bytes]:
         if not URL:
             return None
 
-        response = self._session.get(URL, stream = True)
-        if not response.content:
-            return None
-
-        return response.content
+        return self._webSession.Get(URL, text = False, stream = True)
 
     def _InternallyScanStory(
         self,

fiction_dl/Concepts/Formatter.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

fiction_dl/Concepts/Image.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -218,7 +218,7 @@ def CreateImageFromDataUsingPIL(data: bytes, side: Optional[int] = None, quality
     if side and (scale := side / max(width, height)) < 1:
         image = image.resize(
             (int(scale * width), int(scale * height)),
-            Image.ANTIALIAS
+            PIL.Image.ANTIALIAS
         )
 
     # Encode image data and store it.

fiction_dl/Concepts/Metadata.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

fiction_dl/Concepts/Processor.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

fiction_dl/Concepts/Story.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -31,7 +31,7 @@
 from fiction_dl.Concepts.Image import Image
 from fiction_dl.Concepts.Metadata import Metadata
 from fiction_dl.Utilities.HTML import StripHTML
-import fiction_dl.Configuration
+import fiction_dl.Configuration as Configuration
 
 # Standard packages.
 
@@ -89,7 +89,7 @@ def FillTemplate(self, template: str, escapeHTMLEntities: bool = False) -> str:
         #
         ##
 
-        template = FillTemplate(fiction_dl.Configuration, template)
+        template = FillTemplate(Configuration, template)
         template = FillTemplate(self.Metadata.GetPrettified(escapeHTMLEntities), template)
 
         return template

fiction_dl/Concepts/StoryPackage.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

fiction_dl/Configuration.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -40,7 +40,7 @@
 #
 
 ApplicationName = "fiction-dl"
-ApplicationVersion = "1.7.0"
+ApplicationVersion = "1.8.0"
 ApplicationShortDescription = (
     "A content downloader, capable of retrieving works of (fan)fiction from the web and saving them in a few common file"
     " formats."
@@ -50,8 +50,6 @@
 CreatorName = "Benedykt Synakiewicz"
 CreatorContact = "[email protected]"
 
-UserAgent = f"{ApplicationName} {ApplicationVersion}"
-
 RedditClientID = "ScszEQn1cI7GgQ"
 RedditRedirectURI = "http://localhost:8080"
 

fiction_dl/Core/Application.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -29,6 +29,7 @@
 # Application.
 
 from fiction_dl.Concepts.Chapter import Chapter
+from fiction_dl.Concepts.Extractor import Extractor
 from fiction_dl.Concepts.Story import Story
 from fiction_dl.Concepts.StoryPackage import StoryPackage
 from fiction_dl.Core.Cache import Cache
@@ -62,6 +63,7 @@
 
 # Non-standard packages.
 
+from cloudscraper.exceptions import CloudflareChallengeError
 from dreamy_utilities.Containers import RemoveDuplicates
 from dreamy_utilities.Filesystem import FindExecutable, GetSanitizedFileName, WriteTextFile
 from dreamy_utilities.Interface import Interface
@@ -175,6 +177,11 @@ def Launch(self) -> None:
                     self._interface.Error(f"An SSL error has occurred: {caughtException}")
                     self._interface.GrabUserAttention()
 
+                except CloudflareChallengeError as caughtException:
+
+                    self._interface.Error("A Cloudflare challenge error has occurred. Try again later.")
+                    self._interface.GrabUserAttention()
+
                 except BaseException as caughtException:
 
                     self._interface.Error(f"An exception has been thrown: {caughtException}")
@@ -281,11 +288,14 @@ def _ProcessURL(self, URL: str) -> Optional[Story]:
 
         if self._arguments.Authenticate and extractor.SupportsAuthentication():
 
-            self._interface.GrabUserAttention()
+            self._interface.Process("Logging-in...", section = True)
 
-            if not extractor.Authenticate():
-                self._interface.Error("Failed to authenticate.")
+            authenticationResult = extractor.Authenticate(self._interface)
 
+            if Extractor.AuthenticationResult.FAILURE == authenticationResult:
+                self._interface.Error("Failed to authenticate.")
+            elif Extractor.AuthenticationResult.ABANDONED == authenticationResult:
+                self._interface.Comment("Proceeding without logging-in...")
             else:
                 self._interface.Comment("Authenticated successfully.")
 
@@ -397,7 +407,7 @@ def _ProcessURL(self, URL: str) -> Optional[Story]:
                 imageCount = len(extractor.Story.Images)
                 downloadedImageCount = 0
 
-                previousImageFailedToDownlod = False
+                previousImageFailedToDownload = False
 
                 for index, image in enumerate(extractor.Story.Images, start = 1):
 

fiction_dl/Core/Cache.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

fiction_dl/Core/InputData.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

fiction_dl/Extractors/ExtractorAH.py

@@ -1,7 +1,7 @@
 ####
 #
 # fiction-dl
-# Copyright (C) (2020) Benedykt Synakiewicz <[email protected]>
+# Copyright (C) (2020 - 2021) Benedykt Synakiewicz <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by