update

ai-to-ai · Nov 23, 2022 · e63e75a · e63e75a
1 parent 6f9a403
commit e63e75a
Show file tree

Hide file tree

Showing 41 changed files with 596 additions and 288 deletions.
diff --git a/.env b/.env
diff --git a/Raw DB Tasks/__pycache__/settings.cpython-310.pyc b/Raw DB Tasks/__pycache__/settings.cpython-310.pyc
diff --git a/Raw DB Tasks/empty collection.py b/Raw DB Tasks/empty collection.py
@@ -0,0 +1,19 @@
+import pymongo
+
+import settings
+
+MONGODB_USER = settings.mongodb_user
+MONGODB_PASSWORD = settings.mongodb_password
+MONGODB_CLUSTER = settings.mongodb_cluster
+
+MONGODB_URI = f"mongodb+srv://{MONGODB_USER}:{MONGODB_PASSWORD}@cluster0.{MONGODB_CLUSTER}.mongodb.net/?retryWrites=true&w=majority"
+myclient = pymongo.MongoClient(MONGODB_URI)
+mydb = myclient["tasks_db"]
+product = mydb["products"]
+price = mydb["productPriceHistory"]
+
+x = product.delete_many({})
+y = price.delete_many({})
+
+print(x.deleted_count, "documents deleted")
+print(y.deleted_count, "documents deleted")
diff --git a/settings.py → Raw DB Tasks/settings.py b/settings.py → Raw DB Tasks/settings.py
@@ -2,47 +2,57 @@
 
 import dotenv
 
+dotenv.load_dotenv()
+
 appId = 1
 # database details where products, categories, etc. are stored
-db_name = ""
-products_table = ""
-products_category_table = ""
-products_sellers_table = ""
-product_price_history_table = ""
-app_settings_table = ""
+db_name = "tasks_db"
+products_table = "products"
+products_category_table = "productCategory"
+products_sellers_table = "productSellers"
+product_price_history_table = "productPriceHistory"
+app_settings_table = "appSettings"
 
+mongodb_user = "david"
+mongodb_password = "YAFV68dBmBQhoNJs"
+mongodb_cluster = "uhrfxiy"
 
 # refer to https://www.scraperapi.com/documentation/python/#getting-started to learn more about the ScraperAPI
 # get proxy from ScraperAPI
 def get_proxy():
-    dotenv.load_dotenv()
     scraperAPI = os.getenv('SCRAPER_API_KEY')
     proxies = f"http://scraperapi.country_code=us.device_type=desktop:{scraperAPI}@proxy-server.scraperapi.com:8001 "
-
-
     return proxies
 
 
 # User agents to experiment with
 
-
-# HEADERS = {"User-Agent": "Mozilla/5.0",
-#            "Accept-Language": "en-US,en;q=0.9"}
-
-
-# HEADERS = {
-#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 '
-#                   'Safari/537.36 OPR/91.0.4516.95',
+# headers = {
+#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
+#                   'Chrome/71.0.3578.98 Safari/537.36',
 #     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 #     'Accept-Language': 'en-US,en;q=0.5',
 #     'Accept-Encoding': 'gzip',
-#     'Referer': 'https://www.google.com/',
 #     'DNT': '1',  # Do Not Track Request Header
 #     'Connection': 'close'
 # }
 
+# headers = {"User-Agent": "Mozilla/5.0",
+#            "Accept-Language": "en-US,en;q=0.9"}
+
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 '
+                  'Safari/537.36 OPR/91.0.4516.95',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Accept-Encoding': 'gzip',
+    'Referer': 'https://www.google.com/',
+    'DNT': '1',  # Do Not Track Request Header
+    'Connection': 'close'
+}
 
-# HEADERS = {
+# headers = {
 #     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
 #                   'Chrome/105.0.0.0 Safari/537.36 OPR/91.0.4516.95',
 #     'Accept-Language': 'en-US,en;q=0.9,es;q=0.8',
@@ -51,4 +61,4 @@ def get_proxy():
 #     'Accept-Encoding': 'gzip',
 #     'Referer': 'https://www.google.com/',
 #     'Upgrade-Insecure-Requests': '1'
-# }
+# }
diff --git a/extractors/extractors/__init__.py → __init__.py b/extractors/extractors/__init__.py → __init__.py
diff --git a/__pycache__/config.cpython-310.pyc b/__pycache__/config.cpython-310.pyc
diff --git a/__pycache__/settings.cpython-310.pyc b/__pycache__/settings.cpython-310.pyc
diff --git a/config.py b/config.py
@@ -0,0 +1,136 @@
+HOST = "127.0.0.1"
+PORT = 3000
+DEBUG = True
+
+SCRAPER_API_KEY = "ef991d16e237a6680f215c650206cf8e"
+
+APP_ID = 1
+# database details where products, categories, etc. are stored
+DB_NAME = "tasks_db"
+PRODUCT_COL = "products"
+PRODUCT_CATEGORY_COL = "productCategory"
+PRODUCT_SELLER_COL = "productSellers"
+PRODUCT_PRICE_HISTORY_COL = "productPriceHistory"
+APP_SETTING_COL = "appSettings"
+
+MONGODB_USER = "david"
+MONGODB_PASSWORD ="YAFV68dBmBQhoNJs"
+MONGODB_CLUSTER = "uhrfxiy"
+
+
+# refer to https://www.scraperapi.com/documentation/python/#getting-started to learn more about the ScraperAPI
+# get proxy from ScraperAPI
+PROXY = f"http://scraperapi.country_code=us.device_type=desktop:{SCRAPER_API_KEY}@proxy-server.scraperapi.com:8001 "
+
+
+# User agents to experiment with
+
+# headers = {
+#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
+#                   'Chrome/71.0.3578.98 Safari/537.36',
+#     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#     'Accept-Language': 'en-US,en;q=0.5',
+#     'Accept-Encoding': 'gzip',
+#     'DNT': '1',  # Do Not Track Request Header
+#     'Connection': 'close'
+# }
+
+# headers = {"User-Agent": "Mozilla/5.0",
+#            "Accept-Language": "en-US,en;q=0.9"}
+
+
+# headers = {
+#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 '
+#                   'Safari/537.36 OPR/91.0.4516.95',
+#     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#     'Accept-Language': 'en-US,en;q=0.5',
+#     'Accept-Encoding': 'gzip',
+#     'Referer': 'https://www.google.com/',
+#     'DNT': '1',  # Do Not Track Request Header
+#     'Connection': 'close'
+# }
+
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
+                  'Chrome/105.0.0.0 Safari/537.36 OPR/91.0.4516.95',
+    'Accept-Language': 'en-US,en;q=0.9,es;q=0.8',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,'
+              'application/signed-exchange;v=b3',
+    'Accept-Encoding': 'gzip',
+    'Referer': 'https://www.google.com/',
+    'Upgrade-Insecure-Requests': '1'
+}
+
+UAGENTS = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.52',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 YaBrowser/21.8.1.468 Yowser/2.5 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64; rv:105.0) Gecko/20100101 Firefox/105.0',
+    'Mozilla/5.0 (X11; CrOS x86_64 14440.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4807.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14467.0.2022) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4838.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14469.7.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.13 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14455.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4827.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14469.11.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.17 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14436.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4803.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14475.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4840.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14469.3.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.9 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14471.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4840.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14388.37.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.9 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14409.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4829.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14395.0.2021) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4765.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14469.8.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.14 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14484.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4840.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14450.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4817.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14473.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4840.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14324.72.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.91 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14454.0.2022) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4824.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14453.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4816.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14447.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4815.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14477.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4840.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14476.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4840.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14469.8.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.9 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.67.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.67.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.69.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.82 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14695.25.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.22 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.89.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.133 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.57.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.64 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.89.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.133 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.84.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.93 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14469.59.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.91.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.55 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14695.23.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.20 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14695.36.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.36 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.41.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.26 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14695.11.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.6 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.67.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14685.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.4992.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.69.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.82 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14682.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.16 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14695.9.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.5 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14574.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4937.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14388.52.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14716.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5002.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14268.81.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.88 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14469.41.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.48 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14388.61.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14695.37.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.37 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.51.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.32 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.89.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.133 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.92.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.56 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.43.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.54 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14505.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4870.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.16.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.25 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.28.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.44 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14543.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4918.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.11.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.6 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.89.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.133 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14588.31.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.19 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14526.6.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.13 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14658.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.4975.0 Safari/537.36',
+    'Mozilla/5.0 (X11; CrOS x86_64 14695.25.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5002.0 Safari/537.36'
+]
diff --git a/extractors/__init__.py b/extractors/__init__.py
diff --git a/extractors/__pycache__/__init__.cpython-310.pyc b/extractors/__pycache__/__init__.cpython-310.pyc
diff --git a/extractors/__pycache__/items.cpython-310.pyc b/extractors/__pycache__/items.cpython-310.pyc
diff --git a/extractors/__pycache__/middlewares.cpython-310.pyc b/extractors/__pycache__/middlewares.cpython-310.pyc
diff --git a/extractors/__pycache__/pipelines.cpython-310.pyc b/extractors/__pycache__/pipelines.cpython-310.pyc
diff --git a/extractors/__pycache__/settings.cpython-310.pyc b/extractors/__pycache__/settings.cpython-310.pyc
diff --git a/extractors/__pycache__/utils.cpython-310.pyc b/extractors/__pycache__/utils.cpython-310.pyc
diff --git a/extractors/extractors/__pycache__/__init__.cpython-310.pyc b/extractors/extractors/__pycache__/__init__.cpython-310.pyc
diff --git a/extractors/extractors/__pycache__/items.cpython-310.pyc b/extractors/extractors/__pycache__/items.cpython-310.pyc
diff --git a/extractors/extractors/__pycache__/middlewares.cpython-310.pyc b/extractors/extractors/__pycache__/middlewares.cpython-310.pyc
diff --git a/extractors/extractors/__pycache__/pipelines.cpython-310.pyc b/extractors/extractors/__pycache__/pipelines.cpython-310.pyc
diff --git a/extractors/extractors/__pycache__/settings.cpython-310.pyc b/extractors/extractors/__pycache__/settings.cpython-310.pyc
diff --git a/extractors/extractors/__pycache__/utils.cpython-310.pyc b/extractors/extractors/__pycache__/utils.cpython-310.pyc
diff --git a/extractors/extractors/spiders/__pycache__/amazon_find_spider.cpython-310.pyc b/extractors/extractors/spiders/__pycache__/amazon_find_spider.cpython-310.pyc
diff --git a/extractors/extractors/spiders/amazon_find_spider.py b/extractors/extractors/spiders/amazon_find_spider.py