scrape-ecommerce.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Scrape e-commerce data search</h1>\n",
    "<br>\n",
    "<h4>by : @qywok_exploiter</h4>\n",
    "<hr>\n",
    "<h2>e-commerce</h2>\n",
    "<h3> -> Tokopedia</h3>\n",
    "<h3> -> Shopee</h3>\n",
    "<h3> -> Bukalapak</h3>\n",
    "<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<img src=\"https://media.tenor.com/images/33c8c26810ae39857de47b663463cd66/tenor.gif\"/>"
      ],
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#BERCANDA\n",
    "from IPython.display import Image\n",
    "from IPython.core.display import HTML \n",
    "Image(url= \"https://media.tenor.com/images/33c8c26810ae39857de47b663463cd66/tenor.gif\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import json\n",
    "import urllib\n",
    "import pandas as pd\n",
    "import cloudscraper\n",
    "import requests as req\n",
    "from bs4 import BeautifulSoup\n",
    "from requests.adapters import HTTPAdapter\n",
    "from requests.packages.urllib3.util.retry import Retry"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "**[ 1 ] TOKOPEDIA**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tokopedia:\n",
    "    \n",
    "    def __init__(self, Search = \"ban bekas\") :\n",
    "        \n",
    "        self.PARAMS   = f'device=mobile&ob=23&page=1&q={Search}&related=true&rows=999999999&safe_search=false&scheme=https&shipping=&source=search&st=product&start=0&topads_bucket=true&unique_id=504d5a929adeabc5d177d7473394f0f7&user_addressId=&user_cityId=&user_districtId=&user_id=&user_lat=&user_long=&user_postCode=&variants='\n",
    "        self.ENDPOINT = \"https://gql.tokopedia.com/\"\n",
    "        self.PAYLOAD  =  {\n",
    "            \n",
    "            'operationName': 'SearchProductQueryV4',\n",
    "            \n",
    "            'variables': \n",
    "                {'params': \n",
    "                 \n",
    "                     self.PARAMS\n",
    "                 \n",
    "                },\n",
    "            \n",
    "             'query': 'query SearchProductQueryV4($params: String!) {\\n  ace_search_product_v4(params: $params) {\\n    header {\\n      totalData\\n      totalDataText\\n      processTime\\n      responseCode\\n      errorMessage\\n      additionalParams\\n      keywordProcess\\n      __typename\\n    }\\n    data {\\n      isQuerySafe\\n      ticker {\\n        text\\n        query\\n        typeId\\n        __typename\\n      }\\n      redirection {\\n        redirectUrl\\n        departmentId\\n        __typename\\n      }\\n      related {\\n        relatedKeyword\\n        otherRelated {\\n          keyword\\n          url\\n          product {\\n            id\\n            name\\n            price\\n            imageUrl\\n            rating\\n            countReview\\n            url\\n            priceStr\\n            wishlist\\n            shop {\\n              city\\n              isOfficial\\n              isPowerBadge\\n              __typename\\n            }\\n            ads {\\n              adsId: id\\n              productClickUrl\\n              productWishlistUrl\\n              shopClickUrl\\n              productViewUrl\\n              __typename\\n            }\\n            badges {\\n              title\\n              imageUrl\\n              show\\n              __typename\\n            }\\n            ratingAverage\\n            labelGroups {\\n              position\\n              type\\n              title\\n              __typename\\n            }\\n            __typename\\n          }\\n          __typename\\n        }\\n        __typename\\n      }\\n      suggestion {\\n        currentKeyword\\n        suggestion\\n        suggestionCount\\n        instead\\n        insteadCount\\n        query\\n        text\\n        __typename\\n      }\\n      products {\\n        id\\n        name\\n        ads {\\n          adsId: id\\n          productClickUrl\\n          productWishlistUrl\\n          productViewUrl\\n          __typename\\n        }\\n        badges {\\n          title\\n          imageUrl\\n          show\\n          __typename\\n        }\\n        category: departmentId\\n        categoryBreadcrumb\\n        categoryId\\n        categoryName\\n        countReview\\n        discountPercentage\\n        gaKey\\n        imageUrl\\n        labelGroups {\\n          position\\n          title\\n          type\\n          __typename\\n        }\\n        originalPrice\\n        price\\n        priceRange\\n        rating\\n        ratingAverage\\n        shop {\\n          id\\n          name\\n          url\\n          city\\n          isOfficial\\n          isPowerBadge\\n          __typename\\n        }\\n        url\\n        wishlist\\n        sourceEngine: source_engine\\n        __typename\\n      }\\n      __typename\\n    }\\n    __typename\\n  }\\n}\\n'}\n",
    "        \n",
    "    def getTokopedia(self):\n",
    "        \n",
    "        sorting          = []\n",
    "        request_products = json.loads(req.post(self.ENDPOINT, json = self.PAYLOAD).content)\n",
    "        \n",
    "        def price(value):\n",
    "            \n",
    "            Price = \"\"\n",
    "            \n",
    "            for indexPrice in range(len(value)):\n",
    "                \n",
    "                Index = format(ord(value[indexPrice]), \"x\")\n",
    "                if (Index == \"52\")|(Index == \"70\")|(Index == \"2e\") :\n",
    "                    \n",
    "                    pass\n",
    "                \n",
    "                else:\n",
    "                    Price+=value[indexPrice]\n",
    "            \n",
    "            \n",
    "            return int(Price)\n",
    "            \n",
    "        \n",
    "        for indexProducts in range(len(request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"])) :\n",
    "            \n",
    "            Product = {\n",
    "                \n",
    "                \"nama_produk\" : request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"name\"],\n",
    "                \"harga\"       : price(request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"price\"]),\n",
    "                \"diskon\"      : request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"discountPercentage\"],\n",
    "                \"rating\"      : float(request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"ratingAverage\"]),\n",
    "                \"foto_produk\" : request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"imageUrl\"],\n",
    "                \"url_produk\"  : request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"url\"],\n",
    "                \"nama_toko\"   : request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"shop\"][\"name\"],\n",
    "                \"lokasi_toko\" : request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"shop\"][\"city\"],\n",
    "                \"url_toko\"    : request_products[\"data\"][\"ace_search_product_v4\"][\"data\"][\"products\"][indexProducts][\"shop\"][\"url\"],\n",
    "                \"asal_ecommerce\" : \"tokopedia\"\n",
    "                \n",
    "                \n",
    "            }\n",
    "            \n",
    "            sorting.append(\n",
    "            \n",
    "                Product\n",
    "            \n",
    "            )\n",
    "            \n",
    "#         return pd.DataFrame(sorting).sort_values(by=[\"harga\",\"rating\",\"diskon\"])\n",
    "        return sorting\n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "**[ 2 ] SHOPEE**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Shopee:\n",
    "    \n",
    "    def __init__(self, Search = \"ban\"):\n",
    "        \n",
    "        self.ENDPOINT = f\"https://shopee.co.id/api/v4/search/search_items?by=relevancy&keyword={Search}&limit=100&newest=100&order=desc&page_type=search&scenario=PAGE_GLOBAL_SEARCH&version=2\"\n",
    "        self.IMAGE_ENDPOINT = \"https://cf.shopee.co.id/file/\"\n",
    "        self.SHOP = \"https://shopee.co.id/api/v4/product/get_shop_info?shopid=\"\n",
    "        \n",
    "        \n",
    "    def getShopee(self) :\n",
    "        \n",
    "        sorting          = []\n",
    "        request_products = json.loads(req.get(self.ENDPOINT).content)\n",
    "        \n",
    "        for indexProducts in range(len(request_products[\"items\"])) :\n",
    "            \n",
    "            price        = str(request_products[\"items\"][indexProducts][\"item_basic\"][\"price\"])\n",
    "            discount     = str(request_products[\"items\"][indexProducts][\"item_basic\"][\"discount\"])\n",
    "            rating       = str(request_products[\"items\"][indexProducts][\"item_basic\"][\"item_rating\"][\"rating_star\"])\n",
    "            requestShop  = json.loads(req.get(\"https://shopee.co.id/api/v4/product/get_shop_info?shopid=\"+str(request_products[\"items\"][indexProducts][\"item_basic\"][\"shopid\"])).content)\n",
    "            \n",
    "            def urlProduk():\n",
    "                \n",
    "                endpoint = \"https://shopee.co.id/-\"\n",
    "                Name     = request_products[\"items\"][indexProducts][\"item_basic\"][\"name\"]\n",
    "                result   = \"\"\n",
    "                shopid   = str(request_products[\"items\"][indexProducts][\"item_basic\"][\"shopid\"])\n",
    "                itemid   = str(request_products[\"items\"][indexProducts][\"item_basic\"][\"itemid\"])\n",
    "                for indexResult in range(len(Name)):\n",
    "                    if format(ord(Name[indexResult]), \"x\") == \"20\":\n",
    "                        result+=\"-\"\n",
    "                    else:\n",
    "                        result+=Name[indexResult]\n",
    "                return f\"{endpoint}{result}-i.{shopid}.{itemid}\"\n",
    "            \n",
    "            \n",
    "            Product = {\n",
    "                \n",
    "                \"nama_produk\" : request_products[\"items\"][indexProducts][\"item_basic\"][\"name\"],\n",
    "                \"harga\"       : int(price[0:len(price)-5]),\n",
    "                \"diskon\"      : int(discount[0:len(discount)-1]) if type(discount[0:len(discount)-1]) == \"int\" else discount[0:len(discount)-1]  ,\n",
    "                \"rating\"      : float(rating[0:3]),\n",
    "                \"foto_produk\" : self.IMAGE_ENDPOINT+request_products[\"items\"][indexProducts][\"item_basic\"][\"image\"],\n",
    "                \"url_produk\"  : urlProduk(),\n",
    "                \"nama_toko\"   : requestShop[\"data\"][\"account\"][\"username\"],\n",
    "                \"lokasi_toko\" : requestShop[\"data\"][\"shop_location\"],\n",
    "                \"url_toko\"    : \"https://shopee.co.id/\"+requestShop[\"data\"][\"account\"][\"username\"],\n",
    "                \"asal_ecommerce\" : \"shopee\"\n",
    "                \n",
    "                \n",
    "            }\n",
    "            \n",
    "            sorting.append(\n",
    "            \n",
    "                Product\n",
    "            \n",
    "            )\n",
    "            \n",
    "        return sorting\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "**[ 3 ] Bukalapak**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Bukalapak:\n",
    "    \n",
    "    def __init__(self, Search):\n",
    "        \n",
    "        self.ENDPOINT         = f\"https://api.bukalapak.com/multistrategy-products?prambanan_override=true&keywords={Search}&limit=50&offset=50&facet=true&page=1&access_token=\"\n",
    "        self.TOKEN_ENDPOINT  = f\"http://www.bukalapak.com/404?marketplace_path=%2Fu%2Fwesteros_auth_proxies\"\n",
    "        \n",
    "        self.cookies_token = {\n",
    "                \n",
    "                \"__asc\" : \"a3e0eb9e17a03a6e7a5988fed4f\",\n",
    "                \"__auc\": \"19928746177874debc7bca2450d\",\n",
    "                \"__cfruid\" : \"9d80714daf3fb1ed5e80d7e10123c28d8ec630cc-1623556183\",\n",
    "                \"_dd_s\":\"logs=1&id=c00b0782-a851-4b87-84b3-6167846a3936&created=1623556188705&expire=1623560339953\",\n",
    "                \"_fbp\":\"fb.1.1623524287461.1658652966\",\n",
    "                \"_ga\":\"GA1.2.864919285.1623524285\",\n",
    "                \"_gcl_au\":\"1.1.934570982.1623524285\",\n",
    "                \"_gid\":\"GA1.2.293862253.1623524285\",\n",
    "                \"_hjAbsoluteSessionInProgress\":\"1\",\n",
    "                \"_hjid\":\"ee5bbddd-8300-48a8-9a0b-b3d71f96a07c\",\n",
    "                \"_hjIncludedInSessionSample\":\"1\",\n",
    "                \"_hjTLDTest\":\"1\",\n",
    "                \"_td\":\"477254b3-df17-44f4-adc6-feb14c354d9c\",\n",
    "                \"_vwo_uuid_v2\":\"DB279B8E19F57F720EEA02225A0A044EF|32ed062cbc20a26c644b5144a3694c5f\",\n",
    "                \"ab.storage.deviceId.a987eba1-624b-42ab-ba73-75c895c038fd\":'{\"g\":\"7ece91da-843d-7367-0e53-5b0fb8674a02\",\"c\":1623524294771,\"l\":1623524294771}',\n",
    "                \"browser_id\":\"5f9c869cb72a70c17301aec27d5d1e09\",\n",
    "                \"cto_bundle\":\"2k9gpF92RUlOMHdJaGRVU1ElMkZrRWx0JTJCTXdleG5HM1p5eTA3bHk0WEclMkJwc1lZOWg3dWdjMjJJdW9SUWtkSklnVm9HYXZrSWY2NiUyQkUwaXRiUU5PeGs5d25mJTJCaCUyQm5hdnFsTER6dk5XdDhVZDclMkZDJTJCQlB5Qjd0T053OCUyQkdnJTJCencxTjNJdWxEbnBJeE9oU0RkanNiMzJRbEslMkYlMkJ2M2clM0QlM0Q\",\n",
    "                \"identity\":\"f605042192d5be2b30c78e6b4322cb63\",\n",
    "                \"kppid_managed\":\"kppidff_OLWOYDUw\",\n",
    "                \"lskjfewjrh34ghj23brjh234\":\"a0dqRGZUSS96ZnF1MVRkeGRqVGloaHovZ3o4YzFXT3ZUS3lsM0dWUFJjS0hKNytRdXRmYlV3V2dHNXVDdTJIVGNTZnM1R2Qxb2JYV2U3YTBWM01zd2xiQjFLM3RTUmIrcDdDOTQvMENnaEhJcU5JcDJxdGpKTytOWkRpTjdFY0l4WkloMXhvUzd3cFFsY1NlSkQrMDhzQUkxd2tQM0FRSHpmRFFqcVB6OE5FQzZ5clMzbk9qRmNjRFRiME8vaWlrLS1xV0k3NGFEaEhERlhHWnYyQkpyM3hBPT0=--e6c0ff3abeddb96ccbc4ded5549b9b2eaff204cd\",\n",
    "                \"session_id\":\"\"\n",
    "                \n",
    "            }\n",
    "        \n",
    "        self.headers_token = {\n",
    "                \n",
    "                \"Host\": \"www.bukalapak.com\",\n",
    "                \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0\",\n",
    "                \"Accept\": \"application/json, text/plain, */*\",\n",
    "                \"Accept-Language\": \"en-US,en;q=0.5\",\n",
    "                \"Accept-Encoding\": \"gzip, deflate, br\",\n",
    "                \"Content-Type\": \"application/json;charset=utf-8\",\n",
    "                \"Content-Length\": \"44\",\n",
    "                \"Origin\": \"https://www.bukalapak.com\",\n",
    "                \"Referer\": \"https://www.bukalapak.com/\",\n",
    "                \"Connection\": \"keep-alive\",\n",
    "                \"Cookie\": \"identity=f605042192d5be2b30c78e6b4322cb63; browser_id=5f9c869cb72a70c17301aec27d5d1e09; __auc=19928746177874debc7bca2450d; session_id=; lskjfewjrh34ghj23brjh234=a0dqRGZUSS96ZnF1MVRkeGRqVGloaHovZ3o4YzFXT3ZUS3lsM0dWUFJjS0hKNytRdXRmYlV3V2dHNXVDdTJIVGNTZnM1R2Qxb2JYV2U3YTBWM01zd2xiQjFLM3RTUmIrcDdDOTQvMENnaEhJcU5JcDJxdGpKTytOWkRpTjdFY0l4WkloMXhvUzd3cFFsY1NlSkQrMDhzQUkxd2tQM0FRSHpmRFFqcVB6OE5FQzZ5clMzbk9qRmNjRFRiME8vaWlrLS1xV0k3NGFEaEhERlhHWnYyQkpyM3hBPT0%3D--e6c0ff3abeddb96ccbc4ded5549b9b2eaff204cd; __cfruid=9d80714daf3fb1ed5e80d7e10123c28d8ec630cc-1623556183; _ga=GA1.2.864919285.1623524285; _gid=GA1.2.293862253.1623524285; _gcl_au=1.1.934570982.1623524285; _fbp=fb.1.1623524287461.1658652966; kppid_managed=kppidff_OLWOYDUw; _hjTLDTest=1; _hjid=ee5bbddd-8300-48a8-9a0b-b3d71f96a07c; cto_bundle=2k9gpF92RUlOMHdJaGRVU1ElMkZrRWx0JTJCTXdleG5HM1p5eTA3bHk0WEclMkJwc1lZOWg3dWdjMjJJdW9SUWtkSklnVm9HYXZrSWY2NiUyQkUwaXRiUU5PeGs5d25mJTJCaCUyQm5hdnFsTER6dk5XdDhVZDclMkZDJTJCQlB5Qjd0T053OCUyQkdnJTJCencxTjNJdWxEbnBJeE9oU0RkanNiMzJRbEslMkYlMkJ2M2clM0QlM0Q; ab.storage.deviceId.a987eba1-624b-42ab-ba73-75c895c038fd=%7B%22g%22%3A%227ece91da-843d-7367-0e53-5b0fb8674a02%22%2C%22c%22%3A1623524294771%2C%22l%22%3A1623524294771%7D; _dd_s=logs=1&id=c00b0782-a851-4b87-84b3-6167846a3936&created=1623556188705&expire=1623560339953; _hjIncludedInSessionSample=1; _hjAbsoluteSessionInProgress=1; _vwo_uuid_v2=DB279B8E19F57F720EEA02225A0A044EF|32ed062cbc20a26c644b5144a3694c5f; _td=477254b3-df17-44f4-adc6-feb14c354d9c; __asc=a3e0eb9e17a03a6e7a5988fed4f\",\n",
    "                \"Cache-Control\": \"max-age=0\",\n",
    "                \"TE\": \"Trailers\"\n",
    "                \n",
    "            }\n",
    "        \n",
    "        self.headers_endpoint = {\n",
    "                \n",
    "                \"Host\": \"api.bukalapak.com\",\n",
    "                \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0\",\n",
    "                \"Accept\": \"application/json, text/plain, */*\",\n",
    "                \"Accept-Language\": \"en-US,en;q=0.5\",\n",
    "                \"Accept-Encoding\": \"gzip, deflate, br\",\n",
    "                \"Origin\": \"https://www.bukalapak.com\",\n",
    "                \"Referer\": \"https://www.bukalapak.com/\",\n",
    "                \"Connection\": \"keep-alive\",\n",
    "                \"TE\": \"Trailers\"\n",
    "                \n",
    "            }\n",
    "        \n",
    "        self.cookies_endpoint = {\n",
    "                \n",
    "                \"__asc\" : \"b558002717a0436b60305005513\",\n",
    "                \"__auc\": \"19928746177874debc7bca2450d\",\n",
    "                \"_fbp\":\"fb.1.1623572747148.1262560140\",\n",
    "                \"_gat_UA-12425854-1\":\"1\",\n",
    "                \"_gat\":\"1\",\n",
    "                \"_ga\":\"GA1.2.864919285.1623524285\",\n",
    "                \"_gcl_au\":\"1.1.934570982.1623524285\",\n",
    "                \"_gid\":\"GA1.2.293862253.1623524285\",\n",
    "                \"_hjAbsoluteSessionInProgress\":\"0\",\n",
    "                \"_hjFirstSeen\":\"1\",\n",
    "                \"_hjid\":\"ee5bbddd-8300-48a8-9a0b-b3d71f96a07c\",\n",
    "                \"_hjTLDTest\":\"1\",\n",
    "                \"_td\":\"477254b3-df17-44f4-adc6-feb14c354d9c\",\n",
    "                \"_vwo_uuid_v2\":\"DB279B8E19F57F720EEA02225A0A044EF|32ed062cbc20a26c644b5144a3694c5f\",\n",
    "                \"ab.storage.deviceId.a987eba1-624b-42ab-ba73-75c895c038fd\":'{\"g\":\"7ece91da-843d-7367-0e53-5b0fb8674a02\",\"c\":1623524294771,\"l\":1623524294771}',\n",
    "                \"browser_id\":\"5f9c869cb72a70c17301aec27d5d1e09\",\n",
    "                \"cto_bundle\":\"8Ogo9F92RUlOMHdJaGRVU1ElMkZrRWx0JTJCTXdlNGg3S3ZGNVQxdmNIVmFXYVd2MDNWeXgya003bTlzQnBNYURLY0tUenNjY2Q4b0J5MGx3Qzk3TzdRTHI1bnl5ZHp0bTRFOHJNaUNxUndyZWVFMiUyQmZWZUpkUUhZeUl0c3Q0VWNyN3pCV2todzFRYnpTRnZQbExscXp1MVhsMldsOGclM0QlM0Q\",\n",
    "                \"identity\":\"f605042192d5be2b30c78e6b4322cb63\",\n",
    "                \"lskjfewjrh34ghj23brjh234\":\"NmQzNGhpZmJ4MzJCZisxVFZXMjlNd1JHL00wSmViTWlZQ2lGdzcxa0RGZkdGdmdIS0V1MmwyRGt4UTJwbXFMNk1wMDFFUUhzS0pneUV5blBrcHBza04rOTZGajdMT1JSOFM0djRjb09YeW5CYUpVZGliQzkrR3BOVlFuZ2QrbmJnNXRiVW9DZlhJa25scFJKaG1YL0NhUDNjY0tZTGZBdDY4M0xNaXdKVTJGdVBPVEo3QkpQa0NGZmxyTXVvdjNHLS1yT1JTUFFHWTJSV1lDTEJWaTA3S2hRPT0%3D--9c05ee25869348a89781e38c2d8950b28da9bd3a\",\n",
    "                \"session_id\":\"\"\n",
    "                \n",
    "            }\n",
    "        \n",
    "    def getBukalapak(self) :\n",
    "        \n",
    "        def findAccess_token():\n",
    "            \n",
    "            request = req.get(self.TOKEN_ENDPOINT, headers=self.headers_token, cookies=self.cookies_token)\n",
    "            parseRequest = BeautifulSoup(request.text, 'html.parser')\n",
    "            scripts = BeautifulSoup(str(parseRequest.find('meta',attrs={'name':'oauth-access-token'})), 'html.parser').find(\"meta\")\n",
    "            \n",
    "            return str(scripts)[15:61]\n",
    "            \n",
    "        \n",
    "        sorting = []\n",
    "        scraper = cloudscraper.create_scraper()\n",
    "        scraper = cloudscraper.CloudScraper()\n",
    "        request_product = json.loads(scraper.get(self.ENDPOINT+findAccess_token()).text)\n",
    "        \n",
    "        for indexSorting in range(len(request_product[\"data\"])):\n",
    "            \n",
    "            Product = {\n",
    "                \n",
    "                \"nama_produk\" : request_product[\"data\"][indexSorting][\"name\"],\n",
    "                \"harga\"       : int(request_product[\"data\"][indexSorting][\"price\"]),\n",
    "                \"diskon\"      : int(request_product[\"data\"][indexSorting][\"discount_percentage\"]),\n",
    "                \"rating\"      : float(request_product[\"data\"][indexSorting][\"rating\"][\"average_rate\"]),\n",
    "                \"foto_produk\" : request_product[\"data\"][indexSorting][\"images\"][\"large_urls\"][0],\n",
    "                \"url_produk\"  : request_product[\"data\"][indexSorting][\"url\"],\n",
    "                \"nama_toko\"   : request_product[\"data\"][indexSorting][\"store\"][\"name\"],\n",
    "                \"lokasi_toko\" : f'{request_product[\"data\"][indexSorting][\"store\"][\"address\"][\"city\"]} - {request_product[\"data\"][indexSorting][\"store\"][\"address\"][\"province\"]}',\n",
    "                \"url_toko\"    : request_product[\"data\"][indexSorting][\"store\"][\"url\"],\n",
    "                \"asal_ecommerce\" : \"bukalapak\"\n",
    "                \n",
    "                \n",
    "            }\n",
    "            \n",
    "            sorting.append(\n",
    "            \n",
    "                Product\n",
    "            \n",
    "            )\n",
    "            \n",
    "        return sorting\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "\n",
    "**[ * ] SORTING DATA**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ PLEASE WAIT...... ]\n",
      "\n",
      "[ DONE...... ]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "class ecommerceData:\n",
    "    \n",
    "    def SORT(self, Search):\n",
    "        \n",
    "        toped = Tokopedia(Search).getTokopedia()\n",
    "        shopee = Shopee(Search).getShopee()\n",
    "        BL = Bukalapak(Search).getBukalapak()\n",
    "        \n",
    "        return pd.DataFrame(toped+shopee+BL).sort_values(by=[\"harga\",\"rating\",\"diskon\"])\n",
    "\n",
    "\n",
    "print(\"[ PLEASE WAIT...... ]\\n\")\n",
    "\n",
    "ecommerceData().SORT(\n",
    "    \n",
    "    Search = \"case xiaomi 4x\" \n",
    "\n",
    ").to_csv(\"hasil.csv\")\n",
    "\n",
    "print(\"[ DONE...... ]\\n\")     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}