twitter_v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import time\n",
    "import datetime\n",
    "import pymongo\n",
    "from tqdm import tqdm\n",
    "import copy\n",
    "from loguru import logger\n",
    "\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.support.wait import WebDriverWait\n",
    "\n",
    "from selenium.common.exceptions import NoSuchElementException,StaleElementReferenceException,TimeoutException"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tweet:\n",
    "\n",
    "    def __init__(self, query, uid, ptime, pcontent, padditional, nb_reply, nb_retweet, nb_favorite):\n",
    "        self.query = query\n",
    "        self.uid = uid\n",
    "        self.ptime = ptime\n",
    "        self.pcontent = pcontent\n",
    "        self.padditional = padditional  # 转发推文，文章链接，图片，视频\n",
    "        self.nb_retweet = nb_retweet  # nbr of retweet\n",
    "        self.nb_favorite = nb_favorite  # nbr of favorite\n",
    "        self.nb_reply = nb_reply    # nbr of reply\n",
    "\n",
    "    def __repr__(self):\n",
    "        return \"Tweet={}\\nQuery={}\".format(self.pcontent, self.query)\n",
    "\n",
    "\n",
    "class User:\n",
    "\n",
    "    def __init__(self, profile_url):\n",
    "        self.profile_url = profile_url\n",
    "        self.ID = profile_url.split('/')[-1]\n",
    "        self.name = ''\n",
    "        self.avatar = ''\n",
    "        self.query = ''# query相关的大V用户\n",
    "        self.intro = ''\n",
    "        \n",
    "\n",
    "    def __repr__(self):\n",
    "        return \"User {}\".format(self.ID)\n",
    "\n",
    "\n",
    "def compare_time(time1,time2):\n",
    "    s_time = time.mktime(time.strptime(time1,'%Y年%m月%d日'))\n",
    "    e_time = time.mktime(time.strptime(time2,'%Y年%m月%d日'))\n",
    "    return int(s_time) - int(e_time)\n",
    "    \n",
    "def convert_time(x):\n",
    "    '''\n",
    "    for x in ['20分钟','1小时','1天', '10月10日','2018年10月1日']:\n",
    "        print(convert_time(x))\n",
    "    '''\n",
    "    now = datetime.datetime.now()\n",
    "    pattern = r'\\d{4}年\\d+月\\d+日'\n",
    "    if re.match(pattern, x):\n",
    "        return x\n",
    "    pattern = r'\\d+月\\d+日'\n",
    "    if re.match(pattern, x):\n",
    "        return \"{}年\".format(now.year)+x\n",
    "    return \"{}年{}月{}日\".format(now.year, now.month, now.day)\n",
    "\n",
    "def is_non_result(browser):\n",
    "    '''\n",
    "    判断结果是否为空\n",
    "    '''\n",
    "#     result_div_xpath = \"//div[@id='react-root']\"\n",
    "#     wait.until(EC.presence_of_element_located((By.XPATH, result_div_xpath)))\n",
    "#     try:\n",
    "#         result_div = browser.find_element_by_xpath(result_div_xpath)\n",
    "#         return \"未找到结果\" in result_div.text\n",
    "#     except NoSuchElementException as e:\n",
    "    return \"未找到结果\" in browser.find_element_by_tag_name('body').text\n",
    "\n",
    "def get_search_input_v1(browser):\n",
    "    # 定位搜索框\n",
    "    search_input_xpath = \"//input[@placeholder='搜索 Twitter']\"\n",
    "    wait.until(EC.presence_of_element_located((By.XPATH, search_input_xpath)))\n",
    "    search_input = browser.find_element_by_xpath(search_input_xpath)\n",
    "    return search_input\n",
    "\n",
    "# def get_search_input_v2(browser):\n",
    "#     # 请求主站\n",
    "#     browser.get('https://twitter.com/search-home')\n",
    "#     # 定位搜索框\n",
    "#     search_input_id = 'search-home-input'\n",
    "#     wait.until(EC.presence_of_element_located((By.ID, search_input_id)))\n",
    "#     search_input = browser.find_element_by_id(search_input_id)\n",
    "#     return search_input\n",
    "\n",
    "\n",
    "def extract_reply_retweet_favorite(element):\n",
    "    t = []\n",
    "    for x in element.find_elements_by_xpath('./div')[:3]:\n",
    "        if x.text.strip() == '':\n",
    "            t.append(0)\n",
    "        else:\n",
    "            t.append(int(x.text.strip()))\n",
    "    return tuple(t)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### query -> 推文爬取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_tweet_result_div(result_div,query):\n",
    "    count = 0\n",
    "    for div in result_div:\n",
    "        user, tweet = div.find_elements_by_xpath('./div')\n",
    "        profile_url = user.find_element_by_tag_name(\n",
    "            'a').get_attribute('href').strip()\n",
    "        uid = profile_url.split('/')[-1]\n",
    "        a, *b_c, d = tweet.find_elements_by_xpath('./div')  # 按照div分为>=3层\n",
    "        ptime = a.find_elements_by_tag_name('a')[-1].text\n",
    "        ptime = convert_time(ptime)\n",
    "        nb_reply, nb_retweet, nb_favorite = 0,0,0\n",
    "        try:\n",
    "            nb_reply, nb_retweet, nb_favorite = extract_reply_retweet_favorite(\n",
    "                d)\n",
    "        except:\n",
    "            nb_reply, nb_retweet, nb_favorite = 0, 0, 0\n",
    "        pcontent = b_c[0].text\n",
    "        padditional = []\n",
    "        if len(b_c) > 1:\n",
    "            for x in b_c[1:]:\n",
    "                try:\n",
    "                    a = x.find_element_by_tag_name('a').get_attribute('href')\n",
    "                    padditional.append(a)\n",
    "                except NoSuchElementException as e:\n",
    "                    padditional.append(x.text.strip())\n",
    "        user = User(profile_url)\n",
    "        tweet = Tweet(query, uid, ptime, pcontent, padditional,\n",
    "                      nb_reply, nb_retweet, nb_favorite)\n",
    "        # save to databse\n",
    "        user_dict = user.__dict__\n",
    "        user_dict['_id']=user_dict['ID']\n",
    "        if user_table.update_one({'_id': user_dict['_id']}, {'$set': user_dict}, upsert=True) and tweet_table.insert_one(tweet.__dict__):\n",
    "            count += 1\n",
    "    return count\n",
    "\n",
    "def crawl_tweet(browser, query):\n",
    "    count = 0\n",
    "    result_div_xpath = '//div[@data-testid=\"tweet\"]'\n",
    "    wait.until(EC.presence_of_element_located((By.XPATH, result_div_xpath)))\n",
    "    result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "    last_div = result_div[-1]\n",
    "    # 解析结果\n",
    "    count += parse_tweet_result_div(result_div,query)\n",
    "    while count < MAX_TWEET_SIZE:\n",
    "#         logger.info(\"{}/{}\".format(count,MAX_TWEET_SIZE))\n",
    "        result_div_xpath = '//div[@data-testid=\"tweet\"]'\n",
    "        wait.until(EC.presence_of_element_located((By.XPATH, result_div_xpath)))\n",
    "        result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "        last_div = result_div[-1]\n",
    "        try:\n",
    "            count += parse_tweet_result_div(result_div,query)\n",
    "        except StaleElementReferenceException as e:\n",
    "            time.sleep(2)\n",
    "            continue\n",
    "        \n",
    "        # 翻页\n",
    "        try_times = 0\n",
    "        old_height = browser.execute_script(\"return document.body.scrollHeight;\")\n",
    "        while True:\n",
    "            browser.execute_script(\n",
    "                'window.scrollTo(0,document.body.scrollHeight)')\n",
    "            wait.until(EC.presence_of_element_located(\n",
    "                (By.XPATH, result_div_xpath)))\n",
    "            result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "            if result_div[-1] == last_div:\n",
    "                try_times += 1\n",
    "            if result_div[-1] != last_div:\n",
    "                last_div = result_div[-1]\n",
    "                break\n",
    "            time.sleep(3)\n",
    "            new_height = browser.execute_script(\"return document.body.scrollHeight;\")\n",
    "            if old_height == new_height:\n",
    "                try_times += 1\n",
    "                last_div = result_div[-1]\n",
    "            if try_times >= 3:\n",
    "                count = MAX_TWEET_SIZE # 到头了停止翻页采集该query\n",
    "                print('到头了')\n",
    "                break\n",
    "\n",
    "def search_tweet_from_query(browser,query_list,finish_query_list):\n",
    "    '''\n",
    "    更加query采集推文\n",
    "    '''\n",
    "    for query in tqdm(query_list):\n",
    "        logger.info('query = {}'.format(query))\n",
    "        browser.get('https://twitter.com/explore')\n",
    "\n",
    "        # 定位搜索框\n",
    "        if browser.current_url == 'https://twitter.com/explore':\n",
    "            search_input = get_search_input_v1(browser)\n",
    "        else:\n",
    "            print('error')\n",
    "            return\n",
    "        # 搜索query\n",
    "        search_input.clear()\n",
    "        search_input.send_keys(query)\n",
    "        search_input.send_keys(Keys.ENTER)\n",
    "\n",
    "        # 获取结果\n",
    "        if is_non_result(browser):\n",
    "            bad_query_list.append(query)\n",
    "            continue\n",
    "        time.sleep(1)\n",
    "        try:\n",
    "            crawl_tweet(browser, query)\n",
    "        except TimeoutException as e:\n",
    "            print('TimeoutException')\n",
    "            continue\n",
    "        finish_query_list.append(query)\n",
    "    print(bad_query_list)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### query -> 爬取相关用户"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_user_from_query(browser,query_list,finish_query_list):\n",
    "    '''\n",
    "    根据query采集maxsize用户列表\n",
    "    '''\n",
    "    for query in tqdm(query_list):\n",
    "        logger.info('query = {}'.format(query))\n",
    "        browser.get('https://twitter.com/explore')\n",
    "\n",
    "        # 定位搜索框\n",
    "        if browser.current_url == 'https://twitter.com/explore':\n",
    "            search_input = get_search_input_v1(browser)\n",
    "        else:\n",
    "            print('error')\n",
    "            return\n",
    "        # 搜索query\n",
    "        search_input.clear()\n",
    "        search_input.send_keys(query)\n",
    "        search_input.send_keys(Keys.ENTER)\n",
    "\n",
    "        # 获取结果\n",
    "        if is_non_result(browser):\n",
    "            bad_query_list.append(query)\n",
    "            continue\n",
    "        time.sleep(2)\n",
    "        # 请求用户结果页面\n",
    "        browser.get(browser.current_url + '&f=user')\n",
    "        \n",
    "        try:\n",
    "            crawl_user(browser,query)\n",
    "        except TimeoutException as e:\n",
    "            print('TimeoutException')\n",
    "            continue\n",
    "        finish_query_list.append(query)\n",
    "    \n",
    "    print(bad_query_list)\n",
    "\n",
    "def parse_user_result_div(result_div,query):\n",
    "    count = 0\n",
    "    for t in result_div:\n",
    "        left, right = t.find_elements_by_xpath('./div/div')\n",
    "        profile_url = left.find_element_by_tag_name(\n",
    "            'a').get_attribute('href').strip()\n",
    "        uid = profile_url.split('/')[-1]\n",
    "        print('parsing uid = {}'.format(uid))\n",
    "        avatar = left.find_element_by_tag_name('img').get_attribute('src')\n",
    "        a, *b = right.find_elements_by_xpath('./div')  # 按照div分为>=3层\n",
    "        uname= a.text.split('\\n')[0]\n",
    "        intro = ''\n",
    "        if len(b)!=0:\n",
    "            intro = b[-1].text.strip()\n",
    "        \n",
    "        user = User(profile_url)\n",
    "        user.ID = uid\n",
    "        user.avatar = avatar\n",
    "        user.name = uname\n",
    "        user.intro = intro\n",
    "        user.query = query\n",
    "        \n",
    "        user_dict = user.__dict__\n",
    "        user_dict['_id']=user_dict['ID']\n",
    "\n",
    "        # save to databse\n",
    "        if user_table.update_one({'_id': user_dict['_id']}, {'$set': user_dict}, upsert=True):\n",
    "            count += 1\n",
    "    return count\n",
    "\n",
    "def crawl_user(browser, query):\n",
    "    count = 0\n",
    "    result_div_xpath = '//div[@data-testid=\"UserCell\"]'\n",
    "    wait.until(EC.presence_of_element_located((By.XPATH, result_div_xpath)))\n",
    "    result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "    last_div = result_div[-1]\n",
    "    count += parse_user_result_div(result_div,query)\n",
    "    \n",
    "    while count < MAX_USER_SIZE:\n",
    "#         logger.info(\"{}/{}\".format(count,MAX_USER_SIZE))\n",
    "        wait.until(EC.presence_of_element_located((By.XPATH, result_div_xpath)))\n",
    "        result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "        last_div = result_div[-1]\n",
    "        \n",
    "        try:\n",
    "            count += parse_user_result_div(result_div,query)\n",
    "            time.sleep(2)\n",
    "        except StaleElementReferenceException as e:\n",
    "            time.sleep(2)\n",
    "            continue\n",
    "        \n",
    "        # 翻页\n",
    "        try_times = 0\n",
    "        old_height = browser.execute_script(\"return document.body.scrollHeight;\")\n",
    "        while True:\n",
    "            browser.execute_script(\n",
    "                'window.scrollTo(0,document.body.scrollHeight)')\n",
    "            wait.until(EC.presence_of_element_located(\n",
    "                (By.XPATH, result_div_xpath)))\n",
    "            result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "            if result_div[-1] == last_div:\n",
    "                try_times += 1\n",
    "            if result_div[-1] != last_div:\n",
    "                last_div = result_div[-1]\n",
    "                break\n",
    "            time.sleep(3)\n",
    "            new_height = browser.execute_script(\"return document.body.scrollHeight;\")\n",
    "            if old_height == new_height:\n",
    "                try_times += 1\n",
    "                last_div = result_div[-1]\n",
    "            if try_times >= 3:\n",
    "                count = MAX_TWEET_SIZE # 到头了停止翻页采集该query\n",
    "                print('到头了')\n",
    "                break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 爬取特定用户的推文（时间间隔内）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_tweet_from_profile_v2(browser,query_user_list,finish_user_list):\n",
    "    for u in tqdm(query_user_list):\n",
    "        logger.info('user = {}'.format(u['_id']))\n",
    "        user_profile = 'https://twitter.com/'+ u['_id']\n",
    "        browser.get(user_profile)\n",
    "        query = u['query']\n",
    "\n",
    "        # 获取结果\n",
    "        if is_non_result(browser):\n",
    "            bad_query_list.append(query)\n",
    "            continue\n",
    "        time.sleep(1)\n",
    "        try:\n",
    "            crawl_tweet2(browser,query)\n",
    "        except TimeoutException as e:\n",
    "            print('TimeoutException')\n",
    "            continue\n",
    "        \n",
    "        finish_user_list.append(user)\n",
    "    print(bad_query_list)\n",
    "\n",
    "def crawl_tweet2(browser, query):\n",
    "    count = 0\n",
    "    result_div_xpath = '//div[@data-testid=\"tweet\"]'\n",
    "    wait.until(EC.presence_of_element_located((By.XPATH, result_div_xpath)))\n",
    "    result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "    last_div = result_div[-1]\n",
    "    # 解析结果\n",
    "    count += parse_tweet_from_profile(result_div,query)\n",
    "    while count < MAX_TWEET_SIZE:\n",
    "#         logger.info(\"{}/{}\".format(count,MAX_TWEET_SIZE))\n",
    "        result_div_xpath = '//div[@data-testid=\"tweet\"]'\n",
    "        wait.until(EC.presence_of_element_located((By.XPATH, result_div_xpath)))\n",
    "        result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "        last_div = result_div[-1]\n",
    "        try:\n",
    "            count += parse_tweet_from_profile(result_div,query)\n",
    "        except StaleElementReferenceException as e:\n",
    "            time.sleep(2)\n",
    "            continue\n",
    "        \n",
    "        # 翻页\n",
    "        try_times = 0\n",
    "        old_height = browser.execute_script(\"return document.body.scrollHeight;\")\n",
    "        while True:\n",
    "            browser.execute_script(\n",
    "                'window.scrollTo(0,document.body.scrollHeight)')\n",
    "            wait.until(EC.presence_of_element_located(\n",
    "                (By.XPATH, result_div_xpath)))\n",
    "            result_div = browser.find_elements_by_xpath(result_div_xpath)\n",
    "            if result_div[-1] == last_div:\n",
    "                try_times += 1\n",
    "            if result_div[-1] != last_div:\n",
    "                last_div = result_div[-1]\n",
    "                break\n",
    "            time.sleep(3)\n",
    "            new_height = browser.execute_script(\"return document.body.scrollHeight;\")\n",
    "            if old_height == new_height:\n",
    "                try_times += 1\n",
    "                last_div = result_div[-1]\n",
    "            if try_times >= 3:\n",
    "                count = MAX_TWEET_SIZE # 到头了停止翻页采集该query\n",
    "                print('到头了')\n",
    "                break\n",
    "\n",
    "                \n",
    "def parse_tweet_from_profile(result_div,query):\n",
    "    count = 0\n",
    "    top = 0 # 置顶是个数目\n",
    "    # 如果存在置顶推文则不考虑时间\n",
    "    try:\n",
    "        t = browser.find_elements_by_xpath('//div[@class=\"css-1dbjc4n r-1habvwh r-1iusvr4 r-16y2uox r-5f2r5o\"]')\n",
    "        top = len(t)\n",
    "    except NoSuchElementException as e:\n",
    "        pass\n",
    "    \n",
    "    for div in result_div:\n",
    "        user, tweet = div.find_elements_by_xpath('./div')\n",
    "        profile_url = user.find_element_by_tag_name(\n",
    "            'a').get_attribute('href').strip()\n",
    "        uid = profile_url.split('/')[-1]\n",
    "        a, *b_c, d = tweet.find_elements_by_xpath('./div')  # 按照div分为>=3层\n",
    "        ptime = a.find_elements_by_tag_name('a')[-1].text\n",
    "        ptime = convert_time(ptime)\n",
    "        \n",
    "        # 无置顶推文则按照时间过滤\n",
    "        top -= 1\n",
    "        if top<0 and compare_time(ptime,time_interval) < 0:\n",
    "            print('触发时间截止')\n",
    "            return MAX_TWEET_SIZE # 使其直接达到数目规模，从而停止外层循环\n",
    "        \n",
    "        nb_reply, nb_retweet, nb_favorite = 0,0,0\n",
    "        try:\n",
    "            nb_reply, nb_retweet, nb_favorite = extract_reply_retweet_favorite(\n",
    "                d)\n",
    "        except:\n",
    "            nb_reply, nb_retweet, nb_favorite = 0, 0, 0\n",
    "        pcontent = b_c[0].text\n",
    "        padditional = []\n",
    "        if len(b_c) > 1:\n",
    "            for x in b_c[1:]:\n",
    "                try:\n",
    "                    a = x.find_element_by_tag_name('a').get_attribute('href')\n",
    "                    padditional.append(a)\n",
    "                except NoSuchElementException as e:\n",
    "                    padditional.append(x.text.strip())\n",
    "        tweet = Tweet(query, uid, ptime, pcontent, padditional,\n",
    "                      nb_reply, nb_retweet, nb_favorite)\n",
    "\n",
    "        if tweet2_table.insert_one(tweet.__dict__):\n",
    "            count += 1\n",
    "    return count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 启动浏览器并登陆"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = pymongo.MongoClient(\"mongodb://10.108.17.25:27017/\")\n",
    "twitter_db = client[\"twitter_v2\"]\n",
    "user_table = twitter_db['user']\n",
    "tweet_table = twitter_db['tweet_by_query']\n",
    "tweet2_table = twitter_db['tweet_by_user']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 打开浏览器\n",
    "browser = webdriver.Chrome()\n",
    "wait = WebDriverWait(browser, 100)\n",
    "\n",
    "# 人工登录\n",
    "browser.get('https://twitter.com/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.refresh()\n",
    "time.sleep(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 采集推文"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "108"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('./projects.csv',encoding='utf-8')\n",
    "df.columns = ['Project','Country','Type']\n",
    "query_list = [x.strip() for x in df['Project'].tolist() if len(x.split()) <= 5]\n",
    "query_list = query_list\n",
    "\n",
    "# df = pd.read_csv('./policies.csv',encoding='utf-8')\n",
    "# df.columns = ['P','_','__']\n",
    "# query_list = [x.strip() for x in df['P'].tolist() if len(x.split()) <= 10]\n",
    "# query_list = query_list\n",
    "len(query_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/108 [00:00<?, ?it/s]2019-12-30 23:25:20.596 | INFO     | __main__:search_tweet_from_query:82 - query = Padma Rail Link\n",
      "  1%|          | 1/108 [00:24<43:19, 24.30s/it]2019-12-30 23:25:44.892 | INFO     | __main__:search_tweet_from_query:82 - query = Lower Sesan Two Hydropower Dam\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 2/108 [02:07<1:24:42, 47.95s/it]2019-12-30 23:27:28.043 | INFO     | __main__:search_tweet_from_query:82 - query = Central Asia–China gas pipeline\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TimeoutException\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 3/108 [02:32<1:11:50, 41.05s/it]2019-12-30 23:27:52.989 | INFO     | __main__:search_tweet_from_query:82 - query = Doraleh Multi-Purpose Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▎         | 4/108 [03:01<1:04:58, 37.48s/it]2019-12-30 23:28:22.154 | INFO     | __main__:search_tweet_from_query:82 - query = Khorgos Gateway Dry Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▍         | 5/108 [03:23<56:09, 32.72s/it]  2019-12-30 23:28:43.747 | INFO     | __main__:search_tweet_from_query:82 - query = Forest City\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 6/108 [12:54<5:30:22, 194.34s/it]2019-12-30 23:38:15.201 | INFO     | __main__:search_tweet_from_query:82 - query = Melaka Gateway\n",
      "  6%|▋         | 7/108 [13:44<4:14:00, 150.89s/it]2019-12-30 23:39:04.721 | INFO     | __main__:search_tweet_from_query:82 - query = Pakistan-China – Fiber Optic Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 8/108 [14:08<3:08:10, 112.90s/it]2019-12-30 23:39:28.976 | INFO     | __main__:search_tweet_from_query:82 - query = Diamer-Bhasha Dam\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 9/108 [15:36<2:54:05, 105.51s/it]2019-12-30 23:40:57.239 | INFO     | __main__:search_tweet_from_query:82 - query = Gwadar Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 10/108 [20:31<4:25:17, 162.42s/it]2019-12-30 23:45:52.464 | INFO     | __main__:search_tweet_from_query:82 - query = Belgrade-Montenegro Bar Port Motorway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 11/108 [22:15<3:54:11, 144.86s/it]2019-12-30 23:47:36.341 | INFO     | __main__:search_tweet_from_query:82 - query = Sino-Thai – High-Speed Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TimeoutException\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 12/108 [22:37<2:52:39, 107.92s/it]2019-12-30 23:47:58.056 | INFO     | __main__:search_tweet_from_query:82 - query = Colombo South Harbour\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 13/108 [23:01<2:10:58, 82.72s/it] 2019-12-30 23:48:21.990 | INFO     | __main__:search_tweet_from_query:82 - query = Port City Colombo\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 14/108 [24:03<1:59:53, 76.52s/it]2019-12-30 23:49:24.044 | INFO     | __main__:search_tweet_from_query:82 - query = Hambantota Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 15/108 [25:50<2:12:53, 85.74s/it]2019-12-30 23:51:11.281 | INFO     | __main__:search_tweet_from_query:82 - query = Single Gauge Trans-Asian Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▍        | 16/108 [27:34<2:19:43, 91.12s/it]2019-12-30 23:52:54.972 | INFO     | __main__:search_tweet_from_query:82 - query = Karuma Hydropower Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TimeoutException\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 17/108 [28:09<1:52:37, 74.26s/it]2019-12-30 23:53:29.873 | INFO     | __main__:search_tweet_from_query:82 - query = Pap Angren Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 18/108 [28:29<1:27:13, 58.15s/it]2019-12-30 23:53:50.447 | INFO     | __main__:search_tweet_from_query:82 - query = Budapest–Belgrade Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 19/108 [28:58<1:13:05, 49.27s/it]2019-12-30 23:54:18.998 | INFO     | __main__:search_tweet_from_query:82 - query = Yamal LNG Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▊        | 20/108 [29:42<1:10:10, 47.85s/it]2019-12-30 23:55:03.523 | INFO     | __main__:search_tweet_from_query:82 - query = Tehran-Mashhad Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 21/108 [30:02<57:15, 39.49s/it]  2019-12-30 23:55:23.519 | INFO     | __main__:search_tweet_from_query:82 - query = Lagos-Calabar Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 22/108 [30:29<51:09, 35.70s/it]2019-12-30 23:55:50.361 | INFO     | __main__:search_tweet_from_query:82 - query = Lagos-Kano Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██▏       | 23/108 [30:56<46:40, 32.95s/it]2019-12-30 23:56:16.896 | INFO     | __main__:search_tweet_from_query:82 - query = Chad-Cameroon & Chad-Sudan Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 24/108 [32:39<1:15:45, 54.12s/it]2019-12-30 23:58:00.413 | INFO     | __main__:search_tweet_from_query:82 - query = Addis Ababa Light Rail\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TimeoutException\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 25/108 [33:13<1:06:29, 48.07s/it]2019-12-30 23:58:34.376 | INFO     | __main__:search_tweet_from_query:82 - query = Benguela Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 26/108 [33:43<58:12, 42.59s/it]  2019-12-30 23:59:04.163 | INFO     | __main__:search_tweet_from_query:82 - query = Abuja-Kaduna Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 27/108 [34:21<55:37, 41.20s/it]2019-12-30 23:59:42.123 | INFO     | __main__:search_tweet_from_query:82 - query = Khartoum-Port Sudan Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 28/108 [34:35<43:52, 32.91s/it]2019-12-30 23:59:55.682 | INFO     | __main__:search_tweet_from_query:82 - query = Djibouti-Ethiopia Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 29/108 [34:57<39:12, 29.78s/it]2019-12-31 00:00:18.174 | INFO     | __main__:search_tweet_from_query:82 - query = Vientane-Boten Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 30/108 [36:41<1:07:27, 51.89s/it]2019-12-31 00:02:01.636 | INFO     | __main__:search_tweet_from_query:82 - query = Savannakhet-Lao Bao Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TimeoutException\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▊       | 31/108 [36:55<52:05, 40.59s/it]  2019-12-31 00:02:15.870 | INFO     | __main__:search_tweet_from_query:82 - query = Bangkok-Nong Khai Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|██▉       | 32/108 [37:15<43:44, 34.53s/it]2019-12-31 00:02:36.268 | INFO     | __main__:search_tweet_from_query:82 - query = Bangkok-Chiang Mai Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 33/108 [37:37<38:20, 30.68s/it]2019-12-31 00:02:57.956 | INFO     | __main__:search_tweet_from_query:82 - query = Kuala Lumpur-Singapore High Speed Rail\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███▏      | 34/108 [38:03<36:04, 29.26s/it]2019-12-31 00:03:23.893 | INFO     | __main__:search_tweet_from_query:82 - query = Jakarta-Bandung Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 35/108 [38:52<43:00, 35.34s/it]2019-12-31 00:04:13.443 | INFO     | __main__:search_tweet_from_query:82 - query = East Coast Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 36/108 [40:50<1:11:56, 59.95s/it]2019-12-31 00:06:10.821 | INFO     | __main__:search_tweet_from_query:82 - query = Gemas-Johor Bahru Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 37/108 [41:09<56:35, 47.82s/it]  2019-12-31 00:06:30.319 | INFO     | __main__:search_tweet_from_query:82 - query = Dawei Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 38/108 [41:34<47:46, 40.94s/it]2019-12-31 00:06:55.224 | INFO     | __main__:search_tweet_from_query:82 - query = Gujarat Rural Roads (MMGSY) Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 39/108 [43:18<1:08:41, 59.74s/it]2019-12-31 00:08:38.810 | INFO     | __main__:search_tweet_from_query:82 - query = Nurek Hydropower Rehabilitation Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TimeoutException\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 40/108 [43:31<51:51, 45.75s/it]  2019-12-31 00:08:51.926 | INFO     | __main__:search_tweet_from_query:82 - query = Batumi Bypass Road Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 41/108 [43:45<40:22, 36.15s/it]2019-12-31 00:09:05.680 | INFO     | __main__:search_tweet_from_query:82 - query = Natural Gas Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 42/108 [49:24<2:19:56, 127.22s/it]2019-12-31 00:14:45.397 | INFO     | __main__:search_tweet_from_query:82 - query = Tarbela 5 Hydropower Extension Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███▉      | 43/108 [49:44<1:42:59, 95.06s/it] 2019-12-31 00:15:05.423 | INFO     | __main__:search_tweet_from_query:82 - query = M4 Motorway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 44/108 [51:13<1:39:11, 93.00s/it]2019-12-31 00:16:33.610 | INFO     | __main__:search_tweet_from_query:82 - query = Dushanbe-Uzbekistan Border Road Improvement\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 45/108 [51:28<1:13:18, 69.81s/it]2019-12-31 00:16:49.310 | INFO     | __main__:search_tweet_from_query:82 - query = Nenskra Hydropower Plant\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 46/108 [51:50<57:18, 55.45s/it]  2019-12-31 00:17:11.269 | INFO     | __main__:search_tweet_from_query:82 - query = Amaravati Sustainable Capital City\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▎     | 47/108 [52:11<45:47, 45.03s/it]2019-12-31 00:17:31.986 | INFO     | __main__:search_tweet_from_query:82 - query = Madhya Pradesh Rural Connectivity Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 48/108 [52:33<38:02, 38.04s/it]2019-12-31 00:17:53.693 | INFO     | __main__:search_tweet_from_query:82 - query = Mumbai Metro Line 4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 49/108 [53:20<40:03, 40.74s/it]2019-12-31 00:18:40.730 | INFO     | __main__:search_tweet_from_query:82 - query = Sahiwal 2x660MW Coal-fired Power Plant\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▋     | 50/108 [53:40<33:19, 34.48s/it]2019-12-31 00:19:00.612 | INFO     | __main__:search_tweet_from_query:82 - query = UEP 100MW Wind Farm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 51/108 [53:54<26:58, 28.39s/it]2019-12-31 00:19:14.785 | INFO     | __main__:search_tweet_from_query:82 - query = Sachal 50MW Wind Farm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 52/108 [54:14<24:10, 25.91s/it]2019-12-31 00:19:34.911 | INFO     | __main__:search_tweet_from_query:82 - query = Peshawar-Karachi Motorway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 53/108 [54:50<26:41, 29.12s/it]2019-12-31 00:20:11.537 | INFO     | __main__:search_tweet_from_query:82 - query = Havelian Dry Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 54/108 [55:20<26:18, 29.23s/it]2019-12-31 00:20:41.026 | INFO     | __main__:search_tweet_from_query:82 - query = Gwadar International Airport\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 55/108 [56:02<29:09, 33.00s/it]2019-12-31 00:21:22.826 | INFO     | __main__:search_tweet_from_query:82 - query = Myitsone Dam\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 56/108 [56:53<33:23, 38.53s/it]2019-12-31 00:22:14.245 | INFO     | __main__:search_tweet_from_query:82 - query = Balloki Power Plant\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 57/108 [57:41<35:01, 41.20s/it]2019-12-31 00:23:01.694 | INFO     | __main__:search_tweet_from_query:82 - query = Gadani Power Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▎    | 58/108 [58:06<30:23, 36.46s/it]2019-12-31 00:23:27.086 | INFO     | __main__:search_tweet_from_query:82 - query = Hakla–Dera Ismail Khan Motorway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 59/108 [58:28<26:08, 32.00s/it]2019-12-31 00:23:48.689 | INFO     | __main__:search_tweet_from_query:82 - query = Khunjerab Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 60/108 [58:47<22:28, 28.09s/it]2019-12-31 00:24:07.631 | INFO     | __main__:search_tweet_from_query:82 - query = M5 Motorway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▋    | 61/108 [1:00:31<40:00, 51.07s/it]2019-12-31 00:25:52.320 | INFO     | __main__:search_tweet_from_query:82 - query = M8 Motorway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 62/108 [1:01:02<34:30, 45.02s/it]2019-12-31 00:26:23.229 | INFO     | __main__:search_tweet_from_query:82 - query = Matiari–Lahore Transmission Line\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 63/108 [1:01:26<29:00, 38.67s/it]2019-12-31 00:26:47.099 | INFO     | __main__:search_tweet_from_query:82 - query = Orange Line Lahore Metro\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 64/108 [1:01:54<26:05, 35.59s/it]2019-12-31 00:27:15.492 | INFO     | __main__:search_tweet_from_query:82 - query = Pak-China Technical and Vocational Institute\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 65/108 [1:02:31<25:41, 35.85s/it]2019-12-31 00:27:51.942 | INFO     | __main__:search_tweet_from_query:82 - query = Pakistan Port Qasim Power Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 66/108 [1:02:59<23:32, 33.63s/it]2019-12-31 00:28:20.385 | INFO     | __main__:search_tweet_from_query:82 - query = Quaid-e-Azam Solar Park\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 67/108 [1:03:37<23:53, 34.97s/it]2019-12-31 00:28:58.503 | INFO     | __main__:search_tweet_from_query:82 - query = Karakoram Highway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 68/108 [1:06:12<47:19, 70.99s/it]2019-12-31 00:31:33.515 | INFO     | __main__:search_tweet_from_query:82 - query = Sahiwal Coal Power Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 69/108 [1:06:48<39:11, 60.30s/it]2019-12-31 00:32:08.896 | INFO     | __main__:search_tweet_from_query:82 - query = Suki Kinari Hydropower Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▍   | 70/108 [1:07:18<32:29, 51.31s/it]2019-12-31 00:32:39.206 | INFO     | __main__:search_tweet_from_query:82 - query = Nigcomsat Satellites\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 71/108 [1:07:40<26:12, 42.50s/it]2019-12-31 00:33:01.143 | INFO     | __main__:search_tweet_from_query:82 - query = MNC Lido City\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 72/108 [1:08:08<22:50, 38.06s/it]2019-12-31 00:33:28.853 | INFO     | __main__:search_tweet_from_query:82 - query = Harare Airport Expansion\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 73/108 [1:08:29<19:19, 33.13s/it]2019-12-31 00:33:50.478 | INFO     | __main__:search_tweet_from_query:82 - query = Gilgit KIU Hydropower\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▊   | 74/108 [1:08:43<15:29, 27.34s/it]2019-12-31 00:34:04.313 | INFO     | __main__:search_tweet_from_query:82 - query = Cacho 50MW Wind Power Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 75/108 [1:08:57<12:47, 23.26s/it]2019-12-31 00:34:18.043 | INFO     | __main__:search_tweet_from_query:82 - query = Rahimyar Khan Power Plant\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 76/108 [1:09:11<10:51, 20.37s/it]2019-12-31 00:34:31.666 | INFO     | __main__:search_tweet_from_query:82 - query = Kohala Hydel Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████▏  | 77/108 [1:09:33<10:48, 20.93s/it]2019-12-31 00:34:53.911 | INFO     | __main__:search_tweet_from_query:82 - query = Phandar Hydropower Station\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 78/108 [1:09:46<09:17, 18.57s/it]2019-12-31 00:35:06.985 | INFO     | __main__:search_tweet_from_query:82 - query = Karachi Circular Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 79/108 [1:10:44<14:42, 30.44s/it]2019-12-31 00:36:05.112 | INFO     | __main__:search_tweet_from_query:82 - query = Greater Peshawar Mass Transit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 80/108 [1:11:03<12:38, 27.11s/it]2019-12-31 00:36:24.438 | INFO     | __main__:search_tweet_from_query:82 - query = Quetta Mass Transit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 81/108 [1:11:36<12:55, 28.73s/it]2019-12-31 00:36:56.973 | INFO     | __main__:search_tweet_from_query:82 - query = Keti BUnder Sea Port Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 82/108 [1:11:49<10:28, 24.16s/it]2019-12-31 00:37:10.453 | INFO     | __main__:search_tweet_from_query:82 - query = Rashakai Economic Zone\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 83/108 [1:12:20<10:51, 26.05s/it]2019-12-31 00:37:40.903 | INFO     | __main__:search_tweet_from_query:82 - query = China Special Economic Zone Dhabeji\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 84/108 [1:12:39<09:35, 23.97s/it]2019-12-31 00:38:00.018 | INFO     | __main__:search_tweet_from_query:82 - query = Bostan Industrial Zone\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▊  | 85/108 [1:13:00<08:50, 23.07s/it]2019-12-31 00:38:20.984 | INFO     | __main__:search_tweet_from_query:82 - query = Allama Iqbal Industrial City\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████▉  | 86/108 [1:13:29<09:10, 25.03s/it]2019-12-31 00:38:50.581 | INFO     | __main__:search_tweet_from_query:82 - query = ICT Model Industrial Zone\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 87/108 [1:13:43<07:31, 21.48s/it]2019-12-31 00:39:03.791 | INFO     | __main__:search_tweet_from_query:82 - query = Mirpur Special Economic Zone\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████▏ | 88/108 [1:14:04<07:09, 21.49s/it]2019-12-31 00:39:25.287 | INFO     | __main__:search_tweet_from_query:82 - query = Mohmand Marble City\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 89/108 [1:14:45<08:35, 27.14s/it]2019-12-31 00:40:05.612 | INFO     | __main__:search_tweet_from_query:82 - query = Moqpondass Special Economic Zone\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 90/108 [1:14:57<06:51, 22.84s/it]2019-12-31 00:40:18.431 | INFO     | __main__:search_tweet_from_query:82 - query = Haifa Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 91/108 [1:15:18<06:14, 22.04s/it]2019-12-31 00:40:38.601 | INFO     | __main__:search_tweet_from_query:82 - query = Port of Piraeus\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 92/108 [1:16:33<10:08, 38.05s/it]2019-12-31 00:41:54.024 | INFO     | __main__:search_tweet_from_query:82 - query = Kumport Terminal\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 93/108 [1:16:55<08:18, 33.21s/it]2019-12-31 00:42:15.932 | INFO     | __main__:search_tweet_from_query:82 - query = Suez Canal Economic Zone\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 94/108 [1:17:36<08:17, 35.50s/it]2019-12-31 00:42:56.782 | INFO     | __main__:search_tweet_from_query:82 - query = Kyaukpyu Deep Sea Tanker Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 95/108 [1:19:20<12:08, 56.00s/it]2019-12-31 00:44:40.620 | INFO     | __main__:search_tweet_from_query:82 - query = Kyaukpyu Special Economic Zone\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TimeoutException\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 96/108 [1:19:39<08:59, 44.96s/it]2019-12-31 00:44:59.807 | INFO     | __main__:search_tweet_from_query:82 - query = Port Aktau\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|████████▉ | 97/108 [1:20:04<07:09, 39.03s/it]2019-12-31 00:45:25.015 | INFO     | __main__:search_tweet_from_query:82 - query = “Khorgos – Eastern Gate”\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 98/108 [1:20:23<05:29, 32.93s/it]2019-12-31 00:45:43.701 | INFO     | __main__:search_tweet_from_query:82 - query = Khalifa Port Terminal 2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 99/108 [1:20:43<04:22, 29.20s/it]2019-12-31 00:46:04.213 | INFO     | __main__:search_tweet_from_query:82 - query = Greater Peshawar Region Mass Transit\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 100/108 [1:20:58<03:19, 24.88s/it]2019-12-31 00:46:19.018 | INFO     | __main__:search_tweet_from_query:82 - query = Dhaka-Chattogram Rail Route\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▎| 101/108 [1:21:14<02:35, 22.15s/it]2019-12-31 00:46:34.773 | INFO     | __main__:search_tweet_from_query:82 - query = Kuala Tanjung Port\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 102/108 [1:21:39<02:18, 23.08s/it]2019-12-31 00:47:00.042 | INFO     | __main__:search_tweet_from_query:82 - query = Kayan River Hydropower Plant\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 103/108 [1:21:52<01:40, 20.19s/it]2019-12-31 00:47:13.485 | INFO     | __main__:search_tweet_from_query:82 - query = Lake Toba Tourism District\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▋| 104/108 [1:22:05<01:11, 17.90s/it]2019-12-31 00:47:26.037 | INFO     | __main__:search_tweet_from_query:82 - query = International Airport Lembeh\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 105/108 [1:22:20<00:50, 16.91s/it]2019-12-31 00:47:40.638 | INFO     | __main__:search_tweet_from_query:82 - query = Dammam Riyadh Freight Line\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 106/108 [1:22:37<00:34, 17.17s/it]2019-12-31 00:47:58.419 | INFO     | __main__:search_tweet_from_query:82 - query = Hassyan Clean Coal Project\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 107/108 [1:23:02<00:19, 19.56s/it]2019-12-31 00:48:23.542 | INFO     | __main__:search_tweet_from_query:82 - query = Muse-Mandalay Railway\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 108/108 [1:23:31<00:00, 46.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n",
      "[]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 第1步\n",
    "finish_query_list = []\n",
    "bad_query_list = []\n",
    "MAX_TWEET_SIZE = 1000\n",
    "search_tweet_from_query(browser,query_list,finish_query_list)\n",
    "\n",
    "# 第2步\n",
    "# MAX_TWEET_SIZE = 1000\n",
    "# special_list = [\n",
    "#     \"the belt and road\",\n",
    "#     'One Belt one road',\n",
    "#     \"the Silk Road\",\n",
    "#     'the Silk Road Economic Belt',\n",
    "#     'Belt and Road Initiative',\n",
    "#     '21st Century Maritime Silk Road',\n",
    "#     'Spirit of the Silk Road',\n",
    "#     'Silk Road Fund',\n",
    "#     'Silk Road of Green Development'\n",
    "# ]\n",
    "# search_tweet_from_query(browser,special_list,finish_query_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(finish_query_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser.refresh()\n",
    "time.sleep(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 采集用户"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/2 [00:00<?, ?it/s]\u001b[A2019-12-31 10:34:58.260 | INFO     | __main__:search_user_from_query:6 - query = One Belt one road\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "parsing uid = beltandroadnow\n",
      "parsing uid = BirKusakBirYol\n",
      "parsing uid = 1GadgetBoy\n",
      "parsing uid = TheBeltandRoad\n",
      "parsing uid = OBORVC\n",
      "parsing uid = OneBeltOneRoad_\n",
      "parsing uid = OneBelt_OneRoad\n",
      "parsing uid = obormalaysia\n",
      "parsing uid = SilkRoadLAC\n",
      "parsing uid = OneBeltProject\n",
      "parsing uid = VasilGelev\n",
      "parsing uid = beltandroadnow\n",
      "parsing uid = BirKusakBirYol\n",
      "parsing uid = 1GadgetBoy\n",
      "parsing uid = TheBeltandRoad\n",
      "parsing uid = OBORVC\n",
      "parsing uid = OneBeltOneRoad_\n",
      "parsing uid = OneBelt_OneRoad\n",
      "parsing uid = obormalaysia\n",
      "parsing uid = SilkRoadLAC\n",
      "parsing uid = OneBeltProject\n",
      "parsing uid = VasilGelev\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 50%|█████     | 1/2 [00:14<00:14, 14.90s/it]\u001b[A2019-12-31 10:35:13.159 | INFO     | __main__:search_user_from_query:6 - query = Belt and Road Initiative\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n",
      "parsing uid = BeltNRoad\n",
      "parsing uid = Belt_Road_China\n",
      "parsing uid = beltroadnews\n",
      "parsing uid = CPECWire\n",
      "parsing uid = TorrinWilkins\n",
      "parsing uid = khaleefah30\n",
      "parsing uid = BRItv_Plus\n",
      "parsing uid = TJMa_beijing\n",
      "parsing uid = VasilGelev\n",
      "parsing uid = jeremy_garlick\n",
      "parsing uid = jordiyang_srcic\n",
      "parsing uid = BeltandRoadBRC\n",
      "parsing uid = BeltNRoad\n",
      "parsing uid = Belt_Road_China\n",
      "parsing uid = beltroadnews\n",
      "parsing uid = CPECWire\n",
      "parsing uid = TorrinWilkins\n",
      "parsing uid = khaleefah30\n",
      "parsing uid = BRItv_Plus\n",
      "parsing uid = TJMa_beijing\n",
      "parsing uid = VasilGelev\n",
      "parsing uid = jeremy_garlick\n",
      "parsing uid = jordiyang_srcic\n",
      "parsing uid = BeltandRoadBRC\n",
      "parsing uid = OBOReurope\n",
      "parsing uid = eyckfreymann\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 2/2 [00:29<00:00, 14.68s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "到头了\n",
      "[]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "finish_query_list = []\n",
    "bad_query_list = []\n",
    "MAX_USER_SIZE = 50\n",
    "special_list = [\n",
    "    \"belt and road\",\n",
    "    \"One Belt one road\",\n",
    "    'Belt and Road Initiative'\n",
    "]\n",
    "search_user_from_query(browser,special_list,finish_query_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 采集用户主页推文"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/41 [00:00<?, ?it/s]\u001b[A2019-12-31 10:41:00.348 | INFO     | __main__:search_tweet_from_profile_v2:3 - user = ErikSolheim\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41\n"
     ]
    }
   ],
   "source": [
    "finish_user_list = []\n",
    "bad_query_list = []\n",
    "\n",
    "query_user_list = [u for u in user_table.find() if u['query']!='']# query 不为空的user\n",
    "\n",
    "print(len(query_user_list))\n",
    "\n",
    "MAX_TWEET_SIZE = 1000\n",
    "time_interval = \"2019年1月1日\" # 默认截止到今日\n",
    "search_tweet_from_profile_v2(browser,query_user_list,finish_user_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}