Skip to content

Commit

Permalink
change the parallel request to delay request serial
Browse files Browse the repository at this point in the history
  • Loading branch information
@jiahaoxiang2000
jiahaoxiang2000 committed May 24, 2024
1 parent 8f4e026 commit 25d5669
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 41 deletions.
57 changes: 36 additions & 21 deletions antgst.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"{\"success\":true,\"message\":\"操作成功！\",\"code\":0,\"result\":{\"code\":\"MzRm\",\"key\":\"11fedbcd4982d8aa6904a0e71f0ab2e0\"},\"timestamp\":1713940401752}\n"
"{\"success\":true,\"message\":\"操作成功！\",\"code\":0,\"result\":{\"code\":\"Tht1\",\"key\":\"0cf8cf94ddf7a9c0de379d7ac15b46cc\"},\"timestamp\":1716555274569}\n"
]
}
],
Expand All @@ -102,7 +102,9 @@
"import json\n",
"import time\n",
"\n",
"url_checkKey = \"https://web.antgst.com/antgst/sys/getCheckCode?_t=\" + str(int(time.time()))\n",
"url_checkKey = \"https://web.antgst.com/antgst/sys/getCheckCode?_t=\" + str(\n",
" int(time.time())\n",
")\n",
"\n",
"headers = {\"Content-Type\": \"application/json\"}\n",
"\n",
Expand All @@ -119,8 +121,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"{\"success\":true,\"message\":\"登录成功\",\"code\":200,\"result\":{\"multi_depart\":1,\"userInfo\":{\"id\":\"869cb203724655601f86c46b0ec3a303\",\"username\":\"ANT_JYB\",\"realname\":\"ANT_JYB\",\"passwordTxt\":\"321987qq\",\"googleKey\":null,\"googleBind\":0,\"platformLogo\":\"\",\"platformDomain\":\"\",\"brandName\":\"\",\"avatar\":null,\"birthday\":null,\"sex\":null,\"email\":null,\"phone\":null,\"orgCode\":null,\"status\":1,\"delFlag\":\"0\",\"drTimeout\":0,\"workNo\":null,\"post\":null,\"telephone\":null,\"notes\":null,\"createBy\":\"admin\",\"createTime\":\"2023-11-13 14:23:01\",\"updateBy\":\"ANT_JYB\",\"updateTime\":\"2024-04-02 23:07:52\",\"activitiSync\":null,\"departId\":\"97b21d51ebee4d859bbc3d257d9d94eb\",\"countryCode\":\"0062,0091,0086,00234,0066,0052,0055,0060,001,0020,0081,0065,00886,0084,0063,0090,0082,0061,0064,0044,0033,0049,00852,00853,0092,0027,0057,00880,00254,0051,00971,0034,0054,00212,0056,0030,00966,0048,00593,00261,00507,007,00216,0040,00382,0098,00213,00351,00355,00994,00380,00225,00237,00251,00260,00421,00255,0045,00977,0039,0031,0032,0046,0041,0043,00358,00353,00385,00420,00377,00354,00356,00386,00372,00370,00371,0036,00233,00595,00228,00241,00856,00855,0095,00673,00670,0058,00598,00964,00222,00974,0047,00503,00972,00961,00998,00992,0007,00993,00996,0094,00962,00504,0093,00591,00501,00502,00506,0053,0001,00256,00249,00268,00376,001268,00374,00599,00244,00684,00297,00387,001246,00226,00221,00264,00231,00505,00250,00267,00389,00263,00232,00373,00375,00995,00359,00266,00252,00230,00679,00229,00970,00220,00683,00686,00218,00674,00265,00227,00239,00257,00850,00965,001767,00223,00509,00235,00973,00592,00680,00967,00960,00357,00675,00963,00968,00352,001868,00224,00381,00242,00243,00682,00975,00423,00378,00383,00500,00508,00590,00594,00597,00676,00677,00678,00681,00685,00687,00688,00689,00691,00236,00238,00240,00248,00253,00258,00262,00269,00291,00298,00299,00350,00211,00692,001242,001264,001345,001441,001473,001649,001664,001670,001671,001684,001758,001784,001869,001787,00794,001876,00976,0018\",\"departName\":\"ANT_JYB\",\"isAdmin\":\"1\",\"superAdmin\":null,\"accountType\":0,\"agentId\":null,\"isAgent\":null},\"departs\":[{\"id\":\"97b21d51ebee4d859bbc3d257d9d94eb\",\"parentId\":null,\"departName\":\"ANT_JYB\",\"departNameEn\":null,\"departNameAbbr\":null,\"salesPersonId\":null,\"departOrder\":null,\"description\":null,\"orgCategory\":null,\"orgType\":null,\"orgCode\":null,\"countryCode\":\"0062,0091,0086,00234,0066,0052,0055,0060,001,0020,0081,0065,00886,0084,0063,0090,0082,0061,0064,0044,0033,0049,00852,00853,0092,0027,0057,00880,00254,0051,00971,0034,0054,00212,0056,0030,00966,0048,00593,00261,00507,007,00216,0040,00382,0098,00213,00351,00355,00994,00380,00225,00237,00251,00260,00421,00255,0045,00977,0039,0031,0032,0046,0041,0043,00358,00353,00385,00420,00377,00354,00356,00386,00372,00370,00371,0036,00233,00595,00228,00241,00856,00855,0095,00673,00670,0058,00598,00964,00222,00974,0047,00503,00972,00961,00998,00992,0007,00993,00996,0094,00962,00504,0093,00591,00501,00502,00506,0053,0001,00256,00249,00268,00376,001268,00374,00599,00244,00684,00297,00387,001246,00226,00221,00264,00231,00505,00250,00267,00389,00263,00232,00373,00375,00995,00359,00266,00252,00230,00679,00229,00970,00220,00683,00686,00218,00674,00265,00227,00239,00257,00850,00965,001767,00223,00509,00235,00973,00592,00680,00967,00960,00357,00675,00963,00968,00352,001868,00224,00381,00242,00243,00682,00975,00423,00378,00383,00500,00508,00590,00594,00597,00676,00677,00678,00681,00685,00687,00688,00689,00691,00236,00238,00240,00248,00253,00258,00262,00269,00291,00298,00299,00350,00211,00692,001242,001264,001345,001441,001473,001649,001664,001670,001671,001684,001758,001784,001869,001787,00794,001876,00976,0018\",\"currency\":null,\"mobile\":null,\"fax\":null,\"address\":null,\"memo\":null,\"status\":null,\"delFlag\":null,\"senderTemplateFlag\":null,\"percentage\":0,\"notifyPercentage\":0,\"otpPercentage\":0,\"isEnableMultiPercentage\":0,\"paymentType\":null,\"billingMethod\":0,\"createBy\":null,\"createTime\":null,\"updateBy\":null,\"updateTime\":null,\"vssSmsMarketingFeerateGroupId\":null,\"vssSmsNotifyFeerateGroupId\":null,\"vssSmsOtpPremiumFeerateGroupId\":null,\"vssSmsOtpRegulerFeerateGroupId\":null,\"vssVoipFeerateGroupId\":null,\"vssVoipIvrFeerateGroupId\":null,\"vssVoipCheckStatusGroupId\":null,\"vssSmsMarketingDrFakeRate\":null,\"vssSmsNotifyDrFakeRate\":null,\"vssSmsOtpPremiumDrFakeRate\":null,\"vssSmsOtpRegulerDrFakeRate\":null,\"smsBalance\":null,\"smsOverdraftAmount\":0.0,\"voipBalance\":null,\"voipOverdraftAmount\":0.0,\"vssNumberDetectionFeerateGroupId\":null,\"remindAmount\":0.0,\"otpContentVerification\":null,\"agentId\":null,\"isAgent\":null,\"userPassword\":null}],\"token\":\"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MTM5NDQwMDMsInVzZXJuYW1lIjoiQU5UX0pZQiJ9.zQbtVleA6K03cIEWQtvjiw4NmqDFHjFtfqZ238pNLSk\"},\"timestamp\":1713940403217}\n",
"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MTM5NDQwMDMsInVzZXJuYW1lIjoiQU5UX0pZQiJ9.zQbtVleA6K03cIEWQtvjiw4NmqDFHjFtfqZ238pNLSk\n"
"{\"success\":true,\"message\":\"登录成功\",\"code\":200,\"result\":{\"multi_depart\":1,\"userInfo\":{\"id\":\"794dc021ec2f04d61785104f9190b76e\",\"username\":\"456bet\",\"realname\":\"456bet\",\"googleBind\":0,\"platformLogo\":\"\",\"platformDomain\":\"\",\"brandName\":\"\",\"avatar\":null,\"birthday\":null,\"sex\":null,\"email\":null,\"phone\":null,\"orgCode\":null,\"status\":1,\"delFlag\":\"0\",\"drTimeout\":0,\"workNo\":null,\"post\":null,\"telephone\":null,\"notes\":null,\"createBy\":\"ANT_LLT\",\"createTime\":\"2024-03-24 21:06:31\",\"updateBy\":\"456bet\",\"updateTime\":\"2024-03-24 22:11:18\",\"activitiSync\":\"1\",\"departId\":\"84a4a736e56c4e9aa5a665bef41b237b\",\"countryCode\":\"0055\",\"departName\":\"456bet\",\"isAdmin\":\"0\",\"superAdmin\":null,\"accountType\":0,\"agentId\":\"\",\"isAgent\":\"0\",\"ipWhite\":null},\"departs\":[{\"id\":\"84a4a736e56c4e9aa5a665bef41b237b\",\"parentId\":null,\"departName\":\"456bet\",\"departNameEn\":null,\"departNameAbbr\":null,\"salesPersonId\":null,\"departOrder\":null,\"description\":null,\"orgCategory\":null,\"orgType\":null,\"orgCode\":null,\"countryCode\":\"0055\",\"currency\":null,\"mobile\":null,\"fax\":null,\"address\":null,\"memo\":null,\"status\":null,\"delFlag\":null,\"senderTemplateFlag\":null,\"percentage\":0,\"notifyPercentage\":0,\"otpPercentage\":0,\"isEnableMultiPercentage\":0,\"paymentType\":null,\"billingMethod\":0,\"createBy\":null,\"createTime\":null,\"updateBy\":null,\"updateTime\":null,\"vssSmsMarketingFeerateGroupId\":null,\"vssSmsNotifyFeerateGroupId\":null,\"vssSmsOtpPremiumFeerateGroupId\":null,\"vssSmsOtpRegulerFeerateGroupId\":null,\"vssVoipFeerateGroupId\":null,\"vssVoipIvrFeerateGroupId\":null,\"vssVoipCheckStatusGroupId\":null,\"vssSmsMarketingDrFakeRate\":null,\"vssSmsNotifyDrFakeRate\":null,\"vssSmsOtpPremiumDrFakeRate\":null,\"vssSmsOtpRegulerDrFakeRate\":null,\"smsBalance\":null,\"smsOverdraftAmount\":0.0,\"voipBalance\":null,\"voipOverdraftAmount\":0.0,\"vssNumberDetectionFeerateGroupId\":null,\"remindAmount\":0.0,\"otpContentVerification\":null,\"yuloreFlag\":null,\"agentId\":null,\"isAgent\":null,\"userPassword\":null}],\"token\":\"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MTY1NTg4ODAsInVzZXJuYW1lIjoiNDU2YmV0In0.YxNhulj0IgqtFE2abMpUcVs5GXiJIyQu0sLcdfSusmk\"},\"timestamp\":1716555280368}\n",
"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE3MTY1NTg4ODAsInVzZXJuYW1lIjoiNDU2YmV0In0.YxNhulj0IgqtFE2abMpUcVs5GXiJIyQu0sLcdfSusmk\n"
]
}
],
Expand Down Expand Up @@ -148,8 +150,8 @@
"\n",
"# Define the data for the POST request\n",
"data = {\n",
" \"username\": \"ANT_JYB\",\n",
" \"password\": \"321987qq\",\n",
" \"username\": \"456bet\",\n",
" \"password\": \"456bet888\",\n",
" \"remember_me\": True,\n",
" \"captcha\": code,\n",
" \"checkKey\": key,\n",
Expand All @@ -167,7 +169,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -190,7 +192,7 @@
"def fetch_data(\n",
" token,\n",
" option={},\n",
" base_url=\"https://web.antgst.com/antgst/sms/marketing/sendRecordList\",\n",
" base_url=\"https://web.antgst.com/antgst/sms/otpPremium/channel/sendRecordList\",\n",
"):\n",
" timestamp = int(time.time())\n",
" today = date.today()\n",
Expand All @@ -200,8 +202,8 @@
" \"column\": \"createTime\",\n",
" \"order\": \"desc\",\n",
" \"gatewayDr\": \"000\",\n",
" \"field\": \"countryName,smsTo,sendTime\",\n",
" \"pageNo\": 1\n",
" \"field\": \"id,,departName,paymentType,orgCode,countryName,smsBalance,voipBalance,smsFrozenAmount,userPassword,smsOverdraftAmount,ndOverdraftAmount,voipOverdraftAmount,currency,salesPersonId,memo,createTime,createBy,updateTime,updateBy,action\",\n",
" \"pageNo\": 1,\n",
" }\n",
" # Combine the dictionaries\n",
" query = {\n",
Expand All @@ -226,7 +228,7 @@
" return pages\n",
"\n",
"\n",
"response = fetch_data(token, option={\"pageSize\": 50000,\"countryCode\": \"00234\"})"
"response = fetch_data(token, option={\"pageSize\": 100})"
]
},
{
Expand All @@ -238,56 +240,59 @@
"name": "stdout",
"output_type": "stream",
"text": [
"5654\n"
"100\n"
]
}
],
"source": [
"import json\n",
"\n",
"# Assuming response is a JSON string\n",
"json_str = response\n",
"json_str = response[0]\n",
"\n",
"# Convert JSON string to Python object\n",
"data = json.loads(json_str)\n",
"\n",
"print(len(data.get('result').get('records')))"
"print(len(data.get(\"result\").get(\"records\")))"
]
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"res = list(map(lambda x: json.loads(x).get(\"result\").get(\"records\"), response))\n",
"df = pd.concat([pd.DataFrame(item) for item in res])\n",
"\n",
"# Write the DataFrame to an Excel file\n",
"import os\n",
"\n",
"\n",
"def get_unique_filename(base_filename, extension):\n",
" counter = 1\n",
" while os.path.isfile(f\"{base_filename}{counter}.{extension}\"):\n",
" counter += 1\n",
" return f\"{base_filename}{counter}.{extension}\"\n",
"\n",
"\n",
"csv_filename = get_unique_filename(\"output\", \"csv\")\n",
"df.to_csv(csv_filename, index=False)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"00234706***3075\n",
"2024-04-22 22:43:50\n"
"005587988462524\n",
"2024-05-24 20:55:20\n"
]
}
],
Expand Down Expand Up @@ -331,12 +336,14 @@
"# Write the DataFrame to an Excel file\n",
"import os\n",
"\n",
"\n",
"def get_unique_filename(base_filename, extension):\n",
" counter = 1\n",
" while os.path.isfile(f\"{base_filename}{counter}.{extension}\"):\n",
" counter += 1\n",
" return f\"{base_filename}{counter}.{extension}\"\n",
"\n",
"\n",
"csv_filename = get_unique_filename(\"output\", \"csv\")\n",
"df.to_csv(csv_filename, index=False)"
]
Expand All @@ -355,10 +362,18 @@
}
],
"source": [
"option = {\"pageSize\":2000} \n",
"base_query ={\"_t\": 1, \"day\": 2, \"column\": \"createTime\", \"order\": \"desc\", \"field\": \"countryName,smsTo,sendTime\", \"pageNo\": 1, \"pageSize\": 1000}\n",
"option = {\"pageSize\": 2000}\n",
"base_query = {\n",
" \"_t\": 1,\n",
" \"day\": 2,\n",
" \"column\": \"createTime\",\n",
" \"order\": \"desc\",\n",
" \"field\": \"countryName,smsTo,sendTime\",\n",
" \"pageNo\": 1,\n",
" \"pageSize\": 1000,\n",
"}\n",
"# Combine the dictionaries\n",
"query = {**base_query, **option} \n",
"query = {**base_query, **option}\n",
"print(query)"
]
}
Expand Down
36 changes: 23 additions & 13 deletions antgst.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ def login(code, key, options={}):
# Define the data for the POST request
# TODO: need to change the username and password by user. no hard code
data = {
"username": "ANT_JYB",
"password": "321987qq",
"username": "456bet",
"password": "456bet888",
"remember_me": True,
"captcha": code,
"checkKey": key,
Expand Down Expand Up @@ -87,20 +87,30 @@ def fetch_data(
"pageSize": 100,
}

total_pages = int(option["pageSize"]) // 100
total_pages = 1
pages = []
# Get the total number of pages
page = fetch_url((base_url, query, 1, token))
total_pages = json.loads(page).get("result").get("pages")
# by the total number of pages, get all the pages
# we let the fetch_url function request 0.5 s per page to avoid the request being blocked
for i in range(1, total_pages + 1):
print(f"Fetching page {i} of {total_pages}")
pages.append(fetch_url((base_url, query, i, token)))
time.sleep(0.5)

return pages

# Create the URL
# Create a ThreadPoolExecutor
with concurrent.futures.ThreadPoolExecutor() as executor:
# Use the executor to fetch all pages
pages = list(
executor.map(
fetch_url,
[(base_url, query, i, token) for i in range(1, total_pages + 1)],
)
)

return pages
# with concurrent.futures.ThreadPoolExecutor() as executor:
# # Use the executor to fetch all pages
# pages = list(
# executor.map(
# fetch_url,
# [(base_url, query, i, token) for i in range(1, total_pages + 1)],
# )
# )


def save_data(response):
Expand Down
14 changes: 7 additions & 7 deletions app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
}

sms = {
"market": "https://web.antgst.com/antgst/sms/marketing/sendRecordList",
"otp": "https://web.antgst.com/antgst/sms/otpPremium/sendRecordList",
"market": "https://web.antgst.com/antgst/sms/marketing/sendRecordList",
}


Expand All @@ -33,9 +33,9 @@ def submit():
country = country_list.get(country_list.curselection())
if country != "ALL":
option = {**option, "countryCode": countryCode[country]}
page_size = page_size_entry.get()
if page_size != "":
option = {**option, "pageSize": page_size}
date = date_entry.get()
if date != "":
option = {**option, "day": date}
sms_type = sms_list.get()
print(user, option, sms_type)
base_url = sms[sms_type]
Expand All @@ -52,7 +52,7 @@ def submit():

tk.Label(root, text="Username").grid(row=0)
tk.Label(root, text="Password").grid(row=1)
tk.Label(root, text="Data size").grid(row=2)
tk.Label(root, text="Date").grid(row=2)

# Create the status label
status_label = tk.Label(root, text="")
Expand All @@ -72,13 +72,13 @@ def submit():

username_entry = tk.Entry(root)
password_entry = tk.Entry(root, show="*")
page_size_entry = tk.Entry(root)
date_entry = tk.Entry(root)

fetch_button = tk.Button(root, text="Fetch", command=submit)

username_entry.grid(row=0, column=1)
password_entry.grid(row=1, column=1)
page_size_entry.grid(row=2, column=1)
date_entry.grid(row=2, column=1)
sms_list.grid(row=4, column=1)
country_list.grid(row=5, column=0, sticky="nsew")
scrollbar.grid(row=5, column=1, sticky="ns")
Expand Down

0 comments on commit 25d5669

Please sign in to comment.