forked from luckman666/PythonIsTools
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathrun_api.py
41 lines (33 loc) · 992 Bytes
/
run_api.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 通过框架自带命令 启动命令脚本
@Date :2021/09/20
@Author :xhunmon
@Mail :[email protected]
"""
from scrapy.crawler import CrawlerProcess
from scrapy.utils.log import configure_logging
from scrapy.utils.project import get_project_settings
from amazon.spiders.alibaba import AlibabaSpider
from amazon.spiders.amazon import AmazonSpider
from amazon.spiders.checkip import CheckIpSpider
settings = get_project_settings()
configure_logging(settings)
crawler = CrawlerProcess(settings)
def check_ip():
spider = CheckIpSpider
crawler.crawl(spider)
crawler.start()
return spider.ips
def crawl_amazon(keywords: []):
spider = AmazonSpider
spider.keywords = keywords
crawler.crawl(spider)
crawler.start()
return spider.results
def crawl_alibaba(keywords: []):
spider = AlibabaSpider
spider.keywords = keywords
crawler.crawl(spider)
crawler.start()