diff --git a/curl_cffi/requests/__init__.py b/curl_cffi/requests/__init__.py index df1d9317..aa95cd0b 100644 --- a/curl_cffi/requests/__init__.py +++ b/curl_cffi/requests/__init__.py @@ -30,7 +30,7 @@ from .models import Request, Response from .errors import RequestsError from .headers import Headers, HeaderTypes -from .session import AsyncSession, BrowserType, Session +from .session import AsyncSession, BrowserType, Session, ProxySpec from .websockets import WebSocket, WebSocketError, WsCloseCode # ThreadType = Literal["eventlet", "gevent", None] @@ -49,7 +49,9 @@ def request( timeout: Union[float, Tuple[float, float]] = 30, allow_redirects: bool = True, max_redirects: int = -1, - proxies: Optional[dict] = None, + proxies: Optional[ProxySpec] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = None, referer: Optional[str] = None, accept_encoding: Optional[str] = "gzip, deflate, br", @@ -78,6 +80,8 @@ def request( allow_redirects: whether to allow redirection. max_redirects: max redirect counts, default unlimited(-1). proxies: dict of proxies to use, format: {"http": proxy_url, "https": proxy_url}. + proxy: proxy to use, format: "http://proxy_url". Cannot be used with the above parameter. + proxy_auth: HTTP basic auth for proxy, a tuple of (username, password). verify: whether to verify https certs. referer: shortcut for setting referer header. accept_encoding: shortcut for setting accept-encoding header. @@ -108,6 +112,8 @@ def request( allow_redirects=allow_redirects, max_redirects=max_redirects, proxies=proxies, + proxy=proxy, + proxy_auth=proxy_auth, verify=verify, referer=referer, accept_encoding=accept_encoding, diff --git a/curl_cffi/requests/session.py b/curl_cffi/requests/session.py index 8cf3f07c..dc54ce43 100644 --- a/curl_cffi/requests/session.py +++ b/curl_cffi/requests/session.py @@ -8,7 +8,7 @@ from functools import partialmethod from io import BytesIO from json import dumps -from typing import Callable, Dict, List, Any, Optional, Tuple, Union, cast +from typing import Callable, Dict, List, Any, Optional, Tuple, Union, cast, TYPE_CHECKING from urllib.parse import ParseResult, parse_qsl, unquote, urlencode, urlparse from concurrent.futures import ThreadPoolExecutor @@ -31,6 +31,19 @@ except ImportError: pass +if TYPE_CHECKING: + from typing_extensions import TypedDict + + class ProxySpec(TypedDict, total=False): + all: str + http: str + https: str + ws: str + wss: str + +else: + ProxySpec = Dict[str, str] + class BrowserType(str, Enum): edge99 = "edge99" @@ -149,7 +162,9 @@ def __init__( headers: Optional[HeaderTypes] = None, cookies: Optional[CookieTypes] = None, auth: Optional[Tuple[str, str]] = None, - proxies: Optional[dict] = None, + proxies: Optional[ProxySpec] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, params: Optional[dict] = None, verify: bool = True, timeout: Union[float, Tuple[float, float]] = 30, @@ -167,7 +182,6 @@ def __init__( self.headers = Headers(headers) self.cookies = Cookies(cookies) self.auth = auth - self.proxies = proxies or {} self.params = params self.verify = verify self.timeout = timeout @@ -182,6 +196,13 @@ def __init__( self.debug = debug self.interface = interface + if proxy and proxies: + raise TypeError("Cannot specify both 'proxy' and 'proxies'") + if proxy: + proxies = {"all": proxy} + self.proxies: ProxySpec = proxies or {} + self.proxy_auth = proxy_auth + def _set_curl_options( self, curl, @@ -197,7 +218,9 @@ def _set_curl_options( timeout: Optional[Union[float, Tuple[float, float], object]] = not_set, allow_redirects: Optional[bool] = None, max_redirects: Optional[int] = None, - proxies: Optional[dict] = None, + proxies: Optional[ProxySpec] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, verify: Optional[Union[bool, str]] = None, referer: Optional[str] = None, accept_encoding: Optional[str] = "gzip, deflate, br", @@ -336,8 +359,12 @@ def _set_curl_options( ) # proxies - if self.proxies: - proxies = {**self.proxies, **(proxies or {})} + if proxy and proxies: + raise TypeError("Cannot specify both 'proxy' and 'proxies'") + if proxy: + proxies = {"all": proxy} + if proxies is None: + proxies = self.proxies if proxies: parts = urlparse(url) @@ -350,9 +377,11 @@ def _set_curl_options( if proxy is not None: if parts.scheme == "https" and proxy.startswith("https://"): - raise RequestsError( - "You are using http proxy WRONG, the prefix should be 'http://' not 'https://'," - "see: https://github.com/yifeikong/curl_cffi/issues/6" + warnings.warn( + "You may be using http proxy WRONG, the prefix should be 'http://' not 'https://'," + "see: https://github.com/yifeikong/curl_cffi/issues/6", + RuntimeWarning, + stacklevel=2, ) c.setopt(CurlOpt.PROXY, proxy) @@ -360,6 +389,13 @@ def _set_curl_options( if not proxy.startswith("socks"): c.setopt(CurlOpt.HTTPPROXYTUNNEL, 1) + # proxy_auth + proxy_auth = proxy_auth or self.proxy_auth + if proxy_auth: + username, password = proxy_auth + c.setopt(CurlOpt.PROXYUSERNAME, username.encode()) + c.setopt(CurlOpt.PROXYPASSWORD, password.encode()) + # verify if verify is False or not self.verify and verify is None: c.setopt(CurlOpt.SSL_VERIFYPEER, 0) @@ -522,6 +558,8 @@ def __init__( cookies: cookies to add in the session. auth: HTTP basic auth, a tuple of (username, password), only basic auth is supported. proxies: dict of proxies to use, format: {"http": proxy_url, "https": proxy_url}. + proxy: proxy to use, format: "http://proxy_url". Cannot be used with the above parameter. + proxy_auth: HTTP basic auth for proxy, a tuple of (username, password). params: query string for the session. verify: whether to verify https certs. timeout: how many seconds to wait before giving up. In stream mode, only connect_timeout will be set. @@ -630,7 +668,9 @@ def request( timeout: Optional[Union[float, Tuple[float, float]]] = None, allow_redirects: Optional[bool] = None, max_redirects: Optional[int] = None, - proxies: Optional[dict] = None, + proxies: Optional[ProxySpec] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = None, referer: Optional[str] = None, accept_encoding: Optional[str] = "gzip, deflate, br", @@ -666,6 +706,8 @@ def request( allow_redirects=allow_redirects, max_redirects=max_redirects, proxies=proxies, + proxy=proxy, + proxy_auth=proxy_auth, verify=verify, referer=referer, accept_encoding=accept_encoding, @@ -771,6 +813,8 @@ def __init__( cookies: cookies to add in the session. auth: HTTP basic auth, a tuple of (username, password), only basic auth is supported. proxies: dict of proxies to use, format: {"http": proxy_url, "https": proxy_url}. + proxy: proxy to use, format: "http://proxy_url". Cannot be used with the above parameter. + proxy_auth: HTTP basic auth for proxy, a tuple of (username, password). params: query string for the session. verify: whether to verify https certs. timeout: how many seconds to wait before giving up. @@ -885,7 +929,9 @@ async def request( timeout: Optional[Union[float, Tuple[float, float]]] = None, allow_redirects: Optional[bool] = None, max_redirects: Optional[int] = None, - proxies: Optional[dict] = None, + proxies: Optional[ProxySpec] = None, + proxy: Optional[str] = None, + proxy_auth: Optional[Tuple[str, str]] = None, verify: Optional[bool] = None, referer: Optional[str] = None, accept_encoding: Optional[str] = "gzip, deflate, br", @@ -914,6 +960,8 @@ async def request( allow_redirects=allow_redirects, max_redirects=max_redirects, proxies=proxies, + proxy=proxy, + proxy_auth=proxy_auth, verify=verify, referer=referer, accept_encoding=accept_encoding,