Improve proxy support (#205) close #180

* Implement proxy and proxy_auth parameters

* Fix TypedDict import for older versions of python

* Add new parameters to top level request()

* Change https-to-https proxy exception to a warning

* Remove unnecessary __future__ import

curl_cffi/requests/__init__.py

@@ -30,7 +30,7 @@
 from .models import Request, Response
 from .errors import RequestsError
 from .headers import Headers, HeaderTypes
-from .session import AsyncSession, BrowserType, Session
+from .session import AsyncSession, BrowserType, Session, ProxySpec
 from .websockets import WebSocket, WebSocketError, WsCloseCode
 
 # ThreadType = Literal["eventlet", "gevent", None]
@@ -49,7 +49,9 @@ def request(
     timeout: Union[float, Tuple[float, float]] = 30,
     allow_redirects: bool = True,
     max_redirects: int = -1,
-    proxies: Optional[dict] = None,
+    proxies: Optional[ProxySpec] = None,
+    proxy: Optional[str] = None,
+    proxy_auth: Optional[Tuple[str, str]] = None,
     verify: Optional[bool] = None,
     referer: Optional[str] = None,
     accept_encoding: Optional[str] = "gzip, deflate, br",
@@ -78,6 +80,8 @@ def request(
         allow_redirects: whether to allow redirection.
         max_redirects: max redirect counts, default unlimited(-1).
         proxies: dict of proxies to use, format: {"http": proxy_url, "https": proxy_url}.
+        proxy: proxy to use, format: "http://proxy_url". Cannot be used with the above parameter.
+        proxy_auth: HTTP basic auth for proxy, a tuple of (username, password).
         verify: whether to verify https certs.
         referer: shortcut for setting referer header.
         accept_encoding: shortcut for setting accept-encoding header.
@@ -108,6 +112,8 @@ def request(
             allow_redirects=allow_redirects,
             max_redirects=max_redirects,
             proxies=proxies,
+            proxy=proxy,
+            proxy_auth=proxy_auth,
             verify=verify,
             referer=referer,
             accept_encoding=accept_encoding,

curl_cffi/requests/session.py

@@ -8,7 +8,7 @@
 from functools import partialmethod
 from io import BytesIO
 from json import dumps
-from typing import Callable, Dict, List, Any, Optional, Tuple, Union, cast
+from typing import Callable, Dict, List, Any, Optional, Tuple, Union, cast, TYPE_CHECKING
 from urllib.parse import ParseResult, parse_qsl, unquote, urlencode, urlparse
 from concurrent.futures import ThreadPoolExecutor
 
@@ -31,6 +31,19 @@
 except ImportError:
     pass
 
+if TYPE_CHECKING:
+    from typing_extensions import TypedDict
+
+    class ProxySpec(TypedDict, total=False):
+        all: str
+        http: str
+        https: str
+        ws: str
+        wss: str
+
+else:
+    ProxySpec = Dict[str, str]
+
 
 class BrowserType(str, Enum):
     edge99 = "edge99"
@@ -149,7 +162,9 @@ def __init__(
         headers: Optional[HeaderTypes] = None,
         cookies: Optional[CookieTypes] = None,
         auth: Optional[Tuple[str, str]] = None,
-        proxies: Optional[dict] = None,
+        proxies: Optional[ProxySpec] = None,
+        proxy: Optional[str] = None,
+        proxy_auth: Optional[Tuple[str, str]] = None,
         params: Optional[dict] = None,
         verify: bool = True,
         timeout: Union[float, Tuple[float, float]] = 30,
@@ -167,7 +182,6 @@ def __init__(
         self.headers = Headers(headers)
         self.cookies = Cookies(cookies)
         self.auth = auth
-        self.proxies = proxies or {}
         self.params = params
         self.verify = verify
         self.timeout = timeout
@@ -182,6 +196,13 @@ def __init__(
         self.debug = debug
         self.interface = interface
 
+        if proxy and proxies:
+            raise TypeError("Cannot specify both 'proxy' and 'proxies'")
+        if proxy:
+            proxies = {"all": proxy}
+        self.proxies: ProxySpec = proxies or {}
+        self.proxy_auth = proxy_auth
+
     def _set_curl_options(
         self,
         curl,
@@ -197,7 +218,9 @@ def _set_curl_options(
         timeout: Optional[Union[float, Tuple[float, float], object]] = not_set,
         allow_redirects: Optional[bool] = None,
         max_redirects: Optional[int] = None,
-        proxies: Optional[dict] = None,
+        proxies: Optional[ProxySpec] = None,
+        proxy: Optional[str] = None,
+        proxy_auth: Optional[Tuple[str, str]] = None,
         verify: Optional[Union[bool, str]] = None,
         referer: Optional[str] = None,
         accept_encoding: Optional[str] = "gzip, deflate, br",
@@ -336,8 +359,12 @@ def _set_curl_options(
         )
 
         # proxies
-        if self.proxies:
-            proxies = {**self.proxies, **(proxies or {})}
+        if proxy and proxies:
+            raise TypeError("Cannot specify both 'proxy' and 'proxies'")
+        if proxy:
+            proxies = {"all": proxy}
+        if proxies is None:
+            proxies = self.proxies
 
         if proxies:
             parts = urlparse(url)
@@ -350,16 +377,25 @@ def _set_curl_options(
 
             if proxy is not None:
                 if parts.scheme == "https" and proxy.startswith("https://"):
-                    raise RequestsError(
-                        "You are using http proxy WRONG, the prefix should be 'http://' not 'https://',"
-                        "see: https://github.com/yifeikong/curl_cffi/issues/6"
+                    warnings.warn(
+                        "You may be using http proxy WRONG, the prefix should be 'http://' not 'https://',"
+                        "see: https://github.com/yifeikong/curl_cffi/issues/6",
+                        RuntimeWarning,
+                        stacklevel=2,
                     )
 
                 c.setopt(CurlOpt.PROXY, proxy)
                 # for http proxy, need to tell curl to enable tunneling
                 if not proxy.startswith("socks"):
                     c.setopt(CurlOpt.HTTPPROXYTUNNEL, 1)
 
+                # proxy_auth
+                proxy_auth = proxy_auth or self.proxy_auth
+                if proxy_auth:
+                    username, password = proxy_auth
+                    c.setopt(CurlOpt.PROXYUSERNAME, username.encode())
+                    c.setopt(CurlOpt.PROXYPASSWORD, password.encode())
+
         # verify
         if verify is False or not self.verify and verify is None:
             c.setopt(CurlOpt.SSL_VERIFYPEER, 0)
@@ -522,6 +558,8 @@ def __init__(
             cookies: cookies to add in the session.
             auth: HTTP basic auth, a tuple of (username, password), only basic auth is supported.
             proxies: dict of proxies to use, format: {"http": proxy_url, "https": proxy_url}.
+            proxy: proxy to use, format: "http://proxy_url". Cannot be used with the above parameter.
+            proxy_auth: HTTP basic auth for proxy, a tuple of (username, password).
             params: query string for the session.
             verify: whether to verify https certs.
             timeout: how many seconds to wait before giving up. In stream mode, only connect_timeout will be set.
@@ -630,7 +668,9 @@ def request(
         timeout: Optional[Union[float, Tuple[float, float]]] = None,
         allow_redirects: Optional[bool] = None,
         max_redirects: Optional[int] = None,
-        proxies: Optional[dict] = None,
+        proxies: Optional[ProxySpec] = None,
+        proxy: Optional[str] = None,
+        proxy_auth: Optional[Tuple[str, str]] = None,
         verify: Optional[bool] = None,
         referer: Optional[str] = None,
         accept_encoding: Optional[str] = "gzip, deflate, br",
@@ -666,6 +706,8 @@ def request(
             allow_redirects=allow_redirects,
             max_redirects=max_redirects,
             proxies=proxies,
+            proxy=proxy,
+            proxy_auth=proxy_auth,
             verify=verify,
             referer=referer,
             accept_encoding=accept_encoding,
@@ -771,6 +813,8 @@ def __init__(
             cookies: cookies to add in the session.
             auth: HTTP basic auth, a tuple of (username, password), only basic auth is supported.
             proxies: dict of proxies to use, format: {"http": proxy_url, "https": proxy_url}.
+            proxy: proxy to use, format: "http://proxy_url". Cannot be used with the above parameter.
+            proxy_auth: HTTP basic auth for proxy, a tuple of (username, password).
             params: query string for the session.
             verify: whether to verify https certs.
             timeout: how many seconds to wait before giving up.
@@ -885,7 +929,9 @@ async def request(
         timeout: Optional[Union[float, Tuple[float, float]]] = None,
         allow_redirects: Optional[bool] = None,
         max_redirects: Optional[int] = None,
-        proxies: Optional[dict] = None,
+        proxies: Optional[ProxySpec] = None,
+        proxy: Optional[str] = None,
+        proxy_auth: Optional[Tuple[str, str]] = None,
         verify: Optional[bool] = None,
         referer: Optional[str] = None,
         accept_encoding: Optional[str] = "gzip, deflate, br",
@@ -914,6 +960,8 @@ async def request(
             allow_redirects=allow_redirects,
             max_redirects=max_redirects,
             proxies=proxies,
+            proxy=proxy,
+            proxy_auth=proxy_auth,
             verify=verify,
             referer=referer,
             accept_encoding=accept_encoding,