merge upstream

.github/workflows/build-and-test.yaml

@@ -19,14 +19,14 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - run: |
           pip install mypy ruff
           make lint
-          
+
   sdist:
     name: Build sdist wheel
     runs-on: ubuntu-latest
@@ -35,7 +35,7 @@ jobs:
       - run: |
           make preprocess
           pipx run build --sdist
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v3  # https://github.com/actions/upload-artifact/issues/478
         with:
           path: ./dist/*.tar.gz
 
@@ -59,44 +59,43 @@ jobs:
           python-version: '3.10'
 
       - if: runner.os == 'Linux'
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
         with:
           platforms: all
 
       # macOS make is too old
       - if: runner.os == 'macOS'
         run: |
           brew install make automake libtool
-          which pipx || brew install pipx && pipx ensurepath
 
       - name: Build and test wheels
-        uses: pypa/cibuildwheel@v2.16.5
+        uses: pypa/cibuildwheel@v2.17.0
 
       # - name: Setup tmate session
       #   uses: mxschmitt/action-tmate@v3
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v3  # https://github.com/actions/upload-artifact/issues/478
         with:
           path: ./wheelhouse/*.whl
 
   upload_all:
     needs: [bdist, sdist]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v3  # https://github.com/actions/upload-artifact/issues/478
       if: startsWith(github.ref, 'refs/tags/')
       with:
         name: artifact
         path: dist
 
-    - uses: pypa/gh-action-pypi-publish@v1.5.0
+    - uses: pypa/gh-action-pypi-publish@v1.8.14
       if: startsWith(github.ref, 'refs/tags/')
       with:
         password: ${{ secrets.PYPI_TOKEN }}
 
     - name: Upload release files
       if: startsWith(github.ref, 'refs/tags/')
-      uses: softprops/action-gh-release@v1
+      uses: softprops/action-gh-release@v2
       with:
         files: |
           ./dist/*.whl

Makefile

@@ -2,8 +2,8 @@
 SHELL := bash
 
 # this is the upstream libcurl-impersonate version
-VERSION := 0.6.2b2
-CURL_VERSION := curl-8.1.1
+VERSION := 0.7.0b4
+CURL_VERSION := curl-8.5.0
 
 $(CURL_VERSION):
 	curl -L "https://curl.se/download/$(CURL_VERSION).tar.xz" \
@@ -42,7 +42,7 @@ build: .preprocessed
 lint:
 	ruff check
 	ruff format --diff
-	mypy .
+	mypy --install-types --non-interactive .
 
 format:
 	ruff check --fix

README.md

@@ -122,6 +122,9 @@ Supported impersonate versions, as supported by my [fork](https://github.com/yif
 
 However, only Chrome-like browsers are supported. Firefox support is tracked in [#59](https://github.com/yifeikong/curl_cffi/issues/59).
 
+Browser versions will be added **only** when their fingerprints change. If you see a version, e.g.
+chrome122, were skipped, you can simply impersonate it with your own headers and the previous version.
+
 - chrome99
 - chrome100
 - chrome101
@@ -131,6 +134,8 @@ However, only Chrome-like browsers are supported. Firefox support is tracked in
 - chrome116 <sup>[1]</sup>
 - chrome119 <sup>[1]</sup>
 - chrome120 <sup>[1]</sup>
+- chrome123 <sup>[3]</sup>
+- chrome124 <sup>[3]</sup>
 - chrome99_android
 - edge99
 - edge101
@@ -142,6 +147,7 @@ However, only Chrome-like browsers are supported. Firefox support is tracked in
 Notes:
 1. Added in version `0.6.0`.
 2. Fixed in version `0.6.0`, previous http2 fingerprints were [not correct](https://github.com/lwthiker/curl-impersonate/issues/215).
+3. Added in version `0.7.0`.
 
 ### asyncio
 

benchmark/benchmark.py

@@ -69,7 +69,7 @@ def __del__(self):
         stats[name] = dur
         results.append({"name": name, "size": size, "duration": dur})
 
-    print("One worker, {}: ".format(size), stats)
+    print(f"One worker, {size}: {stats}")
 
 df = pd.DataFrame(results)
 df.to_csv("single_worker.csv", index=False, float_format="%.4f")
@@ -133,7 +133,7 @@ async def httpx_worker(q, done, s):
             t.join()
     # print(stats)
 
-    async def test_asyncs_workers():
+    async def test_asyncs_workers(url, size, stats):
         for name, worker, SessionClass in [
             ("aiohttp", aiohttp_worker, aiohttp.ClientSession),
             ("httpx_async", httpx_worker, httpx.AsyncClient),
@@ -157,8 +157,8 @@ async def test_asyncs_workers():
             for w in workers:
                 w.cancel()
 
-    asyncio.run(test_asyncs_workers())
-    print("10 Workers, {}: ".format(size), stats)
+    asyncio.run(test_asyncs_workers(url, size, stats))
+    print(f"10 Workers, {size}: {stats}")
 
 df = pd.DataFrame(results)
 df.to_csv("multiple_workers.csv", index=False, float_format="%.4f")

curl_cffi/aio.py

@@ -38,7 +38,7 @@ def _get_selector(asyncio_loop) -> asyncio.AbstractEventLoop:
         ):
             return asyncio_loop
 
-        warnings.warn(PROACTOR_WARNING, RuntimeWarning)
+        warnings.warn(PROACTOR_WARNING, RuntimeWarning, stacklevel=2)
 
         from ._asyncio_selector import AddThreadSelectorEventLoop
 
@@ -203,7 +203,7 @@ def socket_action(self, sockfd: int, ev_bitmask: int) -> int:
     def process_data(self, sockfd: int, ev_bitmask: int):
         """Call curl_multi_info_read to read data for given socket."""
         if not self._curlm:
-            warnings.warn("Curlm alread closed! quitting from process_data")
+            warnings.warn("Curlm alread closed! quitting from process_data", stacklevel=2)
             return
 
         self.socket_action(sockfd, ev_bitmask)

curl_cffi/curl.py

@@ -69,11 +69,11 @@ def write_callback(ptr, size, nmemb, userdata):
     callback = ffi.from_handle(userdata)
     wrote = callback(ffi.buffer(ptr, nmemb)[:])
     wrote = ensure_int(wrote)
-    if wrote == CURL_WRITEFUNC_PAUSE or wrote == CURL_WRITEFUNC_ERROR:
+    if wrote == CURL_WRITEFUNC_PAUSE or wrote == CURL_WRITEFUNC_ERROR:  # noqa: SIM109
         return wrote
     # should make this an exception in future versions
     if wrote != nmemb * size:
-        warnings.warn("Wrote bytes != received bytes.", RuntimeWarning)
+        warnings.warn("Wrote bytes != received bytes.", RuntimeWarning, stacklevel=2)
     return nmemb * size
 
 
@@ -101,7 +101,7 @@ def __init__(self, cacert: str = "", debug: bool = False, handle=None) -> None:
             debug: whether to show curl debug messages.
             handle: a curl handle instance from ``curl_easy_init``.
         """
-        self._curl = lib.curl_easy_init() if not handle else handle
+        self._curl = handle if handle else lib.curl_easy_init()
         self._headers = ffi.NULL
         self._proxy_headers = ffi.NULL
         self._resolve = ffi.NULL
@@ -118,7 +118,7 @@ def __init__(self, cacert: str = "", debug: bool = False, handle=None) -> None:
     def _set_error_buffer(self) -> None:
         ret = lib._curl_easy_setopt(self._curl, CurlOpt.ERRORBUFFER, self._error_buffer)
         if ret != 0:
-            warnings.warn("Failed to set error buffer")
+            warnings.warn("Failed to set error buffer", stacklevel=2)
         if self._debug:
             self.setopt(CurlOpt.VERBOSE, 1)
             lib._curl_easy_setopt(self._curl, CurlOpt.DEBUGFUNCTION, lib.debug_function)
@@ -189,10 +189,7 @@ def setopt(self, option: CurlOpt, value: Any) -> int:
             lib._curl_easy_setopt(self._curl, CurlOpt.WRITEFUNCTION, lib.write_callback)
             option = CurlOpt.HEADERDATA
         elif value_type == "char*":
-            if isinstance(value, str):
-                c_value = value.encode()
-            else:
-                c_value = value
+            c_value = value.encode() if isinstance(value, str) else value
             # Must keep a reference, otherwise may be GCed.
             if option == CurlOpt.POSTFIELDS:
                 self._body_handle = c_value
@@ -274,6 +271,8 @@ def _ensure_cacert(self) -> None:
         if not self._is_cert_set:
             ret = self.setopt(CurlOpt.CAINFO, self._cacert)
             self._check_error(ret, "set cacert")
+            ret = self.setopt(CurlOpt.PROXY_CAINFO, self._cacert)
+            self._check_error(ret, "set proxy cacert")
 
     def perform(self, clear_headers: bool = True) -> None:
         """Wrapper for ``curl_easy_perform``, performs a curl request.

cycurl/__version__.py

@@ -6,5 +6,5 @@
 # __description__ = metadata.metadata("curl_cffi")["Summary"]
 # __version__ = metadata.version("curl_cffi")
 __description__ = "libcurl cython bindings for Python, with impersonation support"
-__version__ = "0.6.3b1"
+__version__ = "0.7.0b4"
 __curl_version__ = Curl().version().decode()

cycurl/_asyncio_selector.pxi

@@ -15,6 +15,7 @@ import select
 import socket
 import threading
 import typing
+from contextlib import suppress
 from typing import (
     Any,
     Callable,
@@ -48,10 +49,8 @@ def _atexit_callback() -> None:
         with loop._select_cond:
             loop._closing_selector = True
             loop._select_cond.notify()
-        try:
+        with suppress(BlockingIOError):
             loop._waker_w.send(b"a")
-        except BlockingIOError:
-            pass
         if loop._thread is not None:
             # If we don't join our (daemon) thread here, we may get a deadlock
             # during interpreter shutdown. I don't really understand why. This
@@ -152,16 +151,12 @@ class SelectorThread:
     def _wake_selector(self) -> None:
         if self._closed:
             return
-        try:
+        with suppress(BlockingIOError):
             self._waker_w.send(b"a")
-        except BlockingIOError:
-            pass
 
     def _consume_waker(self) -> None:
-        try:
+        with suppress(BlockingIOError):
             self._waker_r.recv(1024)
-        except BlockingIOError:
-            pass
 
     def _start_select(self) -> None:
         # Capture reader and writer sets here in the event loop