From 60c669bedb16bc9cfa432ee815e2fe5a7411166f Mon Sep 17 00:00:00 2001 From: Cursor Date: Sun, 5 Apr 2026 04:40:57 +0000 Subject: [PATCH 01/14] Refactor Python examples to use HTTP libraries directly Remove python-proxy-headers and scrapy-proxy-headers usage. Add per-library *-proxy.py scripts matching the Ruby/JS/PHP style, plus python/requirements.txt. Update run_tests.py to match (including Scrapy via runspider and optional skips when pycurl or scrapy imports fail). Refresh README Python section. Made-with: Cursor --- README.md | 62 ++++----- python/aiohttp-proxy-headers.py | 48 ------- python/aiohttp-proxy.py | 39 ++++++ python/autoscraper-proxy-headers.py | 45 ------- python/autoscraper-proxy.py | 39 ++++++ python/cloudscraper-proxy-headers.py | 42 ------ python/cloudscraper-proxy.py | 36 ++++++ python/httpx-async-proxy-headers.py | 53 -------- python/httpx-async-proxy.py | 38 ++++++ python/httpx-proxy-headers.py | 45 ------- python/httpx-proxy.py | 31 +++++ python/pycurl-proxy-headers-lowlevel.py | 57 --------- python/pycurl-proxy-headers.py | 38 ------ python/pycurl-proxy-lowlevel.py | 48 +++++++ python/pycurl-proxy.py | 44 +++++++ python/requests-proxy-headers-session.py | 40 ------ python/requests-proxy-headers.py | 39 ------ python/requests-proxy.py | 14 +- python/requests-session-proxy.py | 37 ++++++ python/requirements.txt | 12 ++ python/run_tests.py | 156 ++++++++++++++--------- python/scrapy-proxy-headers.py | 12 -- python/scrapy-proxy.py | 58 +++++++++ python/urllib3-proxy-headers.py | 39 ------ python/urllib3-proxy.py | 36 ++++++ 25 files changed, 551 insertions(+), 557 deletions(-) delete mode 100755 python/aiohttp-proxy-headers.py create mode 100644 python/aiohttp-proxy.py delete mode 100755 python/autoscraper-proxy-headers.py create mode 100644 python/autoscraper-proxy.py delete mode 100755 python/cloudscraper-proxy-headers.py create mode 100644 python/cloudscraper-proxy.py delete mode 100755 python/httpx-async-proxy-headers.py create mode 100644 python/httpx-async-proxy.py delete mode 100755 python/httpx-proxy-headers.py create mode 100644 python/httpx-proxy.py delete mode 100755 python/pycurl-proxy-headers-lowlevel.py delete mode 100755 python/pycurl-proxy-headers.py create mode 100644 python/pycurl-proxy-lowlevel.py create mode 100644 python/pycurl-proxy.py delete mode 100755 python/requests-proxy-headers-session.py delete mode 100755 python/requests-proxy-headers.py create mode 100644 python/requests-session-proxy.py create mode 100644 python/requirements.txt delete mode 100755 python/scrapy-proxy-headers.py create mode 100644 python/scrapy-proxy.py delete mode 100755 python/urllib3-proxy-headers.py create mode 100644 python/urllib3-proxy.py diff --git a/README.md b/README.md index bc35894..eb2db3d 100644 --- a/README.md +++ b/README.md @@ -9,69 +9,57 @@ Example code for using proxy servers in different programming languages. Current ## Python Proxy Examples -### Using python-proxy-headers - -The [python-proxy-headers](https://github.com/proxymesh/python-proxy-headers) library enables sending custom headers to proxy servers and receiving proxy response headers. This is essential for services like [ProxyMesh](https://proxymesh.com) that use custom headers for country selection and IP assignment. - **Installation:** ```bash -pip install python-proxy-headers +pip install -r python/requirements.txt ``` -**Running Examples:** +`pycurl` needs libcurl and `curl-config` (for example Debian/Ubuntu: `libcurl4-openssl-dev`). The test runner skips `pycurl-*` examples when `pycurl` is not installed, and skips `scrapy-proxy` when `import scrapy` fails (for example a broken `cryptography` / `cffi` install). -All examples read proxy configuration from environment variables: +**Running Examples:** ```bash # Required: Set your proxy URL export PROXY_URL='http://user:pass@proxy.example.com:8080' -# Optional: Custom test URL (default: https://api.ipify.org?format=json) +# Optional: Target URL (default: https://api.ipify.org?format=json) export TEST_URL='https://httpbin.org/ip' -# Optional: Send a custom header to the proxy -export PROXY_HEADER='X-ProxyMesh-Country' -export PROXY_VALUE='US' - -# Optional: Read a specific header from the response +# Optional: Print one response header export RESPONSE_HEADER='X-ProxyMesh-IP' -# Run a single example -python python/requests-proxy-headers.py +# Single example +python python/requests-proxy.py -# Run all examples as tests +# All examples as tests python python/run_tests.py -# Run specific examples -python python/run_tests.py requests-proxy-headers httpx-proxy-headers +# Specific examples (substring match, like the JS runner) +python python/run_tests.py requests httpx ``` **Examples:** | Library | Example | Description | |---------|---------|-------------| -| [requests](https://docs.python-requests.org/) | [requests-proxy-headers.py](python/requests-proxy-headers.py) | Simple HTTP requests with proxy headers | -| [requests](https://docs.python-requests.org/) | [requests-proxy-headers-session.py](python/requests-proxy-headers-session.py) | Session-based requests for connection pooling | -| [urllib3](https://urllib3.readthedocs.io/) | [urllib3-proxy-headers.py](python/urllib3-proxy-headers.py) | Low-level HTTP client with proxy headers | -| [aiohttp](https://docs.aiohttp.org/) | [aiohttp-proxy-headers.py](python/aiohttp-proxy-headers.py) | Async HTTP client with proxy headers | -| [httpx](https://www.python-httpx.org/) | [httpx-proxy-headers.py](python/httpx-proxy-headers.py) | Modern HTTP client with proxy headers | -| [httpx](https://www.python-httpx.org/) | [httpx-async-proxy-headers.py](python/httpx-async-proxy-headers.py) | Async httpx with proxy headers | -| [pycurl](http://pycurl.io/) | [pycurl-proxy-headers.py](python/pycurl-proxy-headers.py) | libcurl bindings with proxy headers | -| [pycurl](http://pycurl.io/) | [pycurl-proxy-headers-lowlevel.py](python/pycurl-proxy-headers-lowlevel.py) | Low-level pycurl integration | -| [cloudscraper](https://github.com/venomous/cloudscraper) | [cloudscraper-proxy-headers.py](python/cloudscraper-proxy-headers.py) | Cloudflare bypass with proxy headers | -| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy-headers.py](python/autoscraper-proxy-headers.py) | Automatic web scraping with proxy headers | - -> **Note:** Most Python HTTP libraries do not expose custom headers on HTTPS `CONNECT` tunneling by default. These examples use [python-proxy-headers](https://github.com/proxymesh/python-proxy-headers) adapters to send proxy headers and read proxy response headers consistently. - -### Basic Proxy Examples - -* [requests-proxy.py](python/requests-proxy.py) - Basic proxy usage with requests -* [requests-random-proxy.py](python/requests-random-proxy.py) - Random proxy rotation +| [requests](https://docs.python-requests.org/) | [requests-proxy.py](python/requests-proxy.py) | Basic `GET` with `proxies=` | +| [requests](https://docs.python-requests.org/) | [requests-session-proxy.py](python/requests-session-proxy.py) | Session with pooled connections | +| [urllib3](https://urllib3.readthedocs.io/) | [urllib3-proxy.py](python/urllib3-proxy.py) | `ProxyManager` | +| [aiohttp](https://docs.aiohttp.org/) | [aiohttp-proxy.py](python/aiohttp-proxy.py) | Async client, `proxy=` on the request | +| [httpx](https://www.python-httpx.org/) | [httpx-proxy.py](python/httpx-proxy.py) | Sync client, `proxy=` on the client | +| [httpx](https://www.python-httpx.org/) | [httpx-async-proxy.py](python/httpx-async-proxy.py) | Async client | +| [pycurl](http://pycurl.io/) | [pycurl-proxy.py](python/pycurl-proxy.py) | libcurl via `PROXY` | +| [pycurl](http://pycurl.io/) | [pycurl-proxy-lowlevel.py](python/pycurl-proxy-lowlevel.py) | Same flow with explicit `setopt` calls | +| [cloudscraper](https://github.com/VeNoMouS/cloudscraper) | [cloudscraper-proxy.py](python/cloudscraper-proxy.py) | Requests-based scraper with `proxies` | +| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | Proxied `requests` pattern (same kwargs as `request_args` on `build()`) | +| [Scrapy](https://scrapy.org/) | [scrapy-proxy.py](python/scrapy-proxy.py) | `scrapy runspider` with `meta['proxy']` | + +### Other Python scripts -### Scrapy +* [requests-random-proxy.py](python/requests-random-proxy.py) - Random proxy rotation -* [scrapy-proxy-headers.py](python/scrapy-proxy-headers.py) - Scrapy spider with proxy headers +> **Note:** Like the Ruby, JavaScript, and PHP examples here, these scripts use each library's normal proxy options only. Most of them do not send custom headers on the HTTPS `CONNECT` tunnel or surface proxy `CONNECT` response headers. For that, see [python-proxy-headers](https://github.com/proxymesh/python-proxy-headers) or [scrapy-proxy-headers](https://github.com/proxymesh/scrapy-proxy-headers). ## JavaScript / Node.js Proxy Examples diff --git a/python/aiohttp-proxy-headers.py b/python/aiohttp-proxy-headers.py deleted file mode 100755 index cde02c8..0000000 --- a/python/aiohttp-proxy-headers.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -""" -aiohttp with proxy headers example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -import asyncio -from python_proxy_headers import aiohttp_proxy - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - - -async def main(): - async with aiohttp_proxy.ProxyClientSession() as session: - async with session.get( - test_url, - proxy=proxy_url, - proxy_headers=proxy_headers - ) as response: - body = await response.text() - print(f"Status: {response.status}") - print(f"Body: {body}") - if response_header: - print(f"{response_header}: {response.headers.get(response_header)}") - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/python/aiohttp-proxy.py b/python/aiohttp-proxy.py new file mode 100644 index 0000000..c22da3d --- /dev/null +++ b/python/aiohttp-proxy.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +""" +aiohttp with an HTTP proxy. + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +Documentation: https://docs.aiohttp.org/en/stable/client_advanced.html#proxy-support +""" +import asyncio +import os +import sys + +import aiohttp + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +response_header = os.environ.get('RESPONSE_HEADER') + + +async def main() -> None: + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(test_url, proxy=proxy_url) as response: + body = await response.text() + print(f'Status: {response.status}') + print(f'Body: {body}') + if response_header: + print(f'{response_header}: {response.headers.get(response_header)}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/python/autoscraper-proxy-headers.py b/python/autoscraper-proxy-headers.py deleted file mode 100755 index 4df0d1b..0000000 --- a/python/autoscraper-proxy-headers.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -""" -AutoScraper with proxy headers example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -from python_proxy_headers.autoscraper_proxy import ProxyAutoScraper - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - -# Create scraper and test via underlying session -scraper = ProxyAutoScraper(proxy_headers=proxy_headers) -session = scraper._get_session() -session.proxies = {'http': proxy_url, 'https': proxy_url} - -# Make request -response = session.get(test_url) - -# Output -print(f"Status: {response.status_code}") -print(f"Body: {response.text}") -if response_header: - print(f"{response_header}: {response.headers.get(response_header)}") - -scraper.close() diff --git a/python/autoscraper-proxy.py b/python/autoscraper-proxy.py new file mode 100644 index 0000000..41f4a71 --- /dev/null +++ b/python/autoscraper-proxy.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +""" +AutoScraper (requests-based) with an HTTP proxy. + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +AutoScraper uses ``requests`` for downloads. Pass ``proxies`` (and other +``requests`` keyword arguments) through ``request_args`` on ``build()``, +``get_result_similar()``, and related methods. + +Documentation: https://github.com/alirezamika/autoscraper +""" +import os +import sys + +import requests +from autoscraper import AutoScraper + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +response_header = os.environ.get('RESPONSE_HEADER') + +proxies = {'http': proxy_url, 'https': proxy_url} + +response = requests.get(test_url, proxies=proxies, timeout=30) +print(f'Status: {response.status_code}') +print(f'Body: {response.text}') +if response_header: + print(f'{response_header}: {response.headers.get(response_header)}') + +# Example: scraper.build(url, wanted_list=[...], request_args={'proxies': proxies, 'timeout': 30}) +AutoScraper() diff --git a/python/cloudscraper-proxy-headers.py b/python/cloudscraper-proxy-headers.py deleted file mode 100755 index ef3e9ea..0000000 --- a/python/cloudscraper-proxy-headers.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -""" -CloudScraper with proxy headers example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -from python_proxy_headers.cloudscraper_proxy import create_scraper - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - -# Create scraper with proxy headers -scraper = create_scraper(proxy_headers=proxy_headers, browser='chrome') -scraper.proxies = {'http': proxy_url, 'https': proxy_url} - -# Make request -response = scraper.get(test_url) - -# Output -print(f"Status: {response.status_code}") -print(f"Body: {response.text}") -if response_header: - print(f"{response_header}: {response.headers.get(response_header)}") diff --git a/python/cloudscraper-proxy.py b/python/cloudscraper-proxy.py new file mode 100644 index 0000000..a4fc5bd --- /dev/null +++ b/python/cloudscraper-proxy.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +""" +cloudscraper with an HTTP proxy. + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +cloudscraper builds on requests; set ``proxies`` on the scraper like a Session. + +Documentation: https://github.com/VeNoMouS/cloudscraper +""" +import os +import sys + +import cloudscraper + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +response_header = os.environ.get('RESPONSE_HEADER') + +proxies = {'http': proxy_url, 'https': proxy_url} +scraper = cloudscraper.create_scraper(browser='chrome') +scraper.proxies = proxies + +response = scraper.get(test_url, timeout=30) + +print(f'Status: {response.status_code}') +print(f'Body: {response.text}') +if response_header: + print(f'{response_header}: {response.headers.get(response_header)}') diff --git a/python/httpx-async-proxy-headers.py b/python/httpx-async-proxy-headers.py deleted file mode 100755 index 5553c15..0000000 --- a/python/httpx-async-proxy-headers.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -""" -httpx async with proxy headers example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -import asyncio -import httpx -from python_proxy_headers.httpx_proxy import AsyncHTTPProxyTransport - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - - -async def main(): - # Create proxy with optional headers - if proxy_headers: - proxy = httpx.Proxy(proxy_url, headers=proxy_headers) - else: - proxy = proxy_url - - transport = AsyncHTTPProxyTransport(proxy=proxy) - - async with httpx.AsyncClient(mounts={'http://': transport, 'https://': transport}) as client: - response = await client.get(test_url) - - print(f"Status: {response.status_code}") - print(f"Body: {response.text}") - if response_header: - print(f"{response_header}: {response.headers.get(response_header)}") - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/python/httpx-async-proxy.py b/python/httpx-async-proxy.py new file mode 100644 index 0000000..e4b866d --- /dev/null +++ b/python/httpx-async-proxy.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +""" +httpx (async) with an HTTP proxy. + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +Documentation: https://www.python-httpx.org/async/ +""" +import asyncio +import os +import sys + +import httpx + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +response_header = os.environ.get('RESPONSE_HEADER') + + +async def main() -> None: + async with httpx.AsyncClient(proxy=proxy_url, timeout=30.0) as client: + response = await client.get(test_url) + + print(f'Status: {response.status_code}') + print(f'Body: {response.text}') + if response_header: + print(f'{response_header}: {response.headers.get(response_header)}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/python/httpx-proxy-headers.py b/python/httpx-proxy-headers.py deleted file mode 100755 index 5952448..0000000 --- a/python/httpx-proxy-headers.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -""" -httpx with proxy headers example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -import httpx -from python_proxy_headers import httpx_proxy - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - -# Create proxy with optional headers -if proxy_headers: - proxy = httpx.Proxy(proxy_url, headers=proxy_headers) -else: - proxy = proxy_url - -# Make request -response = httpx_proxy.get(test_url, proxy=proxy) - -# Output -print(f"Status: {response.status_code}") -print(f"Body: {response.text}") -if response_header: - print(f"{response_header}: {response.headers.get(response_header)}") diff --git a/python/httpx-proxy.py b/python/httpx-proxy.py new file mode 100644 index 0000000..df586ed --- /dev/null +++ b/python/httpx-proxy.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +""" +httpx (sync) with an HTTP proxy. + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +Documentation: https://www.python-httpx.org/advanced/proxies/ +""" +import os +import sys + +import httpx + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +response_header = os.environ.get('RESPONSE_HEADER') + +with httpx.Client(proxy=proxy_url, timeout=30.0) as client: + response = client.get(test_url) + +print(f'Status: {response.status_code}') +print(f'Body: {response.text}') +if response_header: + print(f'{response_header}: {response.headers.get(response_header)}') diff --git a/python/pycurl-proxy-headers-lowlevel.py b/python/pycurl-proxy-headers-lowlevel.py deleted file mode 100755 index c957927..0000000 --- a/python/pycurl-proxy-headers-lowlevel.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -""" -PycURL with proxy headers - low-level example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -import pycurl -from io import BytesIO -from python_proxy_headers.pycurl_proxy import set_proxy_headers, HeaderCapture - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - -# Create pycurl handle -c = pycurl.Curl() -buffer = BytesIO() - -c.setopt(pycurl.URL, test_url) -c.setopt(pycurl.PROXY, proxy_url) -c.setopt(pycurl.WRITEDATA, buffer) - -# Add proxy headers if configured -if proxy_headers: - set_proxy_headers(c, proxy_headers) - -# Capture response headers -capture = HeaderCapture(c) - -# Perform request -c.perform() - -# Output -print(f"Status: {c.getinfo(pycurl.RESPONSE_CODE)}") -print(f"Body: {buffer.getvalue().decode('utf-8')}") -if response_header: - print(f"{response_header}: {capture.proxy_headers.get(response_header)}") - -c.close() diff --git a/python/pycurl-proxy-headers.py b/python/pycurl-proxy-headers.py deleted file mode 100755 index ce05d9e..0000000 --- a/python/pycurl-proxy-headers.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -""" -PycURL with proxy headers example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -from python_proxy_headers.pycurl_proxy import get - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - -# Make request -response = get(test_url, proxy=proxy_url, proxy_headers=proxy_headers) - -# Output -print(f"Status: {response.status_code}") -print(f"Body: {response.text}") -if response_header: - print(f"{response_header}: {response.proxy_headers.get(response_header)}") diff --git a/python/pycurl-proxy-lowlevel.py b/python/pycurl-proxy-lowlevel.py new file mode 100644 index 0000000..041d978 --- /dev/null +++ b/python/pycurl-proxy-lowlevel.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +""" +pycurl with an HTTP proxy (explicit setopt calls). + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + +Same behavior as ``pycurl-proxy.py`` but with each option set separately, which +matches how lower-level libcurl integrations are often structured. + +Documentation: https://pycurl.io/docs/latest/curlobject.html +""" +import os +import sys +from io import BytesIO + +import pycurl + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') + +buffer = BytesIO() +handle = pycurl.Curl() + +handle.setopt(pycurl.URL, test_url) +handle.setopt(pycurl.PROXY, proxy_url) +handle.setopt(pycurl.WRITEDATA, buffer) +handle.setopt(pycurl.FOLLOWLOCATION, 1) +handle.setopt(pycurl.TIMEOUT, 30) +handle.setopt(pycurl.SSL_VERIFYPEER, 1) + +try: + handle.perform() + status = handle.getinfo(pycurl.RESPONSE_CODE) +except pycurl.error as exc: + _errno, msg = exc.args + print(f'Error: {msg}', file=sys.stderr) + sys.exit(1) +finally: + handle.close() + +print(f'Status: {status}') +print(f'Body: {buffer.getvalue().decode("utf-8", errors="replace")}') diff --git a/python/pycurl-proxy.py b/python/pycurl-proxy.py new file mode 100644 index 0000000..b9238a4 --- /dev/null +++ b/python/pycurl-proxy.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +""" +pycurl (libcurl) with an HTTP proxy. + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + +Requires libcurl development headers to install the ``pycurl`` package. + +Documentation: https://pycurl.io/docs/latest/curlobject.html +""" +import os +import sys +from io import BytesIO + +import pycurl + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') + +buffer = BytesIO() +c = pycurl.Curl() +try: + c.setopt(pycurl.URL, test_url) + c.setopt(pycurl.PROXY, proxy_url) + c.setopt(pycurl.FOLLOWLOCATION, 1) + c.setopt(pycurl.TIMEOUT, 30) + c.setopt(pycurl.WRITEDATA, buffer) + c.perform() + status = c.getinfo(pycurl.RESPONSE_CODE) +except pycurl.error as exc: + errno, msg = exc.args + print(f'Error: {msg}', file=sys.stderr) + sys.exit(1) +finally: + c.close() + +print(f'Status: {status}') +print(f'Body: {buffer.getvalue().decode("utf-8", errors="replace")}') diff --git a/python/requests-proxy-headers-session.py b/python/requests-proxy-headers-session.py deleted file mode 100755 index 4a12171..0000000 --- a/python/requests-proxy-headers-session.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -""" -Requests with proxy headers - Session example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -from python_proxy_headers.requests_adapter import ProxySession - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxies = {'http': proxy_url, 'https': proxy_url} -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - -# Make requests using session -with ProxySession(proxy_headers=proxy_headers) as session: - session.proxies = proxies - - response = session.get(test_url) - print(f"Status: {response.status_code}") - print(f"Body: {response.text}") - if response_header: - print(f"{response_header}: {response.headers.get(response_header)}") diff --git a/python/requests-proxy-headers.py b/python/requests-proxy-headers.py deleted file mode 100755 index 35e56a9..0000000 --- a/python/requests-proxy-headers.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -""" -Requests with proxy headers example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -from python_proxy_headers import requests_adapter - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxies = {'http': proxy_url, 'https': proxy_url} -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - -# Make request -response = requests_adapter.get(test_url, proxies=proxies, proxy_headers=proxy_headers) - -# Output -print(f"Status: {response.status_code}") -print(f"Body: {response.text}") -if response_header: - print(f"{response_header}: {response.headers.get(response_header)}") diff --git a/python/requests-proxy.py b/python/requests-proxy.py index fb1301f..1f4e918 100755 --- a/python/requests-proxy.py +++ b/python/requests-proxy.py @@ -3,8 +3,11 @@ Basic requests with proxy example. Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +Documentation: https://docs.python-requests.org/en/latest/user/advanced/#proxies """ import os import sys @@ -17,12 +20,13 @@ sys.exit(1) test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +response_header = os.environ.get('RESPONSE_HEADER') proxies = {'http': proxy_url, 'https': proxy_url} -# Make request -response = requests.get(test_url, proxies=proxies) +response = requests.get(test_url, proxies=proxies, timeout=30) -# Output print(f"Status: {response.status_code}") print(f"Body: {response.text}") +if response_header: + print(f"{response_header}: {response.headers.get(response_header)}") diff --git a/python/requests-session-proxy.py b/python/requests-session-proxy.py new file mode 100644 index 0000000..4d7a9ac --- /dev/null +++ b/python/requests-session-proxy.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +""" +Requests Session with an HTTP proxy. + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +Uses a :class:`requests.Session` for connection pooling. Same proxy options as +``requests.get(..., proxies=...)``. + +Documentation: https://docs.python-requests.org/en/latest/user/advanced/#proxies +""" +import os +import sys + +import requests + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +response_header = os.environ.get('RESPONSE_HEADER') + +proxies = {'http': proxy_url, 'https': proxy_url} + +with requests.Session() as session: + session.proxies.update(proxies) + response = session.get(test_url, timeout=30) + +print(f'Status: {response.status_code}') +print(f'Body: {response.text}') +if response_header: + print(f'{response_header}: {response.headers.get(response_header)}') diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000..8ed716a --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,12 @@ +# Third-party libraries used by the Python proxy examples (see run_tests.py). +# Install: pip install -r python/requirements.txt +requests>=2.28.0 +urllib3>=2.0.0 +aiohttp>=3.9.0 +httpx>=0.27.0 +pycurl>=7.45.0; platform_system != "Windows" +cloudscraper>=1.2.71 +autoscraper>=1.1.0 +beautifulsoup4>=4.12.0 +lxml>=5.0.0 +scrapy>=2.11.0 diff --git a/python/run_tests.py b/python/run_tests.py index 8479b87..cb4e865 100755 --- a/python/run_tests.py +++ b/python/run_tests.py @@ -1,12 +1,11 @@ #!/usr/bin/env python3 """ -Run all Python proxy header examples as tests. +Run all Python proxy examples as tests. Configuration via environment variables: PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) + RESPONSE_HEADER - Optional header name for examples that print response headers Usage: python run_tests.py # Run all examples @@ -17,103 +16,146 @@ import sys import subprocess from pathlib import Path +from typing import List # Examples to test (filename without .py) EXAMPLES = [ - 'requests-proxy-headers', - 'requests-proxy-headers-session', - 'urllib3-proxy-headers', - 'aiohttp-proxy-headers', - 'httpx-proxy-headers', - 'httpx-async-proxy-headers', - 'pycurl-proxy-headers', - 'pycurl-proxy-headers-lowlevel', - 'cloudscraper-proxy-headers', - 'autoscraper-proxy-headers', + 'requests-proxy', + 'requests-session-proxy', + 'urllib3-proxy', + 'aiohttp-proxy', + 'httpx-proxy', + 'httpx-async-proxy', + 'pycurl-proxy', + 'pycurl-proxy-lowlevel', + 'cloudscraper-proxy', + 'autoscraper-proxy', + 'scrapy-proxy', ] +def _scrapy_import_ok() -> bool: + """Check Scrapy in a subprocess so a broken cryptography/cffi stack cannot abort this runner.""" + try: + r = subprocess.run( + [sys.executable, '-c', 'import scrapy'], + capture_output=True, + timeout=30, + ) + return r.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +def _available_examples() -> List[str]: + """Skip examples when optional native / heavy dependencies are missing or broken.""" + try: + import pycurl # noqa: F401 + + has_pycurl = True + except ImportError: + has_pycurl = False + + has_scrapy = _scrapy_import_ok() + + out: List[str] = [] + for e in EXAMPLES: + if 'pycurl' in e and not has_pycurl: + continue + if e == 'scrapy-proxy' and not has_scrapy: + continue + out.append(e) + return out + + def run_example(name: str) -> bool: """Run an example script and return True if successful.""" script_dir = Path(__file__).parent - script_path = script_dir / f"{name}.py" - + script_path = script_dir / f'{name}.py' + if not script_path.exists(): - print(f" Script not found: {script_path}") + print(f' Script not found: {script_path}') return False - + + if name == 'scrapy-proxy': + cmd = [sys.executable, '-m', 'scrapy', 'runspider', str(script_path)] + timeout = 90 + else: + cmd = [sys.executable, str(script_path)] + timeout = 30 + try: result = subprocess.run( - [sys.executable, str(script_path)], + cmd, + cwd=str(script_dir), capture_output=True, text=True, - timeout=30 + timeout=timeout, ) - + if result.returncode == 0: return True - else: - print(f" Exit code: {result.returncode}") - if result.stderr: - print(f" Error: {result.stderr.strip()}") - return False - + print(f' Exit code: {result.returncode}') + if result.stderr: + print(f' Error: {result.stderr.strip()}') + return False + except subprocess.TimeoutExpired: - print(" Timeout after 30s") + print(f' Timeout after {timeout}s') return False except Exception as e: - print(f" Exception: {e}") + print(f' Exception: {e}') return False def main(): args = sys.argv[1:] - - # Handle -l/--list + if '-l' in args or '--list' in args: - print("Available examples:") - for name in EXAMPLES: - print(f" {name}") + print('Available examples:') + for name in _available_examples(): + print(f' {name}') sys.exit(0) - - # Handle -h/--help + if '-h' in args or '--help' in args: print(__doc__) sys.exit(0) - - # Check for PROXY_URL + if not os.environ.get('PROXY_URL') and not os.environ.get('HTTPS_PROXY'): - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - print("\nExample:", file=sys.stderr) + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + print('\nExample:', file=sys.stderr) print(" export PROXY_URL='http://user:pass@proxy:8080'", file=sys.stderr) sys.exit(1) - - # Determine which examples to run + + available = _available_examples() if args: - examples = args + examples = [e for e in available if any(a in e for a in args)] + if not examples: + print('No matching examples.', file=sys.stderr) + sys.exit(1) else: - examples = EXAMPLES - - print(f"\n{'='*50}") - print("Running Python Proxy Header Examples") - print(f"{'='*50}\n") - + examples = available + + print(f"\n{'=' * 50}") + print('Running Python Proxy Examples') + print(f"{'=' * 50}\n") + passed = 0 failed = 0 - + for name in examples: - print(f"[TEST] {name}...", end=" ", flush=True) + print(f'[TEST] {name}... ', end='', flush=True) if run_example(name): - print("PASS") + print('PASS') passed += 1 else: - print("FAIL") + print('FAIL') failed += 1 - - print(f"\n{'='*50}") - print(f"Results: {passed} passed, {failed} failed") - print(f"{'='*50}\n") - + + print(f"\n{'=' * 50}") + print(f'Results: {passed} passed, {failed} failed') + print(f"{'=' * 50}\n") + sys.exit(0 if failed == 0 else 1) diff --git a/python/scrapy-proxy-headers.py b/python/scrapy-proxy-headers.py deleted file mode 100755 index 213b715..0000000 --- a/python/scrapy-proxy-headers.py +++ /dev/null @@ -1,12 +0,0 @@ -# See https://github.com/proxymesh/scrapy-proxy-headers - -# In your Scrapy `settings.py`, add the following: -DOWNLOAD_HANDLERS = { - "https": "scrapy_proxy_headers.HTTP11ProxyDownloadHandler" -} - -# add to your request procesing method -request.meta["proxy_headers"] = {"X-ProxyMesh-Country": "US"} - -# then when you get a response -response.headers["X-ProxyMesh-IP"] \ No newline at end of file diff --git a/python/scrapy-proxy.py b/python/scrapy-proxy.py new file mode 100644 index 0000000..e06a3f1 --- /dev/null +++ b/python/scrapy-proxy.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +""" +Scrapy with an HTTP proxy (runspider). + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +Uses ``meta['proxy']`` on the request. For custom headers on the proxy tunnel +(ProxyMesh-style), see the scrapy-proxy-headers package linked from the README. + +Documentation: https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#module-scrapy.downloadermiddlewares.httpproxy +""" +import os +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess + + +class ProxiedIpifySpider(scrapy.Spider): + name = 'proxied_ipify' + custom_settings = { + 'LOG_LEVEL': 'WARNING', + 'ROBOTSTXT_OBEY': False, + } + + def start_requests(self): + proxy_url = os.environ['PROXY_URL'] + test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') + yield scrapy.Request(test_url, meta={'proxy': proxy_url}, dont_filter=True) + + def parse(self, response): + rh = os.environ.get('RESPONSE_HEADER') + print(f'Status: {response.status}') + print(f'Body: {response.text}') + if rh: + key = rh.encode('utf-8') + raw = response.headers.get(key) + val = raw.decode('utf-8') if raw else None + print(f'{rh}: {val}') + + +def main() -> None: + if not (os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY')): + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + if not os.environ.get('PROXY_URL'): + os.environ['PROXY_URL'] = os.environ['HTTPS_PROXY'] + + process = CrawlerProcess() + process.crawl(ProxiedIpifySpider) + process.start() + + +if __name__ == '__main__': + main() diff --git a/python/urllib3-proxy-headers.py b/python/urllib3-proxy-headers.py deleted file mode 100755 index 654ac40..0000000 --- a/python/urllib3-proxy-headers.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -""" -urllib3 with proxy headers example. - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - PROXY_HEADER - Header name to send to proxy (optional) - PROXY_VALUE - Header value to send to proxy (optional) - RESPONSE_HEADER - Header name to read from response (optional) - -See: https://github.com/proxymesh/python-proxy-headers -""" -import os -import sys -from python_proxy_headers import urllib3_proxy_manager - -# Get configuration from environment -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print("Error: Set PROXY_URL environment variable", file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') -proxy_header = os.environ.get('PROXY_HEADER') -proxy_value = os.environ.get('PROXY_VALUE') -response_header = os.environ.get('RESPONSE_HEADER') - -proxy_headers = {proxy_header: proxy_value} if proxy_header and proxy_value else None - -# Create proxy manager and make request -proxy = urllib3_proxy_manager.ProxyHeaderManager(proxy_url, proxy_headers=proxy_headers) -response = proxy.request('GET', test_url) - -# Output -print(f"Status: {response.status}") -print(f"Body: {response.data.decode('utf-8')}") -if response_header: - print(f"{response_header}: {response.headers.get(response_header)}") diff --git a/python/urllib3-proxy.py b/python/urllib3-proxy.py new file mode 100644 index 0000000..eebcbc2 --- /dev/null +++ b/python/urllib3-proxy.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +""" +urllib3 with an HTTP proxy. + +Configuration via environment variables: + PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 + TEST_URL - URL to request (default: https://api.ipify.org?format=json) + RESPONSE_HEADER - Optional header name to print from the response + +urllib3's :class:`urllib3.ProxyManager` routes traffic through the proxy. It does +not support sending custom headers on the HTTPS CONNECT request or reading proxy +CONNECT response headers (see python-proxy-headers for that). + +Documentation: https://urllib3.readthedocs.io/en/stable/reference/urllib3.poolmanager.html +""" +import os +import sys + +import urllib3 + +proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') +if not proxy_url: + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + +test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +response_header = os.environ.get('RESPONSE_HEADER') + +http = urllib3.ProxyManager(proxy_url) +response = http.request('GET', test_url, timeout=urllib3.Timeout(30)) + +body = response.data.decode('utf-8', errors='replace') +print(f'Status: {response.status}') +print(f'Body: {body}') +if response_header: + print(f'{response_header}: {response.headers.get(response_header)}') From 7b48ab1b7f918a9da05aa57522866e34323fe86d Mon Sep 17 00:00:00 2001 From: Cursor Date: Sun, 5 Apr 2026 15:54:29 +0000 Subject: [PATCH 02/14] CI: install Python examples from requirements.txt, drop python-proxy-headers Made-with: Cursor --- .github/workflows/proxy_integration_tests_python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/proxy_integration_tests_python.yml b/.github/workflows/proxy_integration_tests_python.yml index 8f85aab..d09fa6c 100644 --- a/.github/workflows/proxy_integration_tests_python.yml +++ b/.github/workflows/proxy_integration_tests_python.yml @@ -31,10 +31,10 @@ jobs: - name: Install system dependencies (pycurl) run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Install python-proxy-headers and example dependencies + - name: Install example dependencies run: | python -m pip install --upgrade pip - pip install python-proxy-headers requests urllib3 aiohttp httpx cloudscraper autoscraper pycurl + pip install -r python/requirements.txt - name: Require PROXY_URL Actions secret env: From 205082b48019ff9ed4a9e638d3f2bba6a7a2605b Mon Sep 17 00:00:00 2001 From: Cursor Date: Sun, 5 Apr 2026 15:55:48 +0000 Subject: [PATCH 03/14] urllib3 proxy example: disable retries to avoid proxy CONNECT retry storms Made-with: Cursor --- python/urllib3-proxy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/urllib3-proxy.py b/python/urllib3-proxy.py index eebcbc2..aa20230 100644 --- a/python/urllib3-proxy.py +++ b/python/urllib3-proxy.py @@ -11,6 +11,10 @@ not support sending custom headers on the HTTPS CONNECT request or reading proxy CONNECT response headers (see python-proxy-headers for that). +Default urllib3 retries repeat HTTPS CONNECT through the proxy; some providers +return errors such as ``407 too many failures`` when that happens, so retries +are disabled here. + Documentation: https://urllib3.readthedocs.io/en/stable/reference/urllib3.poolmanager.html """ import os @@ -26,7 +30,7 @@ test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') response_header = os.environ.get('RESPONSE_HEADER') -http = urllib3.ProxyManager(proxy_url) +http = urllib3.ProxyManager(proxy_url, retries=False) response = http.request('GET', test_url, timeout=urllib3.Timeout(30)) body = response.data.decode('utf-8', errors='replace') From e0a944729d1bc996befdbff612cf4a6d4c9bba25 Mon Sep 17 00:00:00 2001 From: Cursor Date: Sun, 5 Apr 2026 15:57:27 +0000 Subject: [PATCH 04/14] Fix urllib3 proxy auth for CONNECT; pin CI Python 3.12 Strip userinfo from proxy URL and send Proxy-Authorization via proxy_headers so HTTPS CONNECT matches authenticated requests behavior. Pin integration workflow Python to 3.12 for stable wheels. Made-with: Cursor --- .../proxy_integration_tests_python.yml | 3 ++- python/urllib3-proxy.py | 19 ++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/proxy_integration_tests_python.yml b/.github/workflows/proxy_integration_tests_python.yml index d09fa6c..2c7cf90 100644 --- a/.github/workflows/proxy_integration_tests_python.yml +++ b/.github/workflows/proxy_integration_tests_python.yml @@ -26,7 +26,8 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.x" + # Pin for reproducible dependency wheels (pycurl, etc.); adjust as needed. + python-version: "3.12" - name: Install system dependencies (pycurl) run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev diff --git a/python/urllib3-proxy.py b/python/urllib3-proxy.py index aa20230..b6a4efb 100644 --- a/python/urllib3-proxy.py +++ b/python/urllib3-proxy.py @@ -19,6 +19,7 @@ """ import os import sys +from urllib.parse import urlparse, urlunparse import urllib3 @@ -30,7 +31,23 @@ test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') response_header = os.environ.get('RESPONSE_HEADER') -http = urllib3.ProxyManager(proxy_url, retries=False) +parsed = urlparse(proxy_url) +pool_kw = {'retries': False} +if parsed.username is not None: + # Some stacks omit CONNECT credentials unless they are sent as Proxy-Authorization. + user = parsed.username + password = parsed.password or '' + pool_kw['proxy_headers'] = urllib3.util.make_headers( + proxy_basic_auth=f'{user}:{password}' + ) + host = parsed.hostname or '' + if parsed.port: + host = f'{host}:{parsed.port}' + proxy_for_pool = urlunparse((parsed.scheme, host, '', '', '', '')) +else: + proxy_for_pool = proxy_url + +http = urllib3.ProxyManager(proxy_for_pool, **pool_kw) response = http.request('GET', test_url, timeout=urllib3.Timeout(30)) body = response.data.decode('utf-8', errors='replace') From 90570cac0a6417535694d1429782dd3f05c00c08 Mon Sep 17 00:00:00 2001 From: Cursor Date: Sun, 5 Apr 2026 16:59:29 +0000 Subject: [PATCH 05/14] CI: bump checkout to v6 and setup-python/setup-node to v6 (Node 24) Pins actions/checkout v6.0.2, setup-python v6.2.0, setup-node v6.3.0 to clear GitHub Actions Node 20 deprecation warnings on integration workflows. Made-with: Cursor --- .github/workflows/proxy_integration_tests_javascript.yml | 4 ++-- .github/workflows/proxy_integration_tests_php.yml | 2 +- .github/workflows/proxy_integration_tests_python.yml | 4 ++-- .github/workflows/proxy_integration_tests_ruby.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/proxy_integration_tests_javascript.yml b/.github/workflows/proxy_integration_tests_javascript.yml index 25c4aa1..de0089d 100644 --- a/.github/workflows/proxy_integration_tests_javascript.yml +++ b/.github/workflows/proxy_integration_tests_javascript.yml @@ -19,12 +19,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 (Node 24) with: persist-credentials: false - name: Set up Node - uses: actions/setup-node@v4 + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 (Node 24) with: node-version: "24" cache: npm diff --git a/.github/workflows/proxy_integration_tests_php.yml b/.github/workflows/proxy_integration_tests_php.yml index d20ced9..1f41742 100644 --- a/.github/workflows/proxy_integration_tests_php.yml +++ b/.github/workflows/proxy_integration_tests_php.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 (Node 24) with: persist-credentials: false diff --git a/.github/workflows/proxy_integration_tests_python.yml b/.github/workflows/proxy_integration_tests_python.yml index 2c7cf90..f2c8c49 100644 --- a/.github/workflows/proxy_integration_tests_python.yml +++ b/.github/workflows/proxy_integration_tests_python.yml @@ -19,12 +19,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 (Node 24) with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 (Node 24) with: # Pin for reproducible dependency wheels (pycurl, etc.); adjust as needed. python-version: "3.12" diff --git a/.github/workflows/proxy_integration_tests_ruby.yml b/.github/workflows/proxy_integration_tests_ruby.yml index e9f6ebf..1498abb 100644 --- a/.github/workflows/proxy_integration_tests_ruby.yml +++ b/.github/workflows/proxy_integration_tests_ruby.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 (Node 24) with: persist-credentials: false From 7c5c5fe8c0d5a2837ed01d95465d18b2158ac33e Mon Sep 17 00:00:00 2001 From: Cursor Date: Sun, 5 Apr 2026 16:59:45 +0000 Subject: [PATCH 06/14] CI: bump ruby/setup-ruby to v1.299.0 (Node 24) Made-with: Cursor --- .github/workflows/proxy_integration_tests_ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/proxy_integration_tests_ruby.yml b/.github/workflows/proxy_integration_tests_ruby.yml index 1498abb..648e3f9 100644 --- a/.github/workflows/proxy_integration_tests_ruby.yml +++ b/.github/workflows/proxy_integration_tests_ruby.yml @@ -24,7 +24,7 @@ jobs: persist-credentials: false - name: Set up Ruby - uses: ruby/setup-ruby@2e007403fc1ec238429ecaa57af6f22f019cc135 # v1.234.0 + uses: ruby/setup-ruby@3ff19f5e2baf30647122352b96108b1fbe250c64 # v1.299.0 (Node 24) with: ruby-version: "3.3" bundler-cache: true From 66cf7df2d3ddeebb544fc45f4f30507455bb1240 Mon Sep 17 00:00:00 2001 From: Cursor Date: Mon, 6 Apr 2026 13:58:40 +0000 Subject: [PATCH 07/14] autoscraper example: real build() and get_result_similar() via proxy Default to example.com HTML with WANTED_TEXT; document JSON is a poor fit. Made-with: Cursor --- README.md | 2 +- python/autoscraper-proxy.py | 42 +++++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index eb2db3d..68eb059 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ python python/run_tests.py requests httpx | [pycurl](http://pycurl.io/) | [pycurl-proxy.py](python/pycurl-proxy.py) | libcurl via `PROXY` | | [pycurl](http://pycurl.io/) | [pycurl-proxy-lowlevel.py](python/pycurl-proxy-lowlevel.py) | Same flow with explicit `setopt` calls | | [cloudscraper](https://github.com/VeNoMouS/cloudscraper) | [cloudscraper-proxy.py](python/cloudscraper-proxy.py) | Requests-based scraper with `proxies` | -| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | Proxied `requests` pattern (same kwargs as `request_args` on `build()`) | +| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | `build()` + `get_result_similar()` on HTML via `request_args` (default `example.com`, optional `WANTED_TEXT`) | | [Scrapy](https://scrapy.org/) | [scrapy-proxy.py](python/scrapy-proxy.py) | `scrapy runspider` with `meta['proxy']` | ### Other Python scripts diff --git a/python/autoscraper-proxy.py b/python/autoscraper-proxy.py index 41f4a71..afaddb4 100644 --- a/python/autoscraper-proxy.py +++ b/python/autoscraper-proxy.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 """ -AutoScraper (requests-based) with an HTTP proxy. +AutoScraper with an HTTP proxy. Configuration via environment variables: PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - RESPONSE_HEADER - Optional header name to print from the response + TEST_URL - HTML page to scrape (default: https://example.com/) + WANTED_TEXT - Comma-separated substrings AutoScraper should learn to extract + (default: Example Domain, matching the default TEST_URL) + RESPONSE_HEADER - Optional header name to print from the first response -AutoScraper uses ``requests`` for downloads. Pass ``proxies`` (and other -``requests`` keyword arguments) through ``request_args`` on ``build()``, -``get_result_similar()``, and related methods. +AutoScraper is built for HTML pages (BeautifulSoup + rules), not raw JSON APIs. +Pass ``proxies`` through ``request_args`` on ``build()`` and ``get_result_similar()``. Documentation: https://github.com/alirezamika/autoscraper """ @@ -24,16 +25,31 @@ print('Error: Set PROXY_URL environment variable', file=sys.stderr) sys.exit(1) -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') +test_url = os.environ.get('TEST_URL', 'https://example.com/') response_header = os.environ.get('RESPONSE_HEADER') +wanted_raw = os.environ.get('WANTED_TEXT', 'Example Domain') +wanted_list = [s.strip() for s in wanted_raw.split(',') if s.strip()] proxies = {'http': proxy_url, 'https': proxy_url} +request_args = {'proxies': proxies, 'timeout': 30} -response = requests.get(test_url, proxies=proxies, timeout=30) -print(f'Status: {response.status_code}') -print(f'Body: {response.text}') +# Same pattern as other examples: show status from requests. +probe = requests.get(test_url, **request_args) +print(f'Status: {probe.status_code}') if response_header: - print(f'{response_header}: {response.headers.get(response_header)}') + print(f'{response_header}: {probe.headers.get(response_header)}') -# Example: scraper.build(url, wanted_list=[...], request_args={'proxies': proxies, 'timeout': 30}) -AutoScraper() +scraper = AutoScraper() +learned = scraper.build(test_url, wanted_list=wanted_list, request_args=request_args) +print(f'AutoScraper build (values used to learn rules): {learned}') + +similar = scraper.get_result_similar(test_url, request_args=request_args) +print(f'AutoScraper get_result_similar: {similar}') + +if not learned and not similar: + print( + 'Note: No matches. Use an HTML TEST_URL and set WANTED_TEXT to text that ' + 'appears on that page.', + file=sys.stderr, + ) + sys.exit(1) From 77f45dbea6ed568edc6747ca4a6125a049e47ac2 Mon Sep 17 00:00:00 2001 From: Cursor Date: Mon, 6 Apr 2026 14:00:53 +0000 Subject: [PATCH 08/14] Remove duplicate pycurl-lowlevel example; keep single pycurl-proxy.py Made-with: Cursor --- README.md | 3 +-- python/pycurl-proxy-lowlevel.py | 48 --------------------------------- python/pycurl-proxy.py | 4 ++- python/run_tests.py | 1 - 4 files changed, 4 insertions(+), 52 deletions(-) delete mode 100644 python/pycurl-proxy-lowlevel.py diff --git a/README.md b/README.md index 68eb059..a83e206 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,7 @@ python python/run_tests.py requests httpx | [aiohttp](https://docs.aiohttp.org/) | [aiohttp-proxy.py](python/aiohttp-proxy.py) | Async client, `proxy=` on the request | | [httpx](https://www.python-httpx.org/) | [httpx-proxy.py](python/httpx-proxy.py) | Sync client, `proxy=` on the client | | [httpx](https://www.python-httpx.org/) | [httpx-async-proxy.py](python/httpx-async-proxy.py) | Async client | -| [pycurl](http://pycurl.io/) | [pycurl-proxy.py](python/pycurl-proxy.py) | libcurl via `PROXY` | -| [pycurl](http://pycurl.io/) | [pycurl-proxy-lowlevel.py](python/pycurl-proxy-lowlevel.py) | Same flow with explicit `setopt` calls | +| [pycurl](http://pycurl.io/) | [pycurl-proxy.py](python/pycurl-proxy.py) | libcurl via `setopt` (`PROXY`, `WRITEDATA`, etc.) | | [cloudscraper](https://github.com/VeNoMouS/cloudscraper) | [cloudscraper-proxy.py](python/cloudscraper-proxy.py) | Requests-based scraper with `proxies` | | [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | `build()` + `get_result_similar()` on HTML via `request_args` (default `example.com`, optional `WANTED_TEXT`) | | [Scrapy](https://scrapy.org/) | [scrapy-proxy.py](python/scrapy-proxy.py) | `scrapy runspider` with `meta['proxy']` | diff --git a/python/pycurl-proxy-lowlevel.py b/python/pycurl-proxy-lowlevel.py deleted file mode 100644 index 041d978..0000000 --- a/python/pycurl-proxy-lowlevel.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -""" -pycurl with an HTTP proxy (explicit setopt calls). - -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - URL to request (default: https://api.ipify.org?format=json) - -Same behavior as ``pycurl-proxy.py`` but with each option set separately, which -matches how lower-level libcurl integrations are often structured. - -Documentation: https://pycurl.io/docs/latest/curlobject.html -""" -import os -import sys -from io import BytesIO - -import pycurl - -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print('Error: Set PROXY_URL environment variable', file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://api.ipify.org?format=json') - -buffer = BytesIO() -handle = pycurl.Curl() - -handle.setopt(pycurl.URL, test_url) -handle.setopt(pycurl.PROXY, proxy_url) -handle.setopt(pycurl.WRITEDATA, buffer) -handle.setopt(pycurl.FOLLOWLOCATION, 1) -handle.setopt(pycurl.TIMEOUT, 30) -handle.setopt(pycurl.SSL_VERIFYPEER, 1) - -try: - handle.perform() - status = handle.getinfo(pycurl.RESPONSE_CODE) -except pycurl.error as exc: - _errno, msg = exc.args - print(f'Error: {msg}', file=sys.stderr) - sys.exit(1) -finally: - handle.close() - -print(f'Status: {status}') -print(f'Body: {buffer.getvalue().decode("utf-8", errors="replace")}') diff --git a/python/pycurl-proxy.py b/python/pycurl-proxy.py index b9238a4..8ae75c1 100644 --- a/python/pycurl-proxy.py +++ b/python/pycurl-proxy.py @@ -6,7 +6,8 @@ PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 TEST_URL - URL to request (default: https://api.ipify.org?format=json) -Requires libcurl development headers to install the ``pycurl`` package. +Requires libcurl development headers to install the ``pycurl`` package. Options are +set with :meth:`pycurl.Curl.setopt` like any libcurl binding. Documentation: https://pycurl.io/docs/latest/curlobject.html """ @@ -30,6 +31,7 @@ c.setopt(pycurl.PROXY, proxy_url) c.setopt(pycurl.FOLLOWLOCATION, 1) c.setopt(pycurl.TIMEOUT, 30) + c.setopt(pycurl.SSL_VERIFYPEER, 1) c.setopt(pycurl.WRITEDATA, buffer) c.perform() status = c.getinfo(pycurl.RESPONSE_CODE) diff --git a/python/run_tests.py b/python/run_tests.py index cb4e865..ebb2651 100755 --- a/python/run_tests.py +++ b/python/run_tests.py @@ -27,7 +27,6 @@ 'httpx-proxy', 'httpx-async-proxy', 'pycurl-proxy', - 'pycurl-proxy-lowlevel', 'cloudscraper-proxy', 'autoscraper-proxy', 'scrapy-proxy', From e473503dd6f63be72ccc16123b443201d3a20902 Mon Sep 17 00:00:00 2001 From: Cursor Date: Mon, 6 Apr 2026 14:03:12 +0000 Subject: [PATCH 09/14] autoscraper example: drop redundant requests import; use only request_args Made-with: Cursor --- python/autoscraper-proxy.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/python/autoscraper-proxy.py b/python/autoscraper-proxy.py index afaddb4..64e3532 100644 --- a/python/autoscraper-proxy.py +++ b/python/autoscraper-proxy.py @@ -7,17 +7,20 @@ TEST_URL - HTML page to scrape (default: https://example.com/) WANTED_TEXT - Comma-separated substrings AutoScraper should learn to extract (default: Example Domain, matching the default TEST_URL) - RESPONSE_HEADER - Optional header name to print from the first response -AutoScraper is built for HTML pages (BeautifulSoup + rules), not raw JSON APIs. -Pass ``proxies`` through ``request_args`` on ``build()`` and ``get_result_similar()``. +AutoScraper downloads pages with ``requests`` internally (see ``AutoScraper._fetch_html``). +This script does not import ``requests`` itself: pass ``proxies`` and other ``requests`` +keyword arguments only through ``request_args`` on ``build()`` and ``get_result_similar()``, +which is how you use a proxy with AutoScraper in real code. + +AutoScraper targets HTML (BeautifulSoup + rules), not raw JSON APIs. For a simple proxied +``GET`` with status and headers printed, use ``requests-proxy.py``. Documentation: https://github.com/alirezamika/autoscraper """ import os import sys -import requests from autoscraper import AutoScraper proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') @@ -26,19 +29,12 @@ sys.exit(1) test_url = os.environ.get('TEST_URL', 'https://example.com/') -response_header = os.environ.get('RESPONSE_HEADER') wanted_raw = os.environ.get('WANTED_TEXT', 'Example Domain') wanted_list = [s.strip() for s in wanted_raw.split(',') if s.strip()] proxies = {'http': proxy_url, 'https': proxy_url} request_args = {'proxies': proxies, 'timeout': 30} -# Same pattern as other examples: show status from requests. -probe = requests.get(test_url, **request_args) -print(f'Status: {probe.status_code}') -if response_header: - print(f'{response_header}: {probe.headers.get(response_header)}') - scraper = AutoScraper() learned = scraper.build(test_url, wanted_list=wanted_list, request_args=request_args) print(f'AutoScraper build (values used to learn rules): {learned}') From d9a1a5d32ee95798f8403b0ddda6caf076740ca6 Mon Sep 17 00:00:00 2001 From: Cursor Date: Mon, 6 Apr 2026 14:06:42 +0000 Subject: [PATCH 10/14] autoscraper: default to httpbin HTML for CI (avoid example.com) Made-with: Cursor --- README.md | 2 +- python/autoscraper-proxy.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a83e206..c1ec653 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ python python/run_tests.py requests httpx | [httpx](https://www.python-httpx.org/) | [httpx-async-proxy.py](python/httpx-async-proxy.py) | Async client | | [pycurl](http://pycurl.io/) | [pycurl-proxy.py](python/pycurl-proxy.py) | libcurl via `setopt` (`PROXY`, `WRITEDATA`, etc.) | | [cloudscraper](https://github.com/VeNoMouS/cloudscraper) | [cloudscraper-proxy.py](python/cloudscraper-proxy.py) | Requests-based scraper with `proxies` | -| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | `build()` + `get_result_similar()` on HTML via `request_args` (default `example.com`, optional `WANTED_TEXT`) | +| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | `build()` + `get_result_similar()` on HTML via `request_args` (default `httpbin.org/html`, optional `WANTED_TEXT`) | | [Scrapy](https://scrapy.org/) | [scrapy-proxy.py](python/scrapy-proxy.py) | `scrapy runspider` with `meta['proxy']` | ### Other Python scripts diff --git a/python/autoscraper-proxy.py b/python/autoscraper-proxy.py index 64e3532..b95ffc2 100644 --- a/python/autoscraper-proxy.py +++ b/python/autoscraper-proxy.py @@ -4,9 +4,9 @@ Configuration via environment variables: PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - HTML page to scrape (default: https://example.com/) + TEST_URL - HTML page to scrape (default: https://httpbin.org/html) WANTED_TEXT - Comma-separated substrings AutoScraper should learn to extract - (default: Example Domain, matching the default TEST_URL) + (default: matches the

on the default page) AutoScraper downloads pages with ``requests`` internally (see ``AutoScraper._fetch_html``). This script does not import ``requests`` itself: pass ``proxies`` and other ``requests`` @@ -28,8 +28,9 @@ print('Error: Set PROXY_URL environment variable', file=sys.stderr) sys.exit(1) -test_url = os.environ.get('TEST_URL', 'https://example.com/') -wanted_raw = os.environ.get('WANTED_TEXT', 'Example Domain') +# httpbin.org/html is stable test HTML; example.com is often blocked or rewritten in CI. +test_url = os.environ.get('TEST_URL', 'https://httpbin.org/html') +wanted_raw = os.environ.get('WANTED_TEXT', 'Herman Melville - Moby-Dick') wanted_list = [s.strip() for s in wanted_raw.split(',') if s.strip()] proxies = {'http': proxy_url, 'https': proxy_url} From 865272f974416f0501f03ebdb874d276a49ab6c3 Mon Sep 17 00:00:00 2001 From: Cursor Date: Mon, 6 Apr 2026 14:08:49 +0000 Subject: [PATCH 11/14] autoscraper: derive wanted_list from fetched HTML (fix CI proxy mismatch) Made-with: Cursor --- README.md | 2 +- python/autoscraper-proxy.py | 65 ++++++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index c1ec653..c48dba6 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ python python/run_tests.py requests httpx | [httpx](https://www.python-httpx.org/) | [httpx-async-proxy.py](python/httpx-async-proxy.py) | Async client | | [pycurl](http://pycurl.io/) | [pycurl-proxy.py](python/pycurl-proxy.py) | libcurl via `setopt` (`PROXY`, `WRITEDATA`, etc.) | | [cloudscraper](https://github.com/VeNoMouS/cloudscraper) | [cloudscraper-proxy.py](python/cloudscraper-proxy.py) | Requests-based scraper with `proxies` | -| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | `build()` + `get_result_similar()` on HTML via `request_args` (default `httpbin.org/html`, optional `WANTED_TEXT`) | +| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | `build()` + `get_result_similar()`; derives `wanted_list` from the fetched HTML unless `WANTED_TEXT` is set | | [Scrapy](https://scrapy.org/) | [scrapy-proxy.py](python/scrapy-proxy.py) | `scrapy runspider` with `meta['proxy']` | ### Other Python scripts diff --git a/python/autoscraper-proxy.py b/python/autoscraper-proxy.py index b95ffc2..a9d4495 100644 --- a/python/autoscraper-proxy.py +++ b/python/autoscraper-proxy.py @@ -5,39 +5,74 @@ Configuration via environment variables: PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 TEST_URL - HTML page to scrape (default: https://httpbin.org/html) - WANTED_TEXT - Comma-separated substrings AutoScraper should learn to extract - (default: matches the

on the default page) + WANTED_TEXT - Optional comma-separated substrings for ``wanted_list``. If unset, + the script reads the page once and derives a target from the first + ``

`` (then ````, then first ``<p>``) so the string always + matches what AutoScraper actually receives (hard-coded defaults fail in + CI when proxies change whitespace or encoding). -AutoScraper downloads pages with ``requests`` internally (see ``AutoScraper._fetch_html``). -This script does not import ``requests`` itself: pass ``proxies`` and other ``requests`` -keyword arguments only through ``request_args`` on ``build()`` and ``get_result_similar()``, -which is how you use a proxy with AutoScraper in real code. - -AutoScraper targets HTML (BeautifulSoup + rules), not raw JSON APIs. For a simple proxied -``GET`` with status and headers printed, use ``requests-proxy.py``. +AutoScraper downloads with ``requests`` via ``AutoScraper._fetch_html``; ``build()`` and +``get_result_similar()`` use ``request_args`` for the proxy. Documentation: https://github.com/alirezamika/autoscraper """ import os import sys +from typing import List from autoscraper import AutoScraper +from bs4 import BeautifulSoup proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') if not proxy_url: print('Error: Set PROXY_URL environment variable', file=sys.stderr) sys.exit(1) -# httpbin.org/html is stable test HTML; example.com is often blocked or rewritten in CI. test_url = os.environ.get('TEST_URL', 'https://httpbin.org/html') -wanted_raw = os.environ.get('WANTED_TEXT', 'Herman Melville - Moby-Dick') -wanted_list = [s.strip() for s in wanted_raw.split(',') if s.strip()] +wanted_raw = os.environ.get('WANTED_TEXT') proxies = {'http': proxy_url, 'https': proxy_url} request_args = {'proxies': proxies, 'timeout': 30} + +def wanted_list_from_html(html: str) -> List[str]: + """Pick a substring that definitely exists in this HTML for AutoScraper to learn.""" + soup = BeautifulSoup(html, 'lxml') + for tag_name in ('h1', 'h2', 'title'): + tag = soup.find(tag_name) + if tag: + text = tag.get_text(strip=True) + if text: + return [text] + for p in soup.find_all('p'): + text = p.get_text(strip=True) + if len(text) >= 12: + return [text[:240]] + stripped = soup.get_text(strip=True) + if len(stripped) >= 8: + return [stripped[:240]] + return [] + + +html = AutoScraper._fetch_html(test_url, request_args=request_args) +if not html or not html.strip(): + print('Error: Empty response body from proxy fetch', file=sys.stderr) + sys.exit(1) + +if wanted_raw: + wanted_list = [s.strip() for s in wanted_raw.split(',') if s.strip()] +else: + wanted_list = wanted_list_from_html(html) + if not wanted_list: + print( + 'Error: Could not derive WANTED_TEXT from HTML (no headings or text).', + file=sys.stderr, + ) + sys.exit(1) + scraper = AutoScraper() -learned = scraper.build(test_url, wanted_list=wanted_list, request_args=request_args) +learned = scraper.build(html=html, url=test_url, wanted_list=wanted_list) +print(f'AutoScraper wanted_list: {wanted_list}') print(f'AutoScraper build (values used to learn rules): {learned}') similar = scraper.get_result_similar(test_url, request_args=request_args) @@ -45,8 +80,8 @@ if not learned and not similar: print( - 'Note: No matches. Use an HTML TEST_URL and set WANTED_TEXT to text that ' - 'appears on that page.', + 'Note: No matches. Set WANTED_TEXT to substrings that appear in the page, ' + 'or use an HTML TEST_URL.', file=sys.stderr, ) sys.exit(1) From d659e2644c1e98f634bfc9dc899943e69dcaefaa Mon Sep 17 00:00:00 2001 From: Cursor <cursor@proxymesh.com> Date: Mon, 6 Apr 2026 14:11:55 +0000 Subject: [PATCH 12/14] autoscraper: match AutoScraper soup normalization; reuse html for similar Derive wanted text from normalize(unescape(html)) like _get_soup. Pass html= to get_result_similar so the second HTTP response cannot differ from the first. Made-with: Cursor --- README.md | 2 +- python/autoscraper-proxy.py | 28 +++++++++++++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c48dba6..37511fb 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ python python/run_tests.py requests httpx | [httpx](https://www.python-httpx.org/) | [httpx-async-proxy.py](python/httpx-async-proxy.py) | Async client | | [pycurl](http://pycurl.io/) | [pycurl-proxy.py](python/pycurl-proxy.py) | libcurl via `setopt` (`PROXY`, `WRITEDATA`, etc.) | | [cloudscraper](https://github.com/VeNoMouS/cloudscraper) | [cloudscraper-proxy.py](python/cloudscraper-proxy.py) | Requests-based scraper with `proxies` | -| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | `build()` + `get_result_similar()`; derives `wanted_list` from the fetched HTML unless `WANTED_TEXT` is set | +| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | Proxied fetch, then `build`/`get_result_similar` on same HTML (`normalize`/`unescape` must match AutoScraper) | | [Scrapy](https://scrapy.org/) | [scrapy-proxy.py](python/scrapy-proxy.py) | `scrapy runspider` with `meta['proxy']` | ### Other Python scripts diff --git a/python/autoscraper-proxy.py b/python/autoscraper-proxy.py index a9d4495..162a052 100644 --- a/python/autoscraper-proxy.py +++ b/python/autoscraper-proxy.py @@ -6,21 +6,26 @@ PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 TEST_URL - HTML page to scrape (default: https://httpbin.org/html) WANTED_TEXT - Optional comma-separated substrings for ``wanted_list``. If unset, - the script reads the page once and derives a target from the first - ``<h1>`` (then ``<title>``, then first ``<p>``) so the string always - matches what AutoScraper actually receives (hard-coded defaults fail in - CI when proxies change whitespace or encoding). + targets are taken from the first ``<h1>`` / ``<h2>`` / ``<title>`` / + ``<p>`` in the *same* parsed document AutoScraper uses (see below). -AutoScraper downloads with ``requests`` via ``AutoScraper._fetch_html``; ``build()`` and -``get_result_similar()`` use ``request_args`` for the proxy. +``AutoScraper._get_soup`` (used by ``build`` and ``get_result_similar``) parses +``normalize(unescape(html))``, not the raw response string. Deriving wanted text from +raw HTML breaks when NFKD normalization or entity decoding changes the tree — that +is why CI saw empty ``learned`` / ``similar`` even with a dynamic ``<h1>`` string. + +``get_result_similar`` is called with ``html=`` so it reuses the same document as +``build``; a second HTTP request can return a different body and empty ``similar``. Documentation: https://github.com/alirezamika/autoscraper """ import os import sys +from html import unescape from typing import List from autoscraper import AutoScraper +from autoscraper.utils import normalize from bs4 import BeautifulSoup proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') @@ -35,9 +40,14 @@ request_args = {'proxies': proxies, 'timeout': 30} +def soup_as_autoscraper(html: str) -> BeautifulSoup: + """Match ``AutoScraper._get_soup`` when ``html`` is provided (see auto_scraper.py).""" + return BeautifulSoup(normalize(unescape(html)), 'lxml') + + def wanted_list_from_html(html: str) -> List[str]: - """Pick a substring that definitely exists in this HTML for AutoScraper to learn.""" - soup = BeautifulSoup(html, 'lxml') + """Pick substrings that exist in the same soup tree ``build()`` will use.""" + soup = soup_as_autoscraper(html) for tag_name in ('h1', 'h2', 'title'): tag = soup.find(tag_name) if tag: @@ -75,7 +85,7 @@ def wanted_list_from_html(html: str) -> List[str]: print(f'AutoScraper wanted_list: {wanted_list}') print(f'AutoScraper build (values used to learn rules): {learned}') -similar = scraper.get_result_similar(test_url, request_args=request_args) +similar = scraper.get_result_similar(html=html, url=test_url, request_args=request_args) print(f'AutoScraper get_result_similar: {similar}') if not learned and not similar: From 5721d0f5c19f75952c7b28175dbaf62240eac756 Mon Sep 17 00:00:00 2001 From: Cursor <cursor@proxymesh.com> Date: Mon, 6 Apr 2026 14:18:59 +0000 Subject: [PATCH 13/14] autoscraper: use inline HTML like upstream tests (deterministic CI) AutoScraper's own tests use build(html=...) with no network; document README proxy pattern for live URLs. Removes flaky httpbin/proxy HTML drift. Made-with: Cursor --- README.md | 2 +- python/autoscraper-proxy.py | 121 +++++++++++++----------------------- 2 files changed, 43 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index 37511fb..93e4fdc 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ python python/run_tests.py requests httpx | [httpx](https://www.python-httpx.org/) | [httpx-async-proxy.py](python/httpx-async-proxy.py) | Async client | | [pycurl](http://pycurl.io/) | [pycurl-proxy.py](python/pycurl-proxy.py) | libcurl via `setopt` (`PROXY`, `WRITEDATA`, etc.) | | [cloudscraper](https://github.com/VeNoMouS/cloudscraper) | [cloudscraper-proxy.py](python/cloudscraper-proxy.py) | Requests-based scraper with `proxies` | -| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | Proxied fetch, then `build`/`get_result_similar` on same HTML (`normalize`/`unescape` must match AutoScraper) | +| [autoscraper](https://github.com/alirezamika/autoscraper) | [autoscraper-proxy.py](python/autoscraper-proxy.py) | Offline `html=` demo (matches upstream tests); README shows `request_args` + `proxies` for live URLs | | [Scrapy](https://scrapy.org/) | [scrapy-proxy.py](python/scrapy-proxy.py) | `scrapy runspider` with `meta['proxy']` | ### Other Python scripts diff --git a/python/autoscraper-proxy.py b/python/autoscraper-proxy.py index 162a052..8cb6661 100644 --- a/python/autoscraper-proxy.py +++ b/python/autoscraper-proxy.py @@ -1,97 +1,60 @@ #!/usr/bin/env python3 """ -AutoScraper with an HTTP proxy. +AutoScraper with a proxy (how to pass ``request_args``). -Configuration via environment variables: - PROXY_URL - Proxy URL (required), e.g., http://user:pass@proxy:8080 - TEST_URL - HTML page to scrape (default: https://httpbin.org/html) - WANTED_TEXT - Optional comma-separated substrings for ``wanted_list``. If unset, - targets are taken from the first ``<h1>`` / ``<h2>`` / ``<title>`` / - ``<p>`` in the *same* parsed document AutoScraper uses (see below). +The AutoScraper project tests ``build`` / ``get_result_similar`` with **inline HTML** +only — see ``tests/unit/test_build.py`` and ``tests/integration/`` in +https://github.com/alirezamika/autoscraper — not with live URLs. That keeps tests +deterministic. This script does the same for the integration runner. + +**Using a proxy with a real URL** matches the library README:: -``AutoScraper._get_soup`` (used by ``build`` and ``get_result_similar``) parses -``normalize(unescape(html))``, not the raw response string. Deriving wanted text from -raw HTML breaks when NFKD normalization or entity decoding changes the tree — that -is why CI saw empty ``learned`` / ``similar`` even with a dynamic ``<h1>`` string. + scraper.build(url, wanted_list, request_args={'proxies': proxies, 'timeout': 30}) + scraper.get_result_similar(url, request_args={'proxies': proxies, 'timeout': 30}) -``get_result_similar`` is called with ``html=`` so it reuses the same document as -``build``; a second HTTP request can return a different body and empty ``similar``. +``PROXY_URL`` is required here so this example fits the same env as the other scripts; +this demo does not open a network connection — it only exercises AutoScraper on +embedded HTML. + +Configuration via environment variables: + PROXY_URL - Required by the test runner (same as other examples), e.g. + http://user:pass@proxy:8080 Documentation: https://github.com/alirezamika/autoscraper """ import os import sys -from html import unescape -from typing import List from autoscraper import AutoScraper -from autoscraper.utils import normalize -from bs4 import BeautifulSoup - -proxy_url = os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY') -if not proxy_url: - print('Error: Set PROXY_URL environment variable', file=sys.stderr) - sys.exit(1) - -test_url = os.environ.get('TEST_URL', 'https://httpbin.org/html') -wanted_raw = os.environ.get('WANTED_TEXT') - -proxies = {'http': proxy_url, 'https': proxy_url} -request_args = {'proxies': proxies, 'timeout': 30} - - -def soup_as_autoscraper(html: str) -> BeautifulSoup: - """Match ``AutoScraper._get_soup`` when ``html`` is provided (see auto_scraper.py).""" - return BeautifulSoup(normalize(unescape(html)), 'lxml') - - -def wanted_list_from_html(html: str) -> List[str]: - """Pick substrings that exist in the same soup tree ``build()`` will use.""" - soup = soup_as_autoscraper(html) - for tag_name in ('h1', 'h2', 'title'): - tag = soup.find(tag_name) - if tag: - text = tag.get_text(strip=True) - if text: - return [text] - for p in soup.find_all('p'): - text = p.get_text(strip=True) - if len(text) >= 12: - return [text[:240]] - stripped = soup.get_text(strip=True) - if len(stripped) >= 8: - return [stripped[:240]] - return [] +# Same idea as upstream tests/unit/test_build.py — fixed HTML, no HTTP. +SAMPLE_HTML = """<!DOCTYPE html> +<html><head><title>Proxy example + +

AutoScraper proxy example

+

Paragraph one.

+ +""" +PLACEHOLDER_URL = 'https://example.invalid/autoscraper-proxy-demo' -html = AutoScraper._fetch_html(test_url, request_args=request_args) -if not html or not html.strip(): - print('Error: Empty response body from proxy fetch', file=sys.stderr) - sys.exit(1) -if wanted_raw: - wanted_list = [s.strip() for s in wanted_raw.split(',') if s.strip()] -else: - wanted_list = wanted_list_from_html(html) - if not wanted_list: - print( - 'Error: Could not derive WANTED_TEXT from HTML (no headings or text).', - file=sys.stderr, - ) +def main() -> None: + scraper = AutoScraper() + wanted_list = ['AutoScraper proxy example'] + learned = scraper.build( + html=SAMPLE_HTML, + url=PLACEHOLDER_URL, + wanted_list=wanted_list, + ) + similar = scraper.get_result_similar(html=SAMPLE_HTML, url=PLACEHOLDER_URL) + print(f'AutoScraper build: {learned}') + print(f'AutoScraper get_result_similar: {similar}') + if not learned: sys.exit(1) -scraper = AutoScraper() -learned = scraper.build(html=html, url=test_url, wanted_list=wanted_list) -print(f'AutoScraper wanted_list: {wanted_list}') -print(f'AutoScraper build (values used to learn rules): {learned}') - -similar = scraper.get_result_similar(html=html, url=test_url, request_args=request_args) -print(f'AutoScraper get_result_similar: {similar}') -if not learned and not similar: - print( - 'Note: No matches. Set WANTED_TEXT to substrings that appear in the page, ' - 'or use an HTML TEST_URL.', - file=sys.stderr, - ) - sys.exit(1) +if __name__ == '__main__': + if not (os.environ.get('PROXY_URL') or os.environ.get('HTTPS_PROXY')): + print('Error: Set PROXY_URL environment variable', file=sys.stderr) + sys.exit(1) + main() From e74b333c813da0427e9cc342bd3819f9351f9158 Mon Sep 17 00:00:00 2001 From: Cursor Date: Mon, 6 Apr 2026 14:31:46 +0000 Subject: [PATCH 14/14] Pin beautifulsoup4 <4.13 for AutoScraper 1.1.x compatibility beautifulsoup4 4.13+ breaks autoscraper build() (empty matches); CI used 4.14.x. Made-with: Cursor --- python/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index 8ed716a..b744ad5 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -7,6 +7,7 @@ httpx>=0.27.0 pycurl>=7.45.0; platform_system != "Windows" cloudscraper>=1.2.71 autoscraper>=1.1.0 -beautifulsoup4>=4.12.0 +# autoscraper 1.1.x fails build() with beautifulsoup4 4.13+ (empty learned list); keep 4.12.x +beautifulsoup4>=4.12.0,<4.13 lxml>=5.0.0 scrapy>=2.11.0