curlx

A powerful, lightweight HTTP client built on top of curl with advanced features including free proxy rotation, Instagram API integration, browser fingerprint randomization, smart rate-limit handling, Tor support, response caching, and circuit breaker patterns.

Table of Contents

1. Installation
2. Quick Start
3. Core HTTP Client
4. Free Proxy Rotation
5. Instagram API
6. Browser Impersonation
7. Fingerprint Randomization
8. Tor IP Rotation
9. Smart Rate Limiting
10. Response Caching
11. Retry Policies
12. Advanced Configuration
13. API Reference

Installation

pip install curlx

With Tor support:

pip install "curlx[tor]"

Requirements:

• curl installed on your system
• Python 3.8 or higher

Quick Start

import curlx

# Simple GET request
resp = curlx.get("https://httpbin.org/get", timeout=15)
print(resp.status_code, resp.json())

# POST with JSON
resp = curlx.post(
    "https://httpbin.org/post",
    json={"key": "value"},
    timeout=15
)
print(resp.json()["json"])

# Using Client with context manager
with curlx.Client(timeout=30, max_retries=3) as client:
    resp = client.get("https://api.github.com/user", auth=("user", "token"))
    print(resp.status_code)

Core HTTP Client

Client Options

HTTP Methods

client = curlx.Client(timeout=30)

client.get(url, params={...})
client.post(url, data={...}, json={...}, files={...})
client.put(url, data={...})
client.delete(url)
client.patch(url, data={...})
client.head(url)
client.options(url)

Response Object

Free Proxy Rotation

Automatically fetch and rotate free proxies from multiple sources.

Supported Proxy Sources

• ProxyScrape
• Proxy-List.download
• Geonode
• GitHub (TheSpeedX proxy list)
• Free-Proxy-List.net

Basic Usage

from curlx.proxies import FreeProxyRotator

rotator = FreeProxyRotator(rotate_every=3)
proxies = rotator.get_proxy_dict()

client = curlx.Client(proxies=proxies, timeout=30)
resp = client.get("https://api.ipify.org?format=json")
print(resp.json())

Advanced Proxy Configuration

from curlx.proxies import FreeProxyRotator, ProxyPool

# Proxy pool with auto-refresh
pool = ProxyPool(auto_refresh=True, refresh_interval=300, max_size=200)
print(f"Available proxies: {pool.size}")

# Manual rotation
rotator = FreeProxyRotator(
    rotate_every=5,
    auto_fetch=True,
    max_retries_per_proxy=3,
    fallback_direct=True,
)

# Get current IP through proxy
ip = rotator.current_ip()
print(f"Current IP: {ip}")

Instagram with Free Proxies

from curlx.instagram import InstagramAPI

api = InstagramAPI(
    use_free_proxies=True,
    random_delay=3.0,
    max_retries=5,
    timeout=30.0,
)

user = api.get_user_info("nasa")
print(f"Followers: {user.followers}")
print(f"Posts: {user.posts_count}")

Instagram API

Full-featured Instagram API wrapper with automatic proxy rotation, rate limiting, and anti-detection measures.

Features

• User profile information
• User posts with pagination
• Follower/following lists
• User search
• Post media downloads
• Profile picture downloads
• Automatic IP rotation
• Smart rate limiting

Authentication

from curlx.instagram import InstagramAPI

# Without authentication (limited)
api = InstagramAPI()

# With session (full access)
api = InstagramAPI(session_id="YOUR_SESSION_ID")

User Information

from curlx.instagram import InstagramAPI

with InstagramAPI(use_free_proxies=True) as api:
    user = api.get_user_info("nasa")
    print(f"Username: {user.username}")
    print(f"Full Name: {user.full_name}")
    print(f"Followers: {user.followers}")
    print(f"Following: {user.following}")
    print(f"Posts: {user.posts_count}")
    print(f"Verified: {user.is_verified}")
    print(f"Private: {user.is_private}")
    print(f"Biography: {user.biography}")

Get User Posts

posts, next_cursor = api.get_user_posts("nasa", count=12)
for post in posts:
    print(f"Likes: {post.likes}, Comments: {post.comments}")
    print(f"Caption: {post.caption[:100]}")
    print(f"Is Video: {post.is_video}")

Download Media

from curlx.instagram import InstagramDownloader, InstagramAPI

api = InstagramAPI(session_id="YOUR_SESSION_ID")
downloader = InstagramDownloader(api=api, output_dir="./downloads")

# Download profile picture
path = downloader.download_profile_picture("nasa", hd=True)

# Download post media
paths = downloader.download_post_media("POST_SHORTCODE")

Browser Impersonation

Mimic real browser TLS fingerprints and HTTP headers.

Available Profiles

Usage

from curlx.curl_offi import build_impersonated_client, impersonate

# Quick impersonation
client = build_impersonated_client("chrome124", timeout=30)
resp = client.get("https://www.instagram.com/")

# API mode (XHR headers)
headers, tls = impersonate("instagram_web", api_mode=True)
client = build_impersonated_client("instagram_web", api_mode=True, timeout=30)

# Random profile
from curlx.curl_offi import random_profile
profile = random_profile()

Fingerprint Randomization

Generate realistic browser fingerprints for anti-detection.

from curlx.fingerprint import (
    FingerprintRandomizer,
    random_user_agent,
    generate_browser_fingerprint,
)

# Random user agent
ua = random_user_agent("chrome")

# Full fingerprint
fp = generate_browser_fingerprint()
print(fp["user_agent"])
print(fp["viewport"])
print(fp["timezone"])

# Randomizer with rotation
randomizer = FingerprintRandomizer()
headers = randomizer.headers
api_headers = randomizer.api_headers()

# Regenerate for next request
randomizer.regenerate()

Tor IP Rotation

from curlx.rotate import TorRotator

rotator = TorRotator(rotate_every=60)

# Get proxy configuration
proxies = rotator.proxy_dict()

# Manual rotation
rotator.rotate()

# Check current IP
print(rotator.current_ip())

# Pre-configured client
with rotator.build_client(timeout=30) as client:
    resp = client.get("https://api.ipify.org?format=json")

Smart Rate Limiting

Adaptive per-domain rate limit tracking.

from curlx.rate_limit import RateLimitHandler

handler = RateLimitHandler(
    min_delay=0.5,
    max_delay=120.0,
    per_domain=True,
    jitter_factor=0.15,
    backoff_factor=2.0,
    decay_factor=0.9,
)

handler.record_rate_limit("instagram.com", retry_after=30)
wait = handler.wait_time("instagram.com")
print(f"Wait {wait}s before next request")

Response Caching

from curlx.cache import ResponseCache, MemoryCache, cache_result

# Memory cache
cache = MemoryCache(default_ttl=300, max_size=1000)
cache.set("key", value, ttl=60)
result = cache.get("key")

# Response cache (memory + file)
resp_cache = ResponseCache(memory_ttl=300, file_ttl=3600)
key = resp_cache.cache_key("GET", "https://api.example.com/data")
resp_cache.set(key, response_data)

# Decorator
@cache_result(ttl=60)
def fetch_data(url):
    return curlx.get(url).json()

Retry Policies

from curlx.retry import RetryPolicy, RetryConfig, CircuitBreaker

# Circuit breaker
cb = CircuitBreaker(failure_threshold=5, recovery_timeout=60)

# Retry policy with circuit breaker
config = RetryConfig(
    max_retries=5,
    base_delay=1.0,
    max_delay=60.0,
    circuit_breaker=cb,
)
policy = RetryPolicy(config)

# Decorator
from curlx.retry import retry_with_policy

@retry_with_policy(max_retries=3, base_delay=1.0)
def fetch_url(url):
    return curlx.get(url)

Advanced Configuration

Custom SSL and TLS

client = curlx.Client(
    verify="/path/to/ca-bundle.crt",
    tls_version="1.3",
    tls_insecure=False,
)

Proxy Configuration

client = curlx.Client(
    proxies={
        "http": "http://proxy:8080",
        "https": "http://proxy:8080",
        "all": "socks5://127.0.0.1:9050",
    }
)

Timeout Configuration

client = curlx.Client(timeout=30)
client = curlx.Client(timeout=(5, 30))
client = curlx.Client(connect_timeout=5, timeout=30)

API Reference

curlx Package

Submodules

License

MIT License - see LICENSE file for details.