Scraping-friendly HTTP, string parsing, and JSON helpers.
Project description
Respondo
Scraping-friendly Python library for response parsing, text extraction, and JSON detection. Zero external dependencies—standard library only.
Installation
pip install respondo
Quick Example
from respondo import between, find_first_json, extract_links, Response
# Extract text between delimiters
between("<title>Hello World</title>", "<title>", "</title>")
# => "Hello World"
# Find JSON embedded in text
find_first_json('callback({"user": "alice", "id": 42})')
# => {"user": "alice", "id": 42}
# Parse HTTP response
resp = Response(200, {"content-type": "text/html"}, b"<a href='/page'>Link</a>")
resp.is_success() # => True
extract_links(resp.text, base="https://example.com")
# => ["https://example.com/page"]
Error Handling
All functions return empty values instead of raising exceptions. This is intentional for scraping where missing data is expected.
between("no match here", "<a>", "</a>") # => ""
find_first_json("not json") # => None
regex_first("abc", r"\d+") # => ""
b64_decode("invalid!!!") # => b""
json_get({"a": 1}, "b", "c") # => None
API Reference
Text Extraction
from respondo import between, betweens, between_last, between_n
# First match between delimiters
between("a]x[b]y[c", "[", "]") # => "b"
# All matches
betweens("a[b]c[d]e", "[", "]") # => ["b", "d"]
# Last occurrence of left delimiter
between_last("a[b]c[d]e", "[", "]") # => "d"
# Nth match (1-indexed)
between_n("a[b]c[d]e", "[", "]", 2) # => "d"
Regex
from respondo import regex_first, regex_all
# First match (returns capture group if present)
regex_first("price: $42.99", r"\$(\d+)") # => "42"
regex_first("abc123def", r"\d+") # => "123"
# All matches
regex_all("<a>1</a><a>2</a>", r"<a>(\d+)</a>") # => ["1", "2"]
HTML Processing
from respondo import (
strip_tags,
get_text,
extract_links,
extract_forms,
extract_tables,
parse_csrf_token,
)
html = """
<html>
<script>var x=1;</script>
<body>
<h1>Title</h1>
<p>Hello & welcome</p>
<a href="/page1">Link 1</a>
<a href="/page2">Link 2</a>
</body>
</html>
"""
# Strip HTML tags (keeps script/style content)
strip_tags("<b>bold</b> text") # => "bold text"
# Extract visible text (removes scripts/styles, decodes entities)
get_text(html) # => "Title Hello & welcome Link 1 Link 2"
# Extract all links
extract_links(html, base="https://example.com")
# => ["https://example.com/page1", "https://example.com/page2"]
# Filter to same host only
extract_links(html, base="https://example.com", same_host=True)
# Filter by extension
extract_links(html, extensions=[".pdf", ".doc"])
# Extract forms with fields
extract_forms('<form action="/login" method="post"><input name="user"></form>')
# => [{"action": "/login", "method": "post", "fields": {"user": ""}}]
# Extract tables as dicts
extract_tables("<table><tr><th>Name</th></tr><tr><td>Alice</td></tr></table>")
# => [{"headers": ["Name"], "rows": [{"Name": "Alice"}]}]
# Find CSRF tokens
parse_csrf_token('<input name="csrf_token" value="abc123">') # => "abc123"
parse_csrf_token('<meta name="csrf-token" content="xyz">') # => "xyz"
JSON Utilities
from respondo import find_first_json, find_all_json, json_get, json_in_html
# Find first valid JSON in text
find_first_json('prefix {"a": 1} middle [1,2,3] suffix')
# => {"a": 1}
# Find all JSON objects/arrays
find_all_json('{"a":1} text [1,2] more {"b":2}')
# => [{"a": 1}, [1, 2], {"b": 2}]
# Safe nested access (returns None if path missing)
data = {"user": {"profile": {"name": "Alice"}}}
json_get(data, "user", "profile", "name") # => "Alice"
json_get(data, "user", "missing", "key") # => None
# Array indexing
json_get([{"id": 1}, {"id": 2}], 0, "id") # => 1
# Extract JSON from HTML script tags
html = '<script type="application/json">{"config": true}</script>'
json_in_html(html) # => [{"config": true}]
Encoding Utilities
from respondo import url_encode, url_decode, b64_encode, b64_decode
# URL encoding
url_encode({"q": "hello world", "page": 1}) # => "q=hello+world&page=1"
url_encode({"tags": ["a", "b"]}) # => "tags=a&tags=b"
# URL decoding (returns lists for multi-values)
url_decode("a=1&b=2&b=3") # => {"a": ["1"], "b": ["2", "3"]}
# Base64
b64_encode("hello") # => "aGVsbG8="
b64_decode("aGVsbG8=") # => b"hello"
# URL-safe base64
b64_encode("data", urlsafe=True)
b64_decode("ZGF0YQ", urlsafe=True)
Response Class
from respondo import Response
# Create from status, headers, body
resp = Response(
status=200,
headers={"Content-Type": "application/json"},
body=b'{"ok": true}'
)
# Status checks
resp.is_success() # 200-299
resp.is_redirect() # 300-399
resp.is_client_error() # 400-499
resp.is_server_error() # 500-599
# Headers (case-insensitive)
resp.header("content-type") # First value: "application/json"
resp.headers_all("set-cookie") # All values: ["a=1", "b=2"]
# Content type parsing
resp.content_type() # => ("application/json", "utf-8")
# Body access
resp.body # Raw bytes
resp.text # Decoded string (utf-8)
resp.json() # Parsed JSON
# Hashing
resp.hash() # SHA-256 of body
resp.hash("md5") # MD5 of body
# Charset detection (from headers or <meta>)
text, charset = resp.charset_sniff()
# Cookie parsing
resp.cookies()
# => [{"name": "session", "value": "abc", "attrs": {"path": "/", "httponly": ""}}]
# HTML helpers (delegates to htmlutil)
resp.extract_links(base="https://example.com", same_host=True)
resp.extract_json() # Find JSON in HTML
resp.strip_scripts_styles() # Clean HTML text
Real-World Examples
Scrape Hacker News titles
import urllib.request
from respondo import Response, regex_all, normalize_space
req = urllib.request.urlopen("https://news.ycombinator.com")
resp = Response(req.status, dict(req.headers), req.read())
titles = regex_all(resp.text, r'class="titleline"[^>]*><a[^>]*>([^<]+)</a>')
for title in titles[:5]:
print(normalize_space(title))
Extract data from JSON API
from respondo import Response, json_get
resp = Response(200, {}, b'{"users": [{"name": "Alice"}, {"name": "Bob"}]}')
data = resp.json()
# Safe traversal
first_user = json_get(data, "users", 0, "name") # => "Alice"
missing = json_get(data, "users", 99, "name") # => None (no error)
Parse form and submit
from respondo import extract_forms, parse_csrf_token, url_encode
html = '''
<form action="/login" method="post">
<input name="csrf" value="token123">
<input name="username">
<input name="password" type="password">
</form>
'''
forms = extract_forms(html, base="https://example.com")
# => [{"action": "https://example.com/login", "method": "post", "fields": {"csrf": "token123", "username": "", "password": ""}}]
# Or just grab the CSRF token
csrf = parse_csrf_token(html) # => "token123"
Running Tests
cd src
python demo_real_sites.py
License
MIT
Download files
Download the file for your platform. If you're not sure which to choose, learn more about installing packages.
Source Distribution
respondo-0.1.0.tar.gz
(10.7 kB
view details)
Built Distribution
Filter files by name, interpreter, ABI, and platform.
If you're not sure about the file name format, learn more about wheel file names.
Copy a direct link to the current filters
respondo-0.1.0-py3-none-any.whl
(10.3 kB
view details)
File details
Details for the file respondo-0.1.0.tar.gz.
File metadata
- Download URL: respondo-0.1.0.tar.gz
- Upload date:
- Size: 10.7 kB
- Tags: Source
- Uploaded using Trusted Publishing? No
- Uploaded via: twine/6.2.0 CPython/3.13.2
File hashes
| Algorithm | Hash digest | |
|---|---|---|
| SHA256 |
2a8ebcd15c7bdcb5324867960bf1efa2719c0cc3a591bac60b2a57a38b52d22e
|
|
| MD5 |
baae5d80a515a208d93babc9839661e3
|
|
| BLAKE2b-256 |
9f86bf860a5bb850e86933be182ca571f32315489c2fd4c439cb4fca493c49d3
|
File details
Details for the file respondo-0.1.0-py3-none-any.whl.
File metadata
- Download URL: respondo-0.1.0-py3-none-any.whl
- Upload date:
- Size: 10.3 kB
- Tags: Python 3
- Uploaded using Trusted Publishing? No
- Uploaded via: twine/6.2.0 CPython/3.13.2
File hashes
| Algorithm | Hash digest | |
|---|---|---|
| SHA256 |
7d3bb1065b5b7f2457618134427975f79c979f8aeb43e323a2975ce647f6eca6
|
|
| MD5 |
ed4b0b3ca21ab7a9583f7e027af61ee3
|
|
| BLAKE2b-256 |
b1200ed7773c8fbce338971d7140add4258419f010fa60ca373a65fa8e0612b2
|
