thetabench 0.1.2

Gymnasium-compatible SDK for ThetaBench web agent training & evaluation

ThetaBench Python SDK

Gymnasium-compatible SDK for training and evaluating web agents on ThetaBench environments.

Installation

pip install thetabench

# With browser mode support (Playwright)
pip install 'thetabench[browser]'
playwright install chromium

Quick Start

REST Mode (Fast Training — 1-5ms/step)

import thetabench

env = thetabench.make("shopify-admin", task_id="prod-001")
obs, info = env.reset()

print(f"Task: {info['task_goal']}")
print(f"Max steps: {info['max_steps']}")

done = False
while not done:
    # Your agent decides what to do
    action = {"action": "update_product", "productId": "1", "fields": {"price": "34.99"}}
    obs, reward, terminated, truncated, info = env.step(action)
    done = terminated or truncated

result = env.finish()
print(f"Score: {result['score']:.0%} | Steps: {result['steps']} | Reward: {result['total_reward']:.2f}")
env.close()

Browser Mode (Realistic Evaluation)

import thetabench

env = thetabench.make("shopify-admin", task_id="prod-001", mode="browser")
obs, info = env.reset()

# obs includes screenshot, accessibility tree, and URL
print(f"URL: {obs['url']}")

action = {"type": "navigate", "url": "/admin/products/1"}
obs, reward, terminated, truncated, info = env.step(action)

action = {"type": "fill", "selector": "#price", "value": "34.99"}
obs, reward, terminated, truncated, info = env.step(action)

action = {"type": "click", "selector": "button:has-text('Save')"}
obs, reward, terminated, truncated, info = env.step(action)

result = env.finish()
env.close()

Curriculum Training

import thetabench

def my_agent(obs, info):
    # Your agent logic here
    return {"action": "navigate", "target": "/admin/products"}

runner = thetabench.CurriculumRunner(
    base_url="http://localhost:3000",
    mastery_threshold=0.8,
)

for stage_result in runner.run(my_agent):
    print(f"Stage {stage_result['stage']}: {stage_result['title']}")
    print(f"  Score: {stage_result['avg_score']:.0%}")
    print(f"  Mastery: {'Yes' if stage_result['mastery_achieved'] else 'No'}")

runner.close()

Browse Tasks

from thetabench import ThetaBenchClient

client = ThetaBenchClient("http://localhost:3000")

# List all tasks
tasks = client.list_tasks()
print(f"Total tasks: {tasks.total}")

# Filter by domain and difficulty
easy_product_tasks = client.list_tasks(domain="products", difficulty="easy")
for t in easy_product_tasks.tasks:
    print(f"  {t.id}: {t.title}")

# Get curriculum
curriculum = client.get_curriculum()
for stage in curriculum.stages:
    print(f"Stage {stage.stage}: {stage.title} ({len(stage.taskIds)} tasks)")

client.close()

API Reference

thetabench.make(site, task_id, mode, base_url=None, **kwargs)

Factory function for creating environments.

Param	Type	Default	Description
site	str	"shopify-admin"	Site identifier — one of shopify-admin, linear, jira, slack, zendesk, plain, inspector
task_id	str	"prod-001"	Task to run
base_url	str | None	None	ThetaBench server URL. If None, resolved from site via DEFAULT_SITE_PORTS (each site is a separate Next.js app on its own localhost port). Pass an explicit URL for Vercel deployments.
mode	str	"rest"	"rest" or "browser"
**kwargs			Forwarded to ThetaBenchEnv — supports seed, config_overrides, headless, viewport, render_mode

# Local dev — auto-resolved to http://localhost:3001
env = thetabench.make("linear", task_id="lin-thr-001")

# Production deploy — explicit URL
env = thetabench.make(
    "shopify-admin",
    task_id="prod-001",
    base_url="https://theta-shopify-admin.vercel.app",
)

ThetaBenchEnv (Gymnasium)

Method	Returns	Description
reset()	(obs, info)	Start new episode
step(action)	(obs, reward, terminated, truncated, info)	Execute action
evaluate()	dict	Mid-episode score check
finish(response?)	dict	End episode, get final score
close()	None	Cleanup

CurriculumRunner

Method	Description
run(agent_fn)	Generator yielding stage results
run_stage(stage, agent_fn)	Run all tasks in one stage
run_task(task_id, agent_fn)	Run a single task