atopile-aletheia

Python SDK for Aletheia — the evaluation platform for electrical-engineering AI agents.

Submit atopile designs, get scored against circuit simulation requirements.

Install

pip install atopile-aletheia

Or with pandas support for dataframe exports:

pip install atopile-aletheia[pandas]

Quick start

from aletheia import Aletheia

client = Aletheia()  # reads ALETHEIA_API_KEY env var
# or: client = Aletheia(api_key="ak_...")

# Browse questions
for q in client.questions.list():
    print(f"{q.id}: {q.title} (difficulty={q.difficulty})")

# Get the full prompt
q = client.questions.get("q1_555_astable")
print(q.prompt)

# Submit and wait for results
sub = client.submit(question_id=q.id, code=my_code)
print(sub.result)
# Score: 85.0% (8.5 / 10.0)
#   [PASS] Build succeeds (0.0/0.0)
#   [PASS] REQ-001: Output frequency (2.0/2.0)
#   [FAIL] REQ-002: Duty cycle (0.0/2.0)

Usage

Browse questions

# List all questions
questions = client.questions.list()

# Filter by difficulty (0-4) or family
easy = client.questions.list(difficulty=0)
analog = client.questions.list(family="analog_design")

# Get full details including prompt and skill weights
q = client.questions.get("q0_rc_lowpass")
print(q.prompt)       # the design task description
print(q.skills)       # {"analog_design": 0.5, ...}
print(q.difficulty)   # "0"
print(q.family)       # "electronics_design"

Submit code for evaluation

# Submit and block until result is ready (default)
sub = client.submit(
    question_id="q0_rc_lowpass",
    code=open("my_solution.ato").read(),
    metadata={"agent": "my-agent-v1"},  # optional, for your tracking
)

# Inspect the result
print(sub.result.score)    # 0.85 (float, 0-1)
print(sub.result.passed)   # True if all checks passed
print(sub.result.points)   # "8.5 / 10.0"

for check in sub.result.checks:
    print(f"{'PASS' if check.passed else 'FAIL'} {check.label}")
    if not check.passed:
        print(f"  {check.details}")

# Build log (compiler output, useful for debugging)
print(sub.result.build_log)

Non-blocking submissions

# Submit without waiting
sub = client.submit(question_id=q.id, code=code, wait=False)
print(sub.status)  # "pending"

# Do other work...

# Check back
sub.refresh()
print(sub.status)  # "running"

# Or block when you're ready
sub.wait(timeout=120)
print(sub.result)

Batch evaluation

Run your agent against the full benchmark:

questions = client.questions.list()

batch = client.submit_batch(
    [{"question_id": q.id, "code": agent.solve(q)} for q in questions],
    metadata={"agent": "v3", "run": "experiment-1"},
)

# Wait for all to complete
results = batch.wait(timeout=600)

print(results.summary())
# Aletheia Benchmark Results
# ----------------------------------------
# Total: 20/27 passed | Score: 78.3%
# Completed: 27 | Failed: 0
# Points: 102.5 / 135.0

# Export to pandas for analysis
df = results.to_dataframe()

Review past submissions

# List recent submissions
for s in client.submissions.list(limit=10):
    score = f"{s.result.score:.0%}" if s.result else "n/a"
    print(f"{s.question_id}: {s.status} ({score})")

# Filter by status
failed = client.submissions.list(status="failed")

# Get full details by ID
sub = client.submissions.get("some-uuid")

Configuration

client = Aletheia(
    api_key="ak_...",
    base_url="http://localhost:8000",  # for local dev
    timeout=60,
)

The client works as a context manager:

with Aletheia() as client:
    result = client.submit(question_id=q.id, code=code)

Exceptions

from aletheia import (
    AletheiaError,          # base for all errors
    AuthenticationError,    # bad or missing API key
    NotFoundError,          # question or submission not found
    ValidationError,        # invalid input (empty code, etc.)
    SubmissionTimeoutError, # .wait() exceeded timeout
    SubmissionFailedError,  # submission completed with status "failed"
)

Get an API key

Sign up at aletheia.atopile.io and create an API key from the dashboard.