llm-std-lib

Python infrastructure library for production-grade LLM API calls.

One unified async client for OpenAI, Anthropic, Google, Azure OpenAI, AWS Bedrock, Ollama, Groq, and LM Studio — with semantic caching, model routing, resilience, and Prometheus/OTLP observability built in.

Features

Installation

# Core only
pip install llm-std-lib

# With common extras
pip install "llm-std-lib[redis,prometheus]"

# Everything
pip install "llm-std-lib[all]"

Available extras: redis, chroma, qdrant, pinecone, prometheus, otlp, local-embeddings, vault, encryption, all.

Quick start

import asyncio
from llm_std_lib import LLMClient, LLMConfig

config = LLMConfig(
    default_model="openai/gpt-4o-mini",
    providers={"openai": {"api_key": "sk-..."}},
)
client = LLMClient(config)

async def main():
    response = await client.acomplete(
        prompt="Explain semantic caching in one sentence.",
    )
    print(response.text)
    print(f"Cost: ${response.cost_usd:.6f}  |  Latency: {response.latency_ms:.0f}ms")

asyncio.run(main())

Or even shorter — load keys from environment variables:

export OPENAI_API_KEY=sk-...

client = LLMClient.from_env()
response = client.complete("What is the capital of France?")  # sync
print(response.text)  # Paris

Semantic cache

from llm_std_lib import LLMClient, LLMConfig, SemanticCache
from llm_std_lib.cache.backends.redis import RedisBackend
from llm_std_lib.cache.encoders.openai import OpenAIEncoder

cache = SemanticCache(
    backend=RedisBackend(url="redis://localhost:6379"),
    encoder=OpenAIEncoder(api_key="sk-..."),
    similarity_threshold=0.92,
)

config = LLMConfig(
    default_model="openai/gpt-4o-mini",
    providers={"openai": {"api_key": "sk-..."}},
    cache=cache,
)
client = LLMClient(config)

r1 = await client.acomplete(prompt="What is the capital of France?")
r2 = await client.acomplete(prompt="Tell me the capital city of France.")
print(r2.cached)   # True — served from cache

Model router

from llm_std_lib import LLMClient, LLMConfig, ModelRouter

router = ModelRouter.complexity_based(tiers=[
    {"models": ["openai/gpt-4o-mini"],      "max_complexity": 0.3},
    {"models": ["openai/gpt-4o"],           "max_complexity": 0.7},
    {"models": ["anthropic/claude-opus-3"], "max_complexity": 1.0},
])

client = LLMClient(LLMConfig.from_env(), router=router)

# Model is selected automatically based on prompt complexity
r = await client.acomplete(prompt="Hi")                              # → gpt-4o-mini
r = await client.acomplete(prompt="Prove Fermat's Last Theorem")     # → claude-opus-3

Resilience

from llm_std_lib.resilience import ResilienceEngine
from llm_std_lib.resilience.circuit_breaker import CircuitBreaker
from llm_std_lib.resilience.retry import RetryPolicy
from llm_std_lib.resilience.backend import InMemoryBackend

engine = ResilienceEngine(
    breaker=CircuitBreaker(
        backend=InMemoryBackend(),
        key="openai",
        failure_threshold_ratio=0.5,
        recovery_timeout=30.0,
    ),
    retryer=RetryPolicy(max_attempts=3, base_delay=0.5),
)

response = await engine.execute(
    lambda: client.acomplete(prompt="Hello")
)

Observability

from llm_std_lib.metrics import MetricsCollector, PrometheusExporter
from llm_std_lib.types import RequestContext, ResponseContext

exporter  = PrometheusExporter()
collector = MetricsCollector(on_record=exporter.update)
exporter.start_http_server(port=8000)   # scrape at http://localhost:8000/metrics

# Attach manually after each response
ctx = RequestContext(prompt="Hello", model="gpt-4o-mini", provider="openai")
response_ctx = await client._dispatch(ctx)
await collector.post_request(ctx, response_ctx)

Import grafana_dashboard.json (included in the repo) for a ready-made Grafana dashboard.

Supported providers

Configuration

From environment variables

from llm_std_lib import LLMClient

client = LLMClient.from_env()

From a YAML file

# config.yaml
default_model: openai/gpt-4o-mini
providers:
  openai:
    api_key: "${OPENAI_API_KEY}"
    timeout_ms: 15000
  anthropic:
    api_key: "${ANTHROPIC_API_KEY}"

from llm_std_lib import LLMConfig

config = LLMConfig.from_file("config.yaml")

From HashiCorp Vault

from llm_std_lib.config import VaultConfig

vault  = VaultConfig(url="https://vault.example.com", secret_path="llm/prod")
config = vault.load()

Documentation

• Getting Started Guide
• Configuration Reference
• Cookbook — 20 recipes
• Migration Guide — from LiteLLM, LangChain, raw API calls
• Contributing

License

MIT