tensorzero 2024.9.3

The Python client for TensorZero

TensorZero Python Client

Website · Docs · Twitter · Slack · Discord

Quick Start (5min) · Comprehensive Tutorial · Deployment Guide · API Reference · Configuration Reference

The tensorzero package provides an async Python client for the TensorZero Gateway. This client allows you to easily make inference requests and assign feedback to them via the gateway.

See our API Reference for more information.

Installation

pip install tensorzero

Basic Usage

Non-Streaming Inference

import asyncio

from tensorzero import AsyncTensorZeroGateway


async def run(topic):
    async with AsyncTensorZeroGateway("http://localhost:3000") as client:
        result = await client.inference(
            function_name="generate_haiku",
            input={
                "messages": [
                    {"role": "user", "content": {"topic": topic}},
                ],
            },
        )

        print(result)


if __name__ == "__main__":
    asyncio.run(run("artificial intelligence"))

Streaming Inference

import asyncio

from tensorzero import AsyncTensorZeroGateway


async def run(topic):
    async with AsyncTensorZeroGateway("http://localhost:3000") as client:
        stream = await client.inference(
            function_name="generate_haiku",
            input={
                "messages": [
                    {"role": "user", "content": {"topic": topic}},
                ],
            },
            stream=True,
        )

        async for chunk in stream:
            print(chunk)


if __name__ == "__main__":
    asyncio.run(run("artificial intelligence"))

Feedback

import asyncio

from tensorzero import AsyncTensorZeroGateway


async def run(inference_id):
    async with AsyncTensorZeroGateway("http://localhost:3000") as client:
        result = await client.feedback(
            metric_name="thumbs_up",
            inference_id=inference_id,
            value=True,  # 👍
        )

        print(result)


if __name__ == "__main__":
    asyncio.run(run("00000000-0000-0000-0000-000000000000"))