from arclm.datasets import load_dataset, validate_records, deduplicate

records = load_dataset([{"text": "ArcLM loads local records."}])
issues = validate_records(records)
clean, report = deduplicate(records)

from arclm.tokenizers import Tokenizer, AutoTokenizer

tok = Tokenizer(max_vocab=512)
tok.build("arc lm trains tiny models")
ids = tok.encode_text("arc lm")

# or from a config
tok2 = AutoTokenizer.from_config(config)  # uses config.tokenizer_type

from arclm.models import ArcLM, AutoModel

model = ArcLM(vocab_size=512, embed_dim=64, block_size=32, num_blocks=2)
# or
model = AutoModel.from_config(config)

from arclm.training import Trainer, TrainingConfig

trainer = Trainer(model=model, config=TrainingConfig(epochs=5, batch_size=8))
history = trainer.train_dataset(train_loader, val_loader=val_loader)

from arclm.finetuning import train_sft

result = train_sft(
    model="Qwen/Qwen3-0.6B",
    dataset="sft.jsonl",   # {"messages": [...]} or prompt/completion
    output_dir="out/sft",
    backend="huggingface",
)

from arclm.inference import load_model, load_any_model

# Native ArcLM checkpoint
model = load_model("models/tiny.pth")
print(model.predict("the future is", max_new_tokens=20))

# Any Hugging Face model — uses the model's own chat template
hf = load_any_model("Qwen/Qwen3-0.6B", device="cpu")
reply = hf.chat([
    {"role": "system", "content": "Be concise."},
    {"role": "user", "content": "What is an LLM?"},
])

from arclm.checkpoints import SmartLoader, safe_load_training_checkpoint, atomic_save_checkpoint

ckpt = safe_load_training_checkpoint("models/tiny.pth")
plan = SmartLoader("path/to/weights").plan()

from arclm.evaluation import Evaluator, Accuracy, Perplexity, predict_top_k

report = Evaluator(model, metrics=[Accuracy(), Perplexity()]).evaluate(loader)
top = predict_top_k(model, tokenizer, "machine learning", k=5)

from arclm.peft import LoRAConfig, apply_lora, merge_lora

apply_lora(model, LoRAConfig(r=8, alpha=16, target_modules=["attn"]))
# ... train ...
merge_lora(model)

from arclm.agents import Agent, tool

@tool
def add(a: int, b: int) -> int:
    """Add two integers."""
    return a + b

agent = Agent(model=my_model_adapter, tools=[add])
result = agent.run("What is 2 + 3?")

from arclm.security import encrypt_bytes, decrypt_bytes, redact_text, security_status

blob = encrypt_bytes(b"secret", key=key)
plain = decrypt_bytes(blob, key=key)
print(security_status())

from arclm.tracking import create_run, set_reproducible_seed
from arclm.search import create_search

set_reproducible_seed(42)
run = create_run(name="baseline", config={...})
search = create_search(space={...}, objective="min_loss")