Installation

uv add gyokusai

ruff check --select I --fix . && ruff format .

@software{shippole2026gyokusai,
  title        = {Gyokusai: A Web-Scale Text Processing Library},
  author       = {Shippole, Enrico and Hu, Andrew and Li, Owen and Huang, Thomas and Jitsev, Jenia},
  year         = {2026},
  url          = {https://github.com/teraflop-ai/gyokusai},
  version      = {0.1.1},
  note         = {Python package available on PyPI at \url{https://pypi.org/project/gyokusai/}}
}

@misc{paster2023openwebmath,
    title={OpenWebMath: An Open Dataset of High-Quality Mathematical Web Text},
    author={Keiran Paster and Marco Dos Santos and Zhangir Azerbayev and Jimmy Ba},
    year={2023},
    eprint={2310.06786},
    archivePrefix={arXiv},
    primaryClass={cs.AI}
}

@misc{nemo_curator,
  title = {NeMo Curator: GPU-Accelerated Data Curation for Training AI Models},
  author = {NVIDIA},
  year = {2024},
  url = {https://github.com/NVIDIA-NeMo/Curator}
}

@article{parmar2024nemotron4,
  title = {Nemotron-4 15B Technical Report},
  author = {Parmar, Jupinder and Satheesh, Shrimai and others},
  journal = {arXiv preprint arXiv:2402.16819},
  year = {2024}
}