SRT-H 0.0.23

SRT-H (wip)

Implementation of the model architecture for SRT-H out of Stanford / Johns Hopkins

Appreciation

• Pranoy for catching an issue with the L1 loss weighting for the task loss!

Citations

@misc{kim2025srthhierarchicalframeworkautonomous,
    title   = {SRT-H: A Hierarchical Framework for Autonomous Surgery via Language Conditioned Imitation Learning},
    author  = {Ji Woong Kim and Juo-Tung Chen and Pascal Hansen and Lucy X. Shi and Antony Goldenberg and Samuel Schmidgall and Paul Maria Scheikl and Anton Deguet and Brandon M. White and De Ru Tsai and Richard Cha and Jeffrey Jopling and Chelsea Finn and Axel Krieger},
    year    = {2025},
    eprint  = {2505.10251},
    archivePrefix = {arXiv},
    primaryClass = {cs.RO},
    url     = {https://arxiv.org/abs/2505.10251},
}

@article{Kim2024SurgicalRT,
    title   = {Surgical Robot Transformer (SRT): Imitation Learning for Surgical Tasks},
    author  = {Ji Woong Kim and Tony Zhao and Samuel Schmidgall and Anton Deguet and Marin Kobilarov and Chelsea Finn and Axel Krieger},
    journal = {ArXiv},
    year    = {2024},
    volume  = {abs/2407.12998},
    url     = {https://api.semanticscholar.org/CorpusID:271270134}
}

@misc{zhao2023learningfinegrainedbimanualmanipulation,
    title   = {Learning Fine-Grained Bimanual Manipulation with Low-Cost Hardware},
    author  = {Tony Z. Zhao and Vikash Kumar and Sergey Levine and Chelsea Finn},
    year    = {2023},
    eprint  = {2304.13705},
    archivePrefix = {arXiv},
    primaryClass = {cs.RO},
    url     = {https://arxiv.org/abs/2304.13705},
}

@misc{heng2025vitacformerlearningcrossmodalrepresentation,
    title   = {ViTacFormer: Learning Cross-Modal Representation for Visuo-Tactile Dexterous Manipulation},
    author  = {Liang Heng and Haoran Geng and Kaifeng Zhang and Pieter Abbeel and Jitendra Malik},
    year    = {2025},
    eprint  = {2506.15953},
    archivePrefix = {arXiv},
    primaryClass = {cs.RO},
    url     = {https://arxiv.org/abs/2506.15953},
}

@misc{shi2024yellrobotimprovingonthefly,
    title   = {Yell At Your Robot: Improving On-the-Fly from Language Corrections},
    author  = {Lucy Xiaoyang Shi and Zheyuan Hu and Tony Z. Zhao and Archit Sharma and Karl Pertsch and Jianlan Luo and Sergey Levine and Chelsea Finn},
    year    = {2024},
    eprint  = {2403.12910},
    archivePrefix = {arXiv},
    primaryClass = {cs.RO},
    url     = {https://arxiv.org/abs/2403.12910},
}