SAC-pytorch 0.2.8

SAC (Soft Actor Critic) - Pytorch

Implementation of Soft Actor Critic and some of its improvements in Pytorch. Interest comes from watching this lecture and Ace from SonyAI

import torch

from SAC_pytorch import (
    SAC,
    Actor,
    Critic,
    MultipleCritics
)

critic1 = Critic(
    dim_state = 5,
    num_cont_actions = 2,
    num_discrete_actions = (5, 5),
    dim_out = 3
)

critic2 = Critic(
    dim_state = 5,
    num_cont_actions = 2,
    num_discrete_actions = (5, 5),
    dim_out = 3
)

actor = Actor(
    dim_state = 5,
    num_cont_actions = 2,
    num_discrete_actions = (5, 5)
)

agent = SAC(
    actor = actor,
    critics = [critic1, critic2],
    quantiled_critics = True
)

state = torch.randn(3, 5)
cont_actions, cont_logprob, cont_entropy, discrete, discrete_logprob = actor(state, sample = True)

agent(
    states = state,
    cont_actions = cont_actions,
    discrete_actions = discrete,
    rewards = torch.randn(1),
    done = torch.zeros(1).bool(),
    next_states = state + 1
)

Train Toy

uv run train_lunar.py

Citations

@article{Haarnoja2018SoftAA,
    title   = {Soft Actor-Critic Algorithms and Applications},
    author  = {Tuomas Haarnoja and Aurick Zhou and Kristian Hartikainen and G. Tucker and Sehoon Ha and Jie Tan and Vikash Kumar and Henry Zhu and Abhishek Gupta and P. Abbeel and Sergey Levine},
    journal = {ArXiv},
    year    = {2018},
    volume  = {abs/1812.05905},
    url     = {https://api.semanticscholar.org/CorpusID:55703664}
}

@article{Hiraoka2021DropoutQF,
    title   = {Dropout Q-Functions for Doubly Efficient Reinforcement Learning},
    author  = {Takuya Hiraoka and Takahisa Imagawa and Taisei Hashimoto and Takashi Onishi and Yoshimasa Tsuruoka},
    journal = {ArXiv},
    year    = {2021},
    volume  = {abs/2110.02034},
    url     = {https://api.semanticscholar.org/CorpusID:238353966}
}

@inproceedings{Kumar2023MaintainingPI,
    title   = {Maintaining Plasticity in Continual Learning via Regenerative Regularization},
    author  = {Saurabh Kumar and Henrik Marklund and Benjamin Van Roy},
    year    = {2023},
    url     = {https://api.semanticscholar.org/CorpusID:261076021}
}

@inproceedings{Kuznetsov2020ControllingOB,
    title   = {Controlling Overestimation Bias with Truncated Mixture of Continuous Distributional Quantile Critics},
    author  = {Arsenii Kuznetsov and Pavel Shvechikov and Alexander Grishin and Dmitry P. Vetrov},
    booktitle = {International Conference on Machine Learning},
    year    = {2020},
    url     = {https://api.semanticscholar.org/CorpusID:218581840}
}

@article{Zagoruyko2017DiracNetsTV,
    title   = {DiracNets: Training Very Deep Neural Networks Without Skip-Connections},
    author={Sergey Zagoruyko and Nikos Komodakis},
    journal = {ArXiv},
    year    = {2017},
    volume  = {abs/1706.00388},
    url     = {https://api.semanticscholar.org/CorpusID:1086822}
}

@article{Abbas2023LossOP,
    title  = {Loss of Plasticity in Continual Deep Reinforcement Learning},
    author = {Zaheer Abbas and Rosie Zhao and Joseph Modayil and Adam White and Marlos C. Machado},
    journal = {ArXiv},
    year    = {2023},
    volume  = {abs/2303.07507},
    url     = {https://api.semanticscholar.org/CorpusID:257504763}
}

@article{Zhang2024ReLU2WD,
    title   = {ReLU2 Wins: Discovering Efficient Activation Functions for Sparse LLMs},
    author  = {Zhengyan Zhang and Yixin Song and Guanghui Yu and Xu Han and Yankai Lin and Chaojun Xiao and Chenyang Song and Zhiyuan Liu and Zeyu Mi and Maosong Sun},
    journal = {ArXiv},
    year    = {2024},
    volume  = {abs/2402.03804},
    url     = {https://api.semanticscholar.org/CorpusID:267499856}
}

@inproceedings{Lee2024SimBaSB,
    title  = {SimBa: Simplicity Bias for Scaling Up Parameters in Deep Reinforcement Learning},
    author = {Hojoon Lee and Dongyoon Hwang and Donghu Kim and Hyunseung Kim and Jun Jet Tai and Kaushik Subramanian and Peter R. Wurman and Jaegul Choo and Peter Stone and Takuma Seno},
    year   = {2024},
    url    = {https://api.semanticscholar.org/CorpusID:273346233}
}

@article{Farebrother2024StopRT,
    title   = {Stop Regressing: Training Value Functions via Classification for Scalable Deep RL},
    author  = {Jesse Farebrother and Jordi Orbay and Quan Ho Vuong and Adrien Ali Taiga and Yevgen Chebotar and Ted Xiao and Alex Irpan and Sergey Levine and Pablo Samuel Castro and Aleksandra Faust and Aviral Kumar and Rishabh Agarwal},
    journal = {ArXiv},
    year   = {2024},
    volume = {abs/2403.03950},
    url    = {https://api.semanticscholar.org/CorpusID:268253088}
}

@misc{clavier2024bootstrappingexpectilesreinforcementlearning,
    title   = {Bootstrapping Expectiles in Reinforcement Learning},
    author  = {Pierre Clavier and Emmanuel Rachelson and Erwan Le Pennec and Matthieu Geist},
    year    = {2024},
    eprint  = {2406.04081},
    archivePrefix = {arXiv},
    primaryClass = {cs.LG},
    url     = {https://arxiv.org/abs/2406.04081},
}

@misc{hendawy2025useonlinenetworkcan,
    title   = {Use the Online Network If You Can: Towards Fast and Stable Reinforcement Learning},
    author  = {Ahmed Hendawy and Henrik Metternich and Théo Vincent and Mahdi Kallel and Jan Peters and Carlo D'Eramo},
    year    = {2025},
    eprint  = {2510.02590},
    archivePrefix = {arXiv},
    primaryClass = {cs.LG},
    url     = {https://arxiv.org/abs/2510.02590},
}

@misc{dellalibera2024softactorcriticbetapolicy,
    title   = {Soft Actor-Critic with Beta Policy via Implicit Reparameterization Gradients},
    author  = {Luca Della Libera},
    year    = {2024},
    eprint  = {2409.04971},
    archivePrefix = {arXiv},
    primaryClass = {cs.LG},
    url     = {https://arxiv.org/abs/2409.04971},
}

@inproceedings{obando-ceron2026simplicial,
    title   = {Simplicial Embeddings Improve Sample Efficiency in Actor{\textendash}Critic Agents},
    author  = {Johan Obando-Ceron and Walter Mayor and Samuel Lavoie and Scott Fujimoto and Aaron Courville and Pablo Samuel Castro},
    booktitle = {The Fourteenth International Conference on Learning Representations},
    year    = {2026},
    url     = {https://openreview.net/forum?id=mCpq1GCKxA}
}

@misc{lavoie2022simplicialembeddingsselfsupervisedlearning,
    title   = {Simplicial Embeddings in Self-Supervised Learning and Downstream Classification},
    author  = {Samuel Lavoie and Christos Tsirigotis and Max Schwarzer and Ankit Vani and Michael Noukhovitch and Kenji Kawaguchi and Aaron Courville},
    year    = {2022},
    eprint  = {2204.00616},
    archivePrefix = {arXiv},
    primaryClass = {cs.LG},
    url     = {https://arxiv.org/abs/2204.00616},
}

@misc{fujimoto2018addressingfunctionapproximationerror,
    title   = {Addressing Function Approximation Error in Actor-Critic Methods},
    author  = {Scott Fujimoto and Herke van Hoof and David Meger},
    year    = {2018},
    eprint  = {1802.09477},
    archivePrefix = {arXiv},
    primaryClass = {cs.AI},
    url     = {https://arxiv.org/abs/1802.09477},
}

@misc{han2026firefrobeniusisometryreinitializationbalancing,
    title   = {FIRE: Frobenius-Isometry Reinitialization for Balancing the Stability-Plasticity Tradeoff},
    author  = {Isaac Han and Sangyeon Park and Seungwon Oh and Donghu Kim and Hojoon Lee and Kyung-Joong Kim},
    year    = {2026},
    eprint  = {2602.08040},
    archivePrefix = {arXiv},
    primaryClass = {cs.LG},
    url     = {https://arxiv.org/abs/2602.08040},
}

@misc{juliani2024studyplasticitylossonpolicy,
    title   = {A Study of Plasticity Loss in On-Policy Deep Reinforcement Learning},
    author  = {Arthur Juliani and Jordan T. Ash},
    year    = {2024},
    eprint  = {2405.19153},
    archivePrefix = {arXiv},
    primaryClass = {cs.LG},
    url     = {https://arxiv.org/abs/2405.19153},
}

@misc{tian2025chunkingcritictransformerbasedsoft,
    title   = {Chunking the Critic: A Transformer-based Soft Actor-Critic with N-Step Returns},
    author  = {Dong Tian and Onur Celik and Gerhard Neumann},
    year    = {2025},
    eprint  = {2503.03660},
    archivePrefix = {arXiv},
    primaryClass = {cs.LG},
    url     = {https://arxiv.org/abs/2503.03660},
}