references.bib

@article{kingma2015adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik and Ba, Jimmy},
  journal={ICLR},
  year={2015}
}

@article{goh2017why,
  author = {Goh, Gabriel},
  title = {Why Momentum Really Works},
  journal = {Distill},
  year = {2017},
  url = {http://distill.pub/2017/momentum},
  doi = {10.23915/distill.00006}
}

@article{polyak1964some,
  title={Some methods of speeding up the convergence of iteration methods},
  author={Polyak, Boris T},
  journal={Ussr computational mathematics and mathematical physics},
  volume={4},
  number={5},
  pages={1--17},
  year={1964},
  publisher={Elsevier}
}

@book{boyd2004convex,
  title={Convex Optimization},
  author={Boyd, Stephen and Vandenberghe, Lieven},
  year={2004},
  publisher={Cambridge University Press}
}

@book{nesterov2018lectures,
  title={Lectures on convex optimization},
  author={Nesterov, Yurii},
  volume={137},
  year={2018},
  publisher={Springer}
}

@article{bubeck2015convex,
  title={Convex optimization: Algorithms and complexity},
  author={Bubeck, S{\'e}bastien and others},
  journal={Foundations and Trends{\textregistered} in Machine Learning},
  volume={8},
  number={3-4},
  pages={231--357},
  year={2015},
  publisher={Now Publishers, Inc.}
}

@article{bertsekas1997nonlinear,
  title={Nonlinear programming},
  author={Bertsekas, Dimitri P},
  journal={Journal of the Operational Research Society},
  volume={48},
  number={3},
  pages={334--334},
  year={1997},
  publisher={Taylor \& Francis}
}

@article{duchi2011adagrad,
  title={Adaptive subgradient methods for online learning and stochastic optimization.},
  author={Duchi, John and Hazan, Elad and Singer, Yoram},
  journal={Journal of machine learning research},
  volume={12},
  number={7},
  year={2011}
}

@book{lan2020first,
  title={First-order and stochastic optimization methods for machine learning},
  author={Lan, Guanghui},
  volume={1},
  year={2020},
  publisher={Springer}
}

@article{hinton2012rmsprop,
  title={Neural networks for machine learning lecture 6a overview of mini-batch gradient descent},
  author={Hinton, Geoffrey and Srivastava, Nitish and Swersky, Kevin},
  journal={Cited on},
  volume={14},
  number={8},
  pages={2},
  year={2012}
}

@inproceedings{reddi2018onconvergenceofadam,
title={On the Convergence of Adam and Beyond},
author={Sashank J. Reddi and Satyen Kale and Sanjiv Kumar},
booktitle={International Conference on Learning Representations},
year={2018},
url={https://openreview.net/forum?id=ryQu7f-RZ},
}

@inproceedings{yushun2022doesadamconverge,
  author = {Zhang, Yushun and Chen, Congliang and Luo, Zhi-Quan},
  title = {Does Adam Converge and When?},
  booktitle = {ICLR Blog Track},
  year = {2022},
  note = {https://iclr-blog-track.github.io/2022/03/25/does-adam/},
  url  = {https://iclr-blog-track.github.io/2022/03/25/does-adam/}
}

@article{zhang2022adamcanconvergewithout,
  title={Adam can converge without any modification on update rules},
  author={Zhang, Yushun and Chen, Congliang and Shi, Naichen and Sun, Ruoyu and Luo, Zhi-Quan},
  journal={Advances in neural information processing systems},
  volume={35},
  pages={28386--28399},
  year={2022}
}


@article{
defossez2022asimpleconvergenceproof,
title={A Simple Convergence Proof of Adam and Adagrad},
author={Alexandre D{\'e}fossez and Leon Bottou and Francis Bach and Nicolas Usunier},
journal={Transactions on Machine Learning Research},
issn={2835-8856},
year={2022},
url={https://openreview.net/forum?id=ZPQhzTSWA7},
}

@inproceedings{
chen2018onconvergenceofadamtypealgorithms,
title={On the Convergence of A Class of Adam-Type Algorithms  for Non-Convex Optimization},
author={Xiangyi Chen and Sijia Liu and Ruoyu Sun and Mingyi Hong},
booktitle={International Conference on Learning Representations},
year={2019},
url={https://openreview.net/forum?id=H1x-x309tm},
}

@inproceedings{
loshchilov2018decoupled,
title={Decoupled Weight Decay Regularization},
author={Ilya Loshchilov and Frank Hutter},
booktitle={International Conference on Learning Representations},
year={2019},
url={https://openreview.net/forum?id=Bkg6RiCqY7},
}

@article{bernstein2024old,
  title={Old optimizer, new norm: An anthology},
  author={Bernstein, Jeremy and Newhouse, Laker},
  journal={arXiv preprint arXiv:2409.20325},
  year={2024}
}

@misc{jordan2024muon,
  author       = {Keller Jordan and Yuchen Jin and Vlado Boza and Jiacheng You and
                  Franz Cesista and Laker Newhouse and Jeremy Bernstein},
  title        = {Muon: An optimizer for hidden layers in neural networks},
  year         = {2024},
  url          = {https://kellerjordan.github.io/posts/muon/}
}

@article{liu2025muon,
  title={Muon is scalable for LLM training},
  author={Liu, Jingyuan and Su, Jianlin and Yao, Xingcheng and Jiang, Zhejun and Lai, Guokun and Du, Yulun and Qin, Yidao and Xu, Weixin and Lu, Enzhe and Yan, Junjie and others},
  journal={arXiv preprint arXiv:2502.16982},
  year={2025}
}

@inproceedings{gupta2018shampoo,
  title={Shampoo: Preconditioned stochastic tensor optimization},
  author={Gupta, Vineet and Koren, Tomer and Singer, Yoram},
  booktitle={International Conference on Machine Learning},
  pages={1842--1850},
  year={2018},
  organization={PMLR}
}

@article{shi2023distributed,
  title={A distributed data-parallel pytorch implementation of the distributed shampoo optimizer for training neural networks at-scale},
  author={Shi, Hao-Jun Michael and Lee, Tsung-Hsien and Iwasaki, Shintaro and Gallego-Posada, Jose and Li, Zhijing and Rangadurai, Kaushik and Mudigere, Dheevatsa and Rabbat, Michael},
  journal={arXiv preprint arXiv:2309.06497},
  year={2023}
}

@article{anil2020scalable,
  title={Scalable second order optimization for deep learning},
  author={Anil, Rohan and Gupta, Vineet and Koren, Tomer and Regan, Kevin and Singer, Yoram},
  journal={arXiv preprint arXiv:2002.09018},
  year={2020}
}

@inproceedings{
loshchilov2017sgdr,
title={{SGDR}: Stochastic Gradient Descent with Warm Restarts},
author={Ilya Loshchilov and Frank Hutter},
booktitle={International Conference on Learning Representations},
year={2017},
url={https://openreview.net/forum?id=Skq89Scxx}
}

@inproceedings{devlin2019bert,
  title={Bert: Pre-training of deep bidirectional transformers for language understanding},
  author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
  booktitle={Proceedings of the 2019 conference of the North American chapter of the association for computational linguistics: human language technologies, volume 1 (long and short papers)},
  pages={4171--4186},
  year={2019}
}

@article{vaswani2017attention,
  title={Attention is all you need},
  author={Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, {\L}ukasz and Polosukhin, Illia},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@article{brown2020language,
  title={Language models are few-shot learners},
  author={Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and others},
  journal={Advances in neural information processing systems},
  volume={33},
  pages={1877--1901},
  year={2020}
}

@inproceedings{
wen2025understanding,
title={Understanding Warmup-Stable-Decay Learning Rates: A River Valley Loss Landscape View},
author={Kaiyue Wen and Zhiyuan Li and Jason S. Wang and David Leo Wright Hall and Percy Liang and Tengyu Ma},
booktitle={The Thirteenth International Conference on Learning Representations},
year={2025},
url={https://openreview.net/forum?id=m51BgoqvbP}
}

@article{yang2022tensor,
  title={Tensor programs v: Tuning large neural networks via zero-shot hyperparameter transfer},
  author={Yang, Greg and Hu, Edward J and Babuschkin, Igor and Sidor, Szymon and Liu, Xiaodong and Farhi, David and Ryder, Nick and Pachocki, Jakub and Chen, Weizhu and Gao, Jianfeng},
  journal={arXiv preprint arXiv:2203.03466},
  year={2022}
}

@article{yang2023spectral,
  title={A spectral condition for feature learning},
  author={Yang, Greg and Simon, James B and Bernstein, Jeremy},
  journal={arXiv preprint arXiv:2310.17813},
  year={2023}
}

@article{malladi2023fine,
  title={Fine-tuning language models with just forward passes},
  author={Malladi, Sadhika and Gao, Tianyu and Nichani, Eshaan and Damian, Alex and Lee, Jason D and Chen, Danqi and Arora, Sanjeev},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  pages={53038--53075},
  year={2023}
}

@inproceedings{zhang2024revisiting,
author = {Zhang, Yihua and Li, Pingzhi and Hong, Junyuan and Li, Jiaxiang and Zhang, Yimeng and Zheng, Wenqing and Chen, Pin-Yu and Lee, Jason D. and Yin, Wotao and Hong, Mingyi and Wang, Zhangyang and Liu, Sijia and Chen, Tianlong},
title = {Revisiting zeroth-order optimization for memory-efficient LLM fine-tuning: a benchmark},
year = {2024},
publisher = {JMLR.org},
booktitle = {Proceedings of the 41st International Conference on Machine Learning},
articleno = {2444},
numpages = {18},
location = {Vienna, Austria},
series = {ICML'24}
}

@article{balasubramanian2022zeroth,
  title={Zeroth-order nonconvex stochastic optimization: Handling constraints, high dimensionality, and saddle points},
  author={Balasubramanian, Krishnakumar and Ghadimi, Saeed},
  journal={Foundations of Computational Mathematics},
  volume={22},
  number={1},
  pages={35--76},
  year={2022},
  publisher={Springer}
}