ACCO: Accumulate While You Communicate for Communication-Overlapped Sharded LLM Training

@Article{Nabli2024ACCOAW,
 author = {Adel Nabli and Louis Fournier and Pierre Erbacher and Louis Serrano and Eugene Belilovsky and Edouard Oyallon},
 booktitle = {arXiv.org},
 journal = {ArXiv},
 title = {ACCO: Accumulate while you Communicate, Hiding Communications in Distributed LLM Training},
 volume = {abs/2406.02613},
 year = {2024}
}