%0 Generic %1 gandhi2023distilwhisper %A Gandhi, Sanchit %A von Platen, Patrick %A Rush, Alexander M. %D 2023 %K chunked-transcription knowledge-distillation low-latency-assistant-model pseudo-labelling speculative-decoding speech-to-text student-teacher %T Distil-Whisper: Robust Knowledge Distillation via Large-Scale Pseudo Labelling %U http://arxiv.org/abs/2311.00430 %X As the size of pre-trained speech recognition models increases, running these large models in low-latency or resource-constrained environments becomes challenging. In this work, we leverage pseudo-labelling to assemble a large-scale open-source dataset which we use to distill the Whisper model into a smaller variant, called Distil-Whisper. Using a simple word error rate (WER) heuristic, we select only the highest quality pseudo-labels for training. The distilled model is 5.8 times faster with 51% fewer parameters, while performing to within 1% WER on out-of-distribution test data in a zero-shot transfer setting. Distil-Whisper maintains the robustness of the Whisper model to difficult acoustic conditions, while being less prone to hallucination errors on long-form audio. Distil-Whisper is designed to be paired with Whisper for speculative decoding, yielding a 2 times speed-up while mathematically ensuring the same outputs as the original model. To facilitate further research in this domain, we make our training code, inference code and models publicly accessible.

@misc{gandhi2023distilwhisper, abstract = {As the size of pre-trained speech recognition models increases, running these large models in low-latency or resource-constrained environments becomes challenging. In this work, we leverage pseudo-labelling to assemble a large-scale open-source dataset which we use to distill the Whisper model into a smaller variant, called Distil-Whisper. Using a simple word error rate (WER) heuristic, we select only the highest quality pseudo-labels for training. The distilled model is 5.8 times faster with 51% fewer parameters, while performing to within 1% WER on out-of-distribution test data in a zero-shot transfer setting. Distil-Whisper maintains the robustness of the Whisper model to difficult acoustic conditions, while being less prone to hallucination errors on long-form audio. Distil-Whisper is designed to be paired with Whisper for speculative decoding, yielding a 2 times speed-up while mathematically ensuring the same outputs as the original model. To facilitate further research in this domain, we make our training code, inference code and models publicly accessible.}, added-at = {2023-11-08T11:58:23.000+0100}, author = {Gandhi, Sanchit and von Platen, Patrick and Rush, Alexander M.}, biburl = {https://www.bibsonomy.org/bibtex/2badbf2451eb9f7218e21d097c093b7e0/jasha10}, description = {2311.00430v1.pdf}, interhash = {c037c532d8178e8f744fc14866a50c82}, intrahash = {badbf2451eb9f7218e21d097c093b7e0}, keywords = {chunked-transcription knowledge-distillation low-latency-assistant-model pseudo-labelling speculative-decoding speech-to-text student-teacher}, note = {cite arxiv:2311.00430Comment: 30 pages, 2 figures, 25 tables}, timestamp = {2023-11-08T11:58:23.000+0100}, title = {Distil-Whisper: Robust Knowledge Distillation via Large-Scale Pseudo Labelling}, url = {http://arxiv.org/abs/2311.00430}, year = 2023 }