%0 Generic %1 zhu2023multilingual %A Zhu, Wenhao %A Liu, Hongyi %A Dong, Qingxiu %A Xu, Jingjing %A Kong, Lingpeng %A Chen, Jiajun %A Li, Lei %A Huang, Shujian %D 2023 %K artificial_intelligence generative_AI large_language_model translation %T Multilingual Machine Translation with Large Language Models: Empirical Results and Analysis %U http://arxiv.org/abs/2304.04675 %X Large language models (LLMs) have demonstrated remarkable potential in handling multilingual machine translation (MMT). In this paper, we systematically investigate the advantages and challenges of LLMs for MMT by answering two questions: 1) How well do LLMs perform in translating a massive number of languages? 2) Which factors affect LLMs' performance in translation? We evaluate popular LLMs, including XGLM, OPT, BLOOMZ, and ChatGPT, on 102 languages. Our empirical results show that even the best model ChatGPT still lags behind the supervised baseline NLLB in 83.33% of translation directions. Through further analysis, we discover that LLMs exhibit new working patterns when used for MMT. First, prompt semantics can surprisingly be ignored when given in-context exemplars, where LLMs still show strong performance even with unreasonable prompts. Second, cross-lingual exemplars can provide better task instruction for low-resource translation than exemplars in the same language pairs. Third, we observe the overestimated performance of BLOOMZ on dataset Flores-101, indicating the potential risk when using public datasets for evaluation.

@misc{zhu2023multilingual, abstract = {Large language models (LLMs) have demonstrated remarkable potential in handling multilingual machine translation (MMT). In this paper, we systematically investigate the advantages and challenges of LLMs for MMT by answering two questions: 1) How well do LLMs perform in translating a massive number of languages? 2) Which factors affect LLMs' performance in translation? We evaluate popular LLMs, including XGLM, OPT, BLOOMZ, and ChatGPT, on 102 languages. Our empirical results show that even the best model ChatGPT still lags behind the supervised baseline NLLB in 83.33% of translation directions. Through further analysis, we discover that LLMs exhibit new working patterns when used for MMT. First, prompt semantics can surprisingly be ignored when given in-context exemplars, where LLMs still show strong performance even with unreasonable prompts. Second, cross-lingual exemplars can provide better task instruction for low-resource translation than exemplars in the same language pairs. Third, we observe the overestimated performance of BLOOMZ on dataset Flores-101, indicating the potential risk when using public datasets for evaluation.}, added-at = {2023-04-11T20:33:39.000+0200}, author = {Zhu, Wenhao and Liu, Hongyi and Dong, Qingxiu and Xu, Jingjing and Kong, Lingpeng and Chen, Jiajun and Li, Lei and Huang, Shujian}, biburl = {https://www.bibsonomy.org/bibtex/23ca9d548f7a7422c39d6cea2dd3fca6b/lumeng}, description = {Multilingual Machine Translation with Large Language Models: Empirical Results and Analysis}, interhash = {af2f04db140a8e093a0f64af4d40f9ba}, intrahash = {3ca9d548f7a7422c39d6cea2dd3fca6b}, keywords = {artificial_intelligence generative_AI large_language_model translation}, note = {cite arxiv:2304.04675}, timestamp = {2023-04-11T20:33:39.000+0200}, title = {Multilingual Machine Translation with Large Language Models: Empirical Results and Analysis}, url = {http://arxiv.org/abs/2304.04675}, year = 2023 }