%0 Conference Paper %1 Santos_2023 %A Santos, Eddie Antonio %A Prasad, Prajish %A Becker, Brett A. %B Proceedings of the ACM Conference on Global Computing Education Vol 1 %D 2023 %I ACM %K debugging error-message explanation llm %P 147–153 %R 10.1145/3576882.3617909 %T Always Provide Context: The Effects of Code Context on Programming Error Message Enhancement %U http://dx.doi.org/10.1145/3576882.3617909 %X Programming error messages (PEMs) are notoriously difficult for novice programmers to utilise. Many efforts have been made to enhance PEMs such that they are reworded to explain problems in terms that novices can understand. However, the effectiveness of these efforts to enhance PEMs has been weak or inconclusive. This work seeks to determine the role that code context has on programming error message enhancement. Erroneous Java code written by novices was sampled from the Blackbox Mini dataset. The erroneous code was presented to expert raters with four different PEM variants: javac (control), Decaf -- an error message enhancing IDE -- and two variants generated using GPT-4: one that enhanced just the javac error message alone, and one that incorporates the code context in the prompt. We find that providing code context to LLMs increases the likelihood of correct explanations for underlying errors, produces more specific fixes for erroneous programs, and produces fixes that are more likely to be correct. In large language models, the community now has a resource that is capable of taking code context into account, to the benefit of novice programmers.

@inproceedings{Santos_2023, abstract = {Programming error messages (PEMs) are notoriously difficult for novice programmers to utilise. Many efforts have been made to enhance PEMs such that they are reworded to explain problems in terms that novices can understand. However, the effectiveness of these efforts to enhance PEMs has been weak or inconclusive. This work seeks to determine the role that code context has on programming error message enhancement. Erroneous Java code written by novices was sampled from the Blackbox Mini dataset. The erroneous code was presented to expert raters with four different PEM variants: javac (control), Decaf -- an error message enhancing IDE -- and two variants generated using GPT-4: one that enhanced just the javac error message alone, and one that incorporates the code context in the prompt. We find that providing code context to LLMs increases the likelihood of correct explanations for underlying errors, produces more specific fixes for erroneous programs, and produces fixes that are more likely to be correct. In large language models, the community now has a resource that is capable of taking code context into account, to the benefit of novice programmers.}, added-at = {2023-12-06T01:11:42.000+0100}, author = {Santos, Eddie Antonio and Prasad, Prajish and Becker, Brett A.}, biburl = {https://www.bibsonomy.org/bibtex/21f7c82fcda1e17b4a72cd843337490b7/brusilovsky}, booktitle = {Proceedings of the ACM Conference on Global Computing Education Vol 1}, collection = {CompEd 2023}, description = {Always Provide Context: The Effects of Code Context on Programming Error Message Enhancement | Proceedings of the ACM Conference on Global Computing Education Vol 1}, doi = {10.1145/3576882.3617909}, interhash = {086baae0a3ab71de07325b3c8b3ce1b9}, intrahash = {1f7c82fcda1e17b4a72cd843337490b7}, keywords = {debugging error-message explanation llm}, month = dec, pages = {147–153}, publisher = {ACM}, series = {CompEd 2023}, timestamp = {2023-12-06T01:17:36.000+0100}, title = {Always Provide Context: The Effects of Code Context on Programming Error Message Enhancement}, url = {http://dx.doi.org/10.1145/3576882.3617909}, year = 2023 }