%0 Journal Article %1 Prather_2023 %A Prather, James %A Reeves, Brent N. %A Denny, Paul %A Becker, Brett A. %A Leinonen, Juho %A Luxton-Reilly, Andrew %A Powell, Garrett %A Finnie-Ansley, James %A Santos, Eddie Antonio %D 2023 %I Association for Computing Machinery (ACM) %J ACM Transactions on Computer-Human Interaction %K llm programming usability user-study %N 1 %P 1–31 %R 10.1145/3617367 %T “It’s Weird That it Knows What I Want”: Usability and Interactions with Copilot for Novice Programmers %U http://dx.doi.org/10.1145/3617367 %V 31 %X Recent developments in deep learning have resulted in code-generation models that produce source code from natural language and code-based prompts with high accuracy. This is likely to have profound effects in the classroom, where novices learning to code can now use free tools to automatically suggest solutions to programming exercises and assignments. However, little is currently known about how novices interact with these tools in practice. We present the first study that observes students at the introductory level using one such code auto-generating tool, Github Copilot, on a typical introductory programming (CS1) assignment. Through observations and interviews we explore student perceptions of the benefits and pitfalls of this technology for learning, present new observed interaction patterns, and discuss cognitive and metacognitive difficulties faced by students. We consider design implications of these findings, specifically in terms of how tools like Copilot can better support and scaffold the novice programming experience.

@article{Prather_2023, abstract = {Recent developments in deep learning have resulted in code-generation models that produce source code from natural language and code-based prompts with high accuracy. This is likely to have profound effects in the classroom, where novices learning to code can now use free tools to automatically suggest solutions to programming exercises and assignments. However, little is currently known about how novices interact with these tools in practice. We present the first study that observes students at the introductory level using one such code auto-generating tool, Github Copilot, on a typical introductory programming (CS1) assignment. Through observations and interviews we explore student perceptions of the benefits and pitfalls of this technology for learning, present new observed interaction patterns, and discuss cognitive and metacognitive difficulties faced by students. We consider design implications of these findings, specifically in terms of how tools like Copilot can better support and scaffold the novice programming experience.}, added-at = {2023-12-08T04:33:48.000+0100}, author = {Prather, James and Reeves, Brent N. and Denny, Paul and Becker, Brett A. and Leinonen, Juho and Luxton-Reilly, Andrew and Powell, Garrett and Finnie-Ansley, James and Santos, Eddie Antonio}, biburl = {https://www.bibsonomy.org/bibtex/2d45034fd3c4fd6c5f8e95d3884ee3abf/brusilovsky}, description = {“It’s Weird That it Knows What I Want”: Usability and Interactions with Copilot for Novice Programmers | ACM Transactions on Computer-Human Interaction}, doi = {10.1145/3617367}, interhash = {a455c11266e61454d3f8462123724d74}, intrahash = {d45034fd3c4fd6c5f8e95d3884ee3abf}, issn = {1557-7325}, journal = {ACM Transactions on Computer-Human Interaction}, keywords = {llm programming usability user-study}, month = nov, number = 1, pages = {1–31}, publisher = {Association for Computing Machinery (ACM)}, timestamp = {2023-12-08T04:33:48.000+0100}, title = {“It’s Weird That it Knows What I Want”: Usability and Interactions with Copilot for Novice Programmers}, url = {http://dx.doi.org/10.1145/3617367}, volume = 31, year = 2023 }