{"id":"https://openalex.org/W4389116573","doi":"https://doi.org/10.48550/arxiv.2311.16017","title":"Decoding Logic Errors: A Comparative Study on Bug Detection by Students and Large Language Models","display_name":"Decoding Logic Errors: A Comparative Study on Bug Detection by Students and Large Language Models","publication_year":2023,"publication_date":"2023-01-01","ids":{"openalex":"https://openalex.org/W4389116573","doi":"https://doi.org/10.48550/arxiv.2311.16017"},"language":"en","primary_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2311.16017","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false},"type":"preprint","type_crossref":"posted-content","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/abs/2311.16017","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5042822346","display_name":"Stephen MacNeil","orcid":"https://orcid.org/0000-0003-2781-6619"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"MacNeil, Stephen","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032890999","display_name":"Paul Denny","orcid":"https://orcid.org/0000-0002-5150-9806"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Denny, Paul","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074358617","display_name":"Andrew Tran","orcid":"https://orcid.org/0000-0002-0094-1113"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tran, Andrew","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041367899","display_name":"Juho Leinonen","orcid":"https://orcid.org/0000-0001-6829-9449"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Leinonen, Juho","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034358464","display_name":"Seth Bernstein","orcid":"https://orcid.org/0000-0002-7552-5448"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bernstein, Seth","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5076828114","display_name":"Arto Hellas","orcid":"https://orcid.org/0000-0001-6502-209X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hellas, Arto","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5064114707","display_name":"Sami Sarsa","orcid":"https://orcid.org/0000-0002-7277-9282"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sarsa, Sami","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5101999351","display_name":"Joanne Kim","orcid":"https://orcid.org/0000-0001-7646-2373"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kim, Joanne","raw_affiliation_strings":[],"affiliations":[]}],"institution_assertions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":0,"max":69},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Empirical Studies in Software Engineering","score":0.9978,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Empirical Studies in Software Engineering","score":0.9978,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12127","display_name":"Log Analysis and System Performance Diagnosis","score":0.969,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Automated Software Testing Techniques","score":0.9689,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/log-analysis","display_name":"Log Analysis","score":0.553402},{"id":"https://openalex.org/keywords/code-clone-detection","display_name":"Code Clone Detection","score":0.545057},{"id":"https://openalex.org/keywords/software-defect-prediction","display_name":"Software Defect Prediction","score":0.542709},{"id":"https://openalex.org/keywords/source-code-analysis","display_name":"Source Code Analysis","score":0.539072},{"id":"https://openalex.org/keywords/bug-localization","display_name":"Bug Localization","score":0.536995},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.49768713}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7509209},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.63344324},{"id":"https://openalex.org/C60048249","wikidata":"https://www.wikidata.org/wiki/Q37437","display_name":"Syntax","level":2,"score":0.51439935},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.49768713},{"id":"https://openalex.org/C57273362","wikidata":"https://www.wikidata.org/wiki/Q576722","display_name":"Decoding methods","level":2,"score":0.47323048},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.45412862},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.4405647},{"id":"https://openalex.org/C122783720","wikidata":"https://www.wikidata.org/wiki/Q183065","display_name":"Interpreter","level":2,"score":0.4380911},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.43011814},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3305571},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.30798936},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.17878261},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2311.16017","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false},{"is_oa":false,"landing_page_url":"https://api.datacite.org/dois/10.48550/arxiv.2311.16017","pdf_url":null,"source":{"id":"https://openalex.org/S4393179698","display_name":"DataCite API","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I4210145204","host_organization_name":"DataCite","host_organization_lineage":["https://openalex.org/I4210145204"],"host_organization_lineage_names":["DataCite"],"type":"metadata"},"license":null,"license_id":null,"version":null}],"best_oa_location":{"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2311.16017","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":["Cornell University"],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false},"sustainable_development_goals":[{"score":0.89,"display_name":"Quality education","id":"https://metadata.un.org/sdg/4"}],"grants":[],"datasets":[],"versions":[],"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4380149490","https://openalex.org/W3089412887","https://openalex.org/W2611574984","https://openalex.org/W2548985027","https://openalex.org/W2463512549","https://openalex.org/W2412406728","https://openalex.org/W2246915267","https://openalex.org/W2004305899","https://openalex.org/W1844878253","https://openalex.org/W1721418433"],"abstract_inverted_index":{"Identifying":[0],"and":[1,71,117,151,175,179,212,226,231,252],"resolving":[2],"logic":[3,29,90,142,185,220],"errors":[4,30,61,143],"can":[5,25,31,62,73],"be":[6,32,51,63,77,155,246],"one":[7],"of":[8,112,136,149,170,184,195,210,229],"the":[9,48,69,131,139,146,168,201,224],"most":[10],"frustrating":[11],"challenges":[12],"for":[13,19,98,109,177,256],"novices":[14],"programmers.":[15],"Unlike":[16],"syntax":[17],"errors,":[18,91],"which":[20,128],"a":[21,27,54,110,181,192,207],"compiler":[22],"or":[23],"interpreter":[24],"issue":[26,49],"message,":[28],"subtle.":[33],"In":[34],"certain":[35],"conditions,":[36],"buggy":[37],"code":[38,126,150],"may":[39,153],"even":[40],"exhibit":[41],"correct":[42],"behavior":[43,135],"--":[44],"in":[45,87,219],"other":[46,140],"cases,":[47],"might":[50],"about":[52],"how":[53,242],"problem":[55],"statement":[56],"has":[57],"been":[58],"interpreted.":[59],"Such":[60],"hard":[64],"to":[65,125,145,159],"spot":[66],"when":[67,93,259],"reading":[68],"code,":[70],"they":[72],"also":[74],"at":[75],"times":[76],"missed":[78],"by":[79,161],"automated":[80],"tests.":[81],"There":[82],"is":[83],"great":[84],"educational":[85],"potential":[86,255],"automatically":[88],"detecting":[89,178],"especially":[92],"paired":[94],"with":[95,130,191],"suitable":[96],"feedback":[97],"novices.":[99],"Large":[100],"language":[101],"models":[102,244],"(LLMs)":[103],"have":[104],"recently":[105],"demonstrated":[106],"surprising":[107],"performance":[108,148,169,190],"range":[111],"computing":[113,197,249],"tasks,":[114],"including":[115],"generating":[116],"explaining":[118],"code.":[119],"These":[120],"capabilities":[121],"are":[122],"closely":[123],"linked":[124],"syntax,":[127],"aligns":[129],"next":[132],"token":[133],"prediction":[134],"LLMs.":[137,162],"On":[138],"hand,":[141],"relate":[144],"runtime":[147],"thus":[152],"not":[154],"as":[156],"well":[157],"suited":[158],"analysis":[160,209],"To":[163],"explore":[164],"this,":[165],"we":[166,215],"investigate":[167],"two":[171],"popular":[172],"LLMs,":[173,230],"GPT-3":[174],"GPT-4,":[176],"providing":[180],"novice-friendly":[182],"explanation":[183],"errors.":[186],"We":[187,240],"compare":[188],"LLM":[189,235],"large":[193],"cohort":[194],"introductory":[196],"students":[198,258],"$(n=964)$":[199],"solving":[200],"same":[202],"error":[203,221],"detection":[204],"task.":[205],"Through":[206],"mixed-methods":[208],"student":[211],"model":[213],"responses,":[214],"observe":[216],"significant":[217],"improvement":[218],"identification":[222],"between":[223],"previous":[225],"current":[227],"generation":[228],"find":[232],"that":[233],"both":[234],"generations":[236],"significantly":[237],"outperform":[238],"students.":[239],"outline":[241],"such":[243],"could":[245],"integrated":[247],"into":[248],"education":[250],"tools,":[251],"discuss":[253],"their":[254],"supporting":[257],"learning":[260],"programming.":[261]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4389116573","counts_by_year":[],"updated_date":"2024-11-15T21:54:07.184442","created_date":"2023-11-29"}
