{"id":"https://openalex.org/W4386555653","doi":"https://doi.org/10.48550/arxiv.2309.01940","title":"CodeApex: A Bilingual Programming Evaluation Benchmark for Large Language Models","display_name":"CodeApex: A Bilingual Programming Evaluation Benchmark for Large Language Models","publication_year":2023,"publication_date":"2023-09-05","ids":{"openalex":"https://openalex.org/W4386555653","doi":"https://doi.org/10.48550/arxiv.2309.01940"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2309.01940","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.01940","pdf_url":"https://arxiv.org/pdf/2309.01940","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2309.01940","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5046118256","display_name":"Lingyue Fu","orcid":"https://orcid.org/0000-0002-7827-244X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Lingyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104297727","display_name":"Huacan Chai","orcid":"https://orcid.org/0009-0006-4313-6516"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chai, Huacan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100741069","display_name":"Shuang Luo","orcid":"https://orcid.org/0000-0003-2621-9193"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luo, Shuang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074618552","display_name":"Kounianhua Du","orcid":"https://orcid.org/0000-0002-2611-5055"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Du, Kounianhua","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067689180","display_name":"Weiming Zhang","orcid":"https://orcid.org/0000-0001-5576-6108"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Weiming","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111037149","display_name":"Longteng Fan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fan, Longteng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101218824","display_name":"Jiayi Lei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lei, Jiayi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043374188","display_name":"Renting Rui","orcid":"https://orcid.org/0009-0000-7031-5786"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rui, Renting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010055657","display_name":"Jianghao Lin","orcid":"https://orcid.org/0000-0002-5935-2110"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lin, Jianghao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101552118","display_name":"Yuchen Fang","orcid":"https://orcid.org/0000-0002-7882-8698"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fang, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5027396594","display_name":"Yifan Liu","orcid":"https://orcid.org/0000-0001-8726-3927"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yifan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5052854594","display_name":"Jingkuan Wang","orcid":"https://orcid.org/0000-0002-4008-7433"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jingkuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043510420","display_name":"Siyuan Qi","orcid":"https://orcid.org/0000-0002-4070-733X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qi, Siyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101860069","display_name":"Kangning Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Kangning","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102020195","display_name":"Weinan Zhang","orcid":"https://orcid.org/0000-0001-5981-4752"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Weinan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100927581","display_name":"Yong Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yu, Yong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9966999888420105,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13629","display_name":"Text Readability and Simplification","score":0.9897000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.6531556248664856},{"id":"https://openalex.org/keywords/comprehension","display_name":"Comprehension","score":0.6237849593162537},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5805977582931519},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5687996745109558},{"id":"https://openalex.org/keywords/coding","display_name":"Coding (social sciences)","score":0.4550853371620178},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.4513600468635559},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3539082407951355},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.2970832586288452},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.10591882467269897},{"id":"https://openalex.org/keywords/management","display_name":"Management","score":0.103325754404068},{"id":"https://openalex.org/keywords/social-science","display_name":"Social science","score":0.08753997087478638},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.08056235313415527},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.07566803693771362}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.6531556248664856},{"id":"https://openalex.org/C511192102","wikidata":"https://www.wikidata.org/wiki/Q5156948","display_name":"Comprehension","level":2,"score":0.6237849593162537},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5805977582931519},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5687996745109558},{"id":"https://openalex.org/C179518139","wikidata":"https://www.wikidata.org/wiki/Q5140297","display_name":"Coding (social sciences)","level":2,"score":0.4550853371620178},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.4513600468635559},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3539082407951355},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.2970832586288452},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.10591882467269897},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.103325754404068},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.08753997087478638},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.08056235313415527},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.07566803693771362},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2309.01940","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.01940","pdf_url":"https://arxiv.org/pdf/2309.01940","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2309.01940","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2309.01940","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2309.01940","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.01940","pdf_url":"https://arxiv.org/pdf/2309.01940","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[{"score":0.6499999761581421,"id":"https://metadata.un.org/sdg/4","display_name":"Quality Education"}],"awards":[{"id":"https://openalex.org/G1740426589","display_name":null,"funder_award_id":"62177033","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386555653.pdf","grobid_xml":"https://content.openalex.org/works/W4386555653.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2378211422","https://openalex.org/W2745001401","https://openalex.org/W4321353415","https://openalex.org/W2130974462","https://openalex.org/W2028665553","https://openalex.org/W2086519370","https://openalex.org/W972276598","https://openalex.org/W2087343574","https://openalex.org/W4246352526","https://openalex.org/W2121910908"],"abstract_inverted_index":{"With":[0],"the":[1,15,26,36,59,133,146,175],"emergence":[2],"of":[3,18,29,39,68,140,178],"Large":[4],"Language":[5],"Models":[6],"(LLMs),":[7],"there":[8,154],"has":[9,43],"been":[10],"a":[11,53,171],"significant":[12,157],"improvement":[13,160],"in":[14,161],"programming":[16,27,60,135],"capabilities":[17,28,177],"models,":[19],"attracting":[20],"growing":[21],"attention":[22],"from":[23],"researchers.":[24],"Evaluating":[25],"LLMs":[30,74,92,108],"is":[31,155],"crucial":[32],"as":[33,170],"it":[34,42],"reflects":[35],"multifaceted":[37],"abilities":[38,67],"LLMs,":[40,124,179],"and":[41,64,84,101,128,143,184],"numerous":[44],"downstream":[45],"applications.":[46],"In":[47],"this":[48],"paper,":[49],"we":[50],"propose":[51],"CodeApex,":[52],"bilingual":[54],"benchmark":[55],"dataset":[56],"focusing":[57],"on":[58,75,98,145],"comprehension,":[61],"code":[62,65,88,104,113],"generation,":[63],"correction":[66,105],"LLMs.":[69],"Programming":[70],"comprehension":[71],"task":[72,90,106],"tests":[73],"multiple-choice":[76],"exam":[77],"questions":[78],"covering":[79],"conceptual":[80],"understanding,":[81],"commonsense":[82],"reasoning,":[83],"multi-hop":[85],"reasoning.":[86],"The":[87,103],"generation":[89],"evaluates":[91],"through":[93],"completing":[94],"C++":[95],"functions":[96],"based":[97],"provided":[99],"descriptions":[100],"prototypes.":[102],"asks":[107],"to":[109,151],"fix":[110],"real-world":[111],"erroneous":[112],"segments":[114],"with":[115],"different":[116],"error":[117],"messages.":[118],"We":[119,164],"evaluate":[120],"12":[121],"widely":[122],"used":[123],"including":[125],"both":[126],"general-purpose":[127],"specialized":[129],"models.":[130],"GPT-4":[131],"exhibits":[132],"best":[134],"capabilities,":[136],"achieving":[137],"approximate":[138],"accuracy":[139],"69%,":[141],"54%,":[142],"66%":[144],"three":[147],"tasks,":[148],"respectively.":[149],"Compared":[150],"human":[152],"performance,":[153],"still":[156],"room":[158],"for":[159,173],"LLM":[162],"programming.":[163],"hope":[165],"that":[166],"CodeApex":[167],"can":[168],"serve":[169],"reference":[172],"evaluating":[174],"coding":[176],"further":[180],"promoting":[181],"their":[182],"development":[183],"growth.":[185]},"counts_by_year":[{"year":2024,"cited_by_count":2}],"updated_date":"2026-08-26T07:47:46.906454","created_date":"2023-09-09T00:00:00"}