{"id":"https://openalex.org/W7166662328","doi":"https://doi.org/10.1145/3797106","title":"RealBench: A Repo-Level Code Generation Benchmark Aligned with Real-World Software Development Practices","display_name":"RealBench: A Repo-Level Code Generation Benchmark Aligned with Real-World Software Development Practices","publication_year":2026,"publication_date":"2026-06-30","ids":{"openalex":"https://openalex.org/W7166662328","doi":"https://doi.org/10.1145/3797106"},"language":"en","primary_location":{"id":"doi:10.1145/3797106","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797106","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1145/3797106","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101755477","display_name":"Jia Li","orcid":"https://orcid.org/0000-0002-9411-971X"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jia Li","raw_affiliation_strings":["Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0000-0002-9411-971X","affiliations":[{"raw_affiliation_string":"Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111691137","display_name":"H. Deng","orcid":null},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongyi Deng","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0007-6658-2823","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107249132","display_name":"Yiran Zhang","orcid":"https://orcid.org/0000-0002-9366-6076"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yiran Zhang","raw_affiliation_strings":["Nanyang Technological University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-9366-6076","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5069561378","display_name":"Kechi Zhang","orcid":"https://orcid.org/0000-0002-3290-0244"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kechi Zhang","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-3290-0244","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134720971","display_name":"Tianqi Shao","orcid":null},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tianqi Shao","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0008-1503-5077","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134012719","display_name":"Tiankuo Zhao","orcid":null},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Tiankuo Zhao","raw_affiliation_strings":["Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0009-0004-1869-7106","affiliations":[{"raw_affiliation_string":"Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100650914","display_name":"Weinan Wang","orcid":"https://orcid.org/0000-0002-7280-9023"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Weinan Wang","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0002-5762-6277","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5049100391","display_name":"Zhi Jin","orcid":"https://orcid.org/0000-0003-1087-226X"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhi Jin","raw_affiliation_strings":["Wuhan University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1087-226X","affiliations":[{"raw_affiliation_string":"Wuhan University, Beijing, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100447682","display_name":"Ge Li","orcid":"https://orcid.org/0000-0002-5828-0186"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ge Li","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5828-0186","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100355692","display_name":"Yang Liu","orcid":"https://orcid.org/0000-0001-7300-9215"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yang Liu","raw_affiliation_strings":["Nanyang Technological University, Singapore, Singapore"],"raw_orcid":"https://orcid.org/0000-0001-7300-9215","affiliations":[{"raw_affiliation_string":"Nanyang Technological University, Singapore, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113127736","display_name":"Yingtao Fang","orcid":"https://orcid.org/0009-0000-8253-226X"},"institutions":[{"id":"https://openalex.org/I37461747","display_name":"Wuhan University","ror":"https://ror.org/033vjfk17","country_code":"CN","type":"education","lineage":["https://openalex.org/I37461747"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yingtao Fang","raw_affiliation_strings":["Wuhan University, Wuhan, China"],"raw_orcid":"https://orcid.org/0009-0000-8253-226X","affiliations":[{"raw_affiliation_string":"Wuhan University, Wuhan, China","institution_ids":["https://openalex.org/I37461747"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5018401644","display_name":"Yihong Dong","orcid":"https://orcid.org/0000-0001-6228-4019"},"institutions":[{"id":"https://openalex.org/I20231570","display_name":"Peking University","ror":"https://ror.org/02v51f717","country_code":"CN","type":"education","lineage":["https://openalex.org/I20231570"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yihong Dong","raw_affiliation_strings":["Peking University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-6228-4019","affiliations":[{"raw_affiliation_string":"Peking University, Beijing, China","institution_ids":["https://openalex.org/I20231570"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.89256677,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"3","issue":"FSE","first_page":"1735","last_page":"1756"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8011999726295471,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.8011999726295471,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11450","display_name":"Model-Driven Software Engineering Techniques","score":0.04690000042319298,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.036400001496076584,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/code-generation","display_name":"Code generation","score":0.6431000232696533},{"id":"https://openalex.org/keywords/software-development","display_name":"Software development","score":0.5706999897956848},{"id":"https://openalex.org/keywords/natural-language-generation","display_name":"Natural language generation","score":0.5169000029563904},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4584999978542328},{"id":"https://openalex.org/keywords/kpi-driven-code-analysis","display_name":"KPI-driven code analysis","score":0.43720000982284546},{"id":"https://openalex.org/keywords/unified-modeling-language","display_name":"Unified Modeling Language","score":0.43470001220703125},{"id":"https://openalex.org/keywords/code-review","display_name":"Code review","score":0.38339999318122864},{"id":"https://openalex.org/keywords/software-quality","display_name":"Software quality","score":0.3619999885559082},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.33799999952316284}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6815999746322632},{"id":"https://openalex.org/C133162039","wikidata":"https://www.wikidata.org/wiki/Q1061077","display_name":"Code generation","level":3,"score":0.6431000232696533},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.6277999877929688},{"id":"https://openalex.org/C529173508","wikidata":"https://www.wikidata.org/wiki/Q638608","display_name":"Software development","level":3,"score":0.5706999897956848},{"id":"https://openalex.org/C2776187449","wikidata":"https://www.wikidata.org/wiki/Q1513879","display_name":"Natural language generation","level":3,"score":0.5169000029563904},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4584999978542328},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.4431999921798706},{"id":"https://openalex.org/C121957198","wikidata":"https://www.wikidata.org/wiki/Q14365593","display_name":"KPI-driven code analysis","level":5,"score":0.43720000982284546},{"id":"https://openalex.org/C145644426","wikidata":"https://www.wikidata.org/wiki/Q169411","display_name":"Unified Modeling Language","level":3,"score":0.43470001220703125},{"id":"https://openalex.org/C150292731","wikidata":"https://www.wikidata.org/wiki/Q1342704","display_name":"Code review","level":5,"score":0.38339999318122864},{"id":"https://openalex.org/C117447612","wikidata":"https://www.wikidata.org/wiki/Q1412670","display_name":"Software quality","level":4,"score":0.3619999885559082},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.33799999952316284},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3197000026702881},{"id":"https://openalex.org/C180152950","wikidata":"https://www.wikidata.org/wiki/Q2904257","display_name":"Software development process","level":4,"score":0.3122999966144562},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2897000014781952},{"id":"https://openalex.org/C509989072","wikidata":"https://www.wikidata.org/wiki/Q15188241","display_name":"Model-driven architecture","level":4,"score":0.28040000796318054},{"id":"https://openalex.org/C149091818","wikidata":"https://www.wikidata.org/wiki/Q2429814","display_name":"Software system","level":3,"score":0.2759999930858612},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.27129998803138733},{"id":"https://openalex.org/C4478048","wikidata":"https://www.wikidata.org/wiki/Q950250","display_name":"Test-driven development","level":4,"score":0.2639999985694885},{"id":"https://openalex.org/C186846655","wikidata":"https://www.wikidata.org/wiki/Q3398377","display_name":"Software construction","level":4,"score":0.2637999951839447},{"id":"https://openalex.org/C2779913896","wikidata":"https://www.wikidata.org/wiki/Q7063001","display_name":"Notice","level":2,"score":0.25920000672340393},{"id":"https://openalex.org/C146222976","wikidata":"https://www.wikidata.org/wiki/Q1204997","display_name":"Business logic","level":2,"score":0.25850000977516174},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2578999996185303},{"id":"https://openalex.org/C51929080","wikidata":"https://www.wikidata.org/wiki/Q2425187","display_name":"Codebase","level":3,"score":0.2531999945640564},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.25119999051094055}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3797106","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797106","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1145/3797106","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3797106","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/9","display_name":"Industry, innovation and infrastructure","score":0.47358328104019165}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":38,"referenced_works":["https://openalex.org/W2955972181","https://openalex.org/W2964315653","https://openalex.org/W3027879771","https://openalex.org/W3161457214","https://openalex.org/W3177813494","https://openalex.org/W4220672926","https://openalex.org/W4226287673","https://openalex.org/W4226485558","https://openalex.org/W4281690148","https://openalex.org/W4287024925","https://openalex.org/W4289670534","https://openalex.org/W4307479317","https://openalex.org/W4309804142","https://openalex.org/W4309986599","https://openalex.org/W4312091822","https://openalex.org/W4360834937","https://openalex.org/W4385562549","https://openalex.org/W4385902209","https://openalex.org/W4387225871","https://openalex.org/W4390897512","https://openalex.org/W4390897882","https://openalex.org/W4391272793","https://openalex.org/W4391558462","https://openalex.org/W4393213295","https://openalex.org/W4393905718","https://openalex.org/W4394744510","https://openalex.org/W4399357038","https://openalex.org/W4399598202","https://openalex.org/W4401671212","https://openalex.org/W4402671782","https://openalex.org/W4403536900","https://openalex.org/W4403579368","https://openalex.org/W4404344157","https://openalex.org/W4405903187","https://openalex.org/W4406273368","https://openalex.org/W4415957649","https://openalex.org/W4416113188","https://openalex.org/W7133187544"],"related_works":[],"abstract_inverted_index":{"Writing":[0],"code":[1,25,44,59,93,110,152,193,207,290,316],"requires":[2],"significant":[3,217],"time":[4],"and":[5,31,53,77,127,173,214,228,247],"effort":[6],"in":[7,50,187,232,307],"software":[8,98,117,198],"development.":[9],"To":[10,101],"automate":[11,97],"this":[12,103],"process,":[13],"researchers":[14],"have":[15,33],"made":[16],"substantial":[17],"progress":[18],"using":[19],"Large":[20],"Language":[21],"Models":[22],"(LLMs)":[23],"for":[24,191,288,313],"generation.":[26,317],"Many":[27],"benchmarks":[28,76],"like":[29],"HumanEval":[30],"EvoCodeBench":[32],"been":[34],"created":[35],"to":[36,42,167,245,277,310],"evaluate":[37,168],"LLMs":[38,170,209,223],"by":[39],"requiring":[40],"them":[41],"generate":[43],"from":[45],"natural":[46,69,124],"language":[47,70,125],"requirements.":[48],"However,":[49],"enterprise":[51],"applications":[52],"team":[54],"development,":[55],"developers":[56,135],"typically":[57,136],"write":[58],"based":[60],"on":[61,140,171,262,297],"structured":[62,158],"designs":[63],"or":[64],"specifications":[65],"rather":[66],"than":[67],"raw":[68],"descriptions.":[71],"This":[72],"gap":[73],"between":[74],"existing":[75],"real":[78],"industry":[79,116],"development":[80,99,118,199],"practices":[81],"means":[82],"that":[83,204],"current":[84,188],"benchmark":[85,112],"scores":[86],"may":[87],"not":[88],"accurately":[89],"reflect":[90],"how":[91,134],"much":[92,211],"generation":[94,111,153,165,194,260,291],"can":[95],"help":[96],"tasks.":[100],"address":[102],"gap,":[104],"we":[105,144,162,202,301],"propose":[106,174],"RealBench,":[107],"a":[108,146,267],"repository-level":[109,289],"aligned":[113,195],"with":[114,157,178,196,270],"real-world":[115,197],"practices.":[119,200],"Each":[120],"example":[121],"includes":[122],"both":[123],"requirements":[126],"UML":[128,233],"diagrams":[129],"as":[130],"system":[131,159,283,298],"design,":[132],"matching":[133],"receive":[137],"specifications.":[138],"Based":[139],"the":[141,236,252,258,271,281,303],"constructed":[142],"benchmarks,":[143],"conduct":[145],"systematic":[147],"evaluation":[148,176],"of":[149,238],"advanced":[150,169],"LLMs'":[151,189],"capabilities":[154,190],"when":[155],"provided":[156],"designs.":[160,299],"Specifically,":[161],"design":[163,284],"three":[164],"strategies":[166],"RealBench":[172],"two":[175],"granularities":[177],"five":[179],"metrics.":[180],"The":[181],"experimental":[182],"results":[183],"reveal":[184],"key":[185],"insights":[186,312],"repo-level":[192,206,315],"First,":[201],"notice":[203],"regarding":[205],"generation,":[208],"show":[210],"worse":[212],"performance":[213,218],"there":[215],"are":[216,224],"gaps":[219],"among":[220],"LLMs.":[221],"Second,":[222],"good":[225],"at":[226,255],"finding":[227],"creating":[229],"modules":[230,240],"defined":[231],"diagrams,":[234],"but":[235],"quality":[237],"generated":[239,308],"is":[241,257,285],"often":[242],"poor":[243],"due":[244],"grammar":[246],"logic":[248],"errors.":[249],"Third,":[250],"generating":[251,266],"entire":[253],"repository":[254,269],"once":[256],"best":[259],"strategy":[261,273],"smaller":[263],"repositories,":[264],"while":[265],"complex":[268],"module-by-module":[272],"works":[274],"better":[275],"compared":[276],"other":[278],"strategies.":[279],"Fourth,":[280],"detailed":[282],"very":[286],"important":[287],"tasks":[292],"through":[293],"conducting":[294],"ablation":[295],"studies":[296],"Lastly,":[300],"discuss":[302],"frequent":[304],"error":[305],"types":[306],"repositories":[309],"provide":[311],"optimizing":[314]},"counts_by_year":[],"updated_date":"2026-07-02T06:12:58.138171","created_date":"2026-07-01T00:00:00"}