{"id":"https://openalex.org/W4416150759","doi":"https://doi.org/10.48550/arxiv.2502.20315","title":"LangProBe: a Language Programs Benchmark","display_name":"LangProBe: a Language Programs Benchmark","publication_year":2025,"publication_date":"2025-02-27","ids":{"openalex":"https://openalex.org/W4416150759","doi":"https://doi.org/10.48550/arxiv.2502.20315"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2502.20315","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.20315","pdf_url":"https://arxiv.org/pdf/2502.20315","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2502.20315","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081573804","display_name":"Shangyin Tan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tan, Shangyin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120308487","display_name":"Lakshya A Agrawal","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Agrawal, Lakshya A","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104180027","display_name":"Arnav Singhvi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Singhvi, Arnav","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013691122","display_name":"L. Lai","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lai, Liheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058109700","display_name":"Michael J. Ryan","orcid":"https://orcid.org/0000-0002-5679-6031"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ryan, Michael J","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004921249","display_name":"Dan Klein","orcid":"https://orcid.org/0000-0002-8881-1902"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Klein, Dan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015201177","display_name":"Omar Khattab","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khattab, Omar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101569699","display_name":"Koushik Sen","orcid":"https://orcid.org/0000-0002-4539-9188"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sen, Koushik","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5005554337","display_name":"Matei Zaharia","orcid":"https://orcid.org/0000-0002-7547-7204"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zaharia, Matei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.179299995303154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.179299995303154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.10350000113248825,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.053700000047683716,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.7724999785423279},{"id":"https://openalex.org/keywords/modular-design","display_name":"Modular design","score":0.5525000095367432},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.508400022983551},{"id":"https://openalex.org/keywords/pareto-principle","display_name":"Pareto principle","score":0.4207000136375427},{"id":"https://openalex.org/keywords/language-model","display_name":"Language model","score":0.41429999470710754},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.3921000063419342},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.3686999976634979},{"id":"https://openalex.org/keywords/source-code","display_name":"Source code","score":0.3499999940395355}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8065000176429749},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.7724999785423279},{"id":"https://openalex.org/C101468663","wikidata":"https://www.wikidata.org/wiki/Q1620158","display_name":"Modular design","level":2,"score":0.5525000095367432},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.508400022983551},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.45980000495910645},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4449999928474426},{"id":"https://openalex.org/C137635306","wikidata":"https://www.wikidata.org/wiki/Q182667","display_name":"Pareto principle","level":2,"score":0.4207000136375427},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.41429999470710754},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.3921000063419342},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.3686999976634979},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.3499999940395355},{"id":"https://openalex.org/C139571649","wikidata":"https://www.wikidata.org/wiki/Q1156793","display_name":"Program optimization","level":3,"score":0.3472999930381775},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34549999237060547},{"id":"https://openalex.org/C132964779","wikidata":"https://www.wikidata.org/wiki/Q2110223","display_name":"Raw data","level":2,"score":0.33399999141693115},{"id":"https://openalex.org/C2777617010","wikidata":"https://www.wikidata.org/wiki/Q18957","display_name":"Mainstream","level":2,"score":0.3337000012397766},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.3163999915122986},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.2937999963760376},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2897999882698059},{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.28040000796318054},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C2781039887","wikidata":"https://www.wikidata.org/wiki/Q1391724","display_name":"Factor (programming language)","level":2,"score":0.26820001006126404},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.25859999656677246},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.2578999996185303}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2502.20315","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.20315","pdf_url":"https://arxiv.org/pdf/2502.20315","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2502.20315","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2502.20315","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2502.20315","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2502.20315","pdf_url":"https://arxiv.org/pdf/2502.20315","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Composing":[0],"language":[1,6,50,97],"models":[2],"(LMs)":[3],"into":[4],"multi-step":[5],"programs":[7,98],"and":[8,46,60,77,83,91,135],"automatically":[9],"optimizing":[10],"their":[11,80],"modular":[12],"prompts":[13],"is":[14,123],"now":[15],"a":[16],"mainstream":[17],"paradigm":[18],"for":[19,42,49,126,138],"building":[20],"AI":[21],"systems,":[22],"but":[23,109],"the":[24,38,44,68,72,133],"tradeoffs":[25,88],"in":[26],"this":[27],"space":[28],"have":[29],"only":[30],"scarcely":[31],"been":[32],"studied":[33],"before.":[34],"We":[35,93,129],"introduce":[36],"LangProBe,":[37,65],"first":[39,69],"large-scale":[40],"benchmark":[41],"evaluating":[43],"architectures":[45,76],"optimization":[47],"strategies":[48],"programs,":[51],"with":[52,84],"over":[53,104],"2000":[54],"combinations":[55],"of":[56,62,74,89],"tasks,":[57],"architectures,":[58],"optimizers,":[59],"choices":[61],"LMs.":[63],"Using":[64],"we":[66],"are":[67],"to":[70,107,121],"study":[71],"impact":[73],"program":[75],"optimizers":[78],"(and":[79],"compositions":[81,120],"together":[82],"different":[85],"models)":[86],"on":[87],"quality":[90],"cost.":[92],"find":[94],"that":[95,112],"optimized":[96],"offer":[99],"strong":[100],"cost--quality":[101],"Pareto":[102],"improvement":[103],"raw":[105],"calls":[106],"models,":[108],"simultaneously":[110],"demonstrate":[111],"human":[113],"judgment":[114],"(or":[115],"empirical":[116],"decisions)":[117],"about":[118],"which":[119],"pursue":[122],"still":[124],"necessary":[125],"best":[127],"performance.":[128],"will":[130],"open":[131],"source":[132],"code":[134],"evaluation":[136],"data":[137],"LangProBe.":[139]},"counts_by_year":[],"updated_date":"2026-07-03T08:13:44.112507","created_date":"2025-10-10T00:00:00"}