{"id":"https://openalex.org/W4415254167","doi":"https://doi.org/10.1145/3797114","title":"Clotho: Measuring Task-Specific Pre-Generation Test Adequacy for LLM Inputs","display_name":"Clotho: Measuring Task-Specific Pre-Generation Test Adequacy for LLM Inputs","publication_year":2026,"publication_date":"2026-06-30","ids":{"openalex":"https://openalex.org/W4415254167","doi":"https://doi.org/10.1145/3797114"},"language":"en","primary_location":{"id":"doi:10.1145/3797114","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3797114","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2509.17314","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101316400","display_name":"Juyeon Yoon","orcid":null},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Juyeon Yoon","raw_affiliation_strings":["Korea Advanced Institute of Science and Technology, Daejeon, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0003-2706-1156","affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science and Technology, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5035122496","display_name":"Somin Kim","orcid":null},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Somin Kim","raw_affiliation_strings":["Korea Advanced Institute of Science and Technology, Daejeon, Republic of Korea"],"raw_orcid":"https://orcid.org/0009-0007-8917-7302","affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science and Technology, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5063787358","display_name":"Robert Feldt","orcid":"https://orcid.org/0000-0002-5179-4205"},"institutions":[{"id":"https://openalex.org/I66862912","display_name":"Chalmers University of Technology","ror":"https://ror.org/040wg7k59","country_code":"SE","type":"education","lineage":["https://openalex.org/I66862912"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Robert Feldt","raw_affiliation_strings":["Chalmers University of Technology, Computer Science and Engineering, Gothenburg, Sweden"],"raw_orcid":"https://orcid.org/0000-0002-5179-4205","affiliations":[{"raw_affiliation_string":"Chalmers University of Technology, Computer Science and Engineering, Gothenburg, Sweden","institution_ids":["https://openalex.org/I66862912"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070401254","display_name":"Shin Yoo","orcid":"https://orcid.org/0000-0002-0836-6993"},"institutions":[{"id":"https://openalex.org/I157485424","display_name":"Korea Advanced Institute of Science and Technology","ror":"https://ror.org/05apxxy63","country_code":"KR","type":"education","lineage":["https://openalex.org/I157485424"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Shin Yoo","raw_affiliation_strings":["Korea Advanced Institute of Science and Technology, Daejeon, Republic of Korea"],"raw_orcid":"https://orcid.org/0000-0002-0836-6993","affiliations":[{"raw_affiliation_string":"Korea Advanced Institute of Science and Technology, Daejeon, Republic of Korea","institution_ids":["https://openalex.org/I157485424"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":13.8137,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.97524752,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"3","issue":"FSE","first_page":"1910","last_page":"1932"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9413999915122986,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9413999915122986,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10743","display_name":"Software Testing and Debugging Techniques","score":0.9115999937057495,"subfield":{"id":"https://openalex.org/subfields/1712","display_name":"Software"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/complement","display_name":"Complement (music)","score":0.6841999888420105},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.595300018787384},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5849000215530396},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.5521000027656555},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5069000124931335},{"id":"https://openalex.org/keywords/measure","display_name":"Measure (data warehouse)","score":0.47940000891685486},{"id":"https://openalex.org/keywords/forcing","display_name":"Forcing (mathematics)","score":0.4781999886035919},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.42289999127388},{"id":"https://openalex.org/keywords/variation","display_name":"Variation (astronomy)","score":0.3312999904155731}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6902999877929688},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.6841999888420105},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.595300018787384},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5849000215530396},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.5521000027656555},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5069000124931335},{"id":"https://openalex.org/C2780009758","wikidata":"https://www.wikidata.org/wiki/Q6804172","display_name":"Measure (data warehouse)","level":2,"score":0.47940000891685486},{"id":"https://openalex.org/C197115733","wikidata":"https://www.wikidata.org/wiki/Q1003136","display_name":"Forcing (mathematics)","level":2,"score":0.4781999886035919},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4390000104904175},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.42289999127388},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.41990000009536743},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.38609999418258667},{"id":"https://openalex.org/C2778334786","wikidata":"https://www.wikidata.org/wiki/Q1586270","display_name":"Variation (astronomy)","level":2,"score":0.3312999904155731},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.3075999915599823},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.2939999997615814},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.28360000252723694},{"id":"https://openalex.org/C196083921","wikidata":"https://www.wikidata.org/wiki/Q7915758","display_name":"Variance (accounting)","level":2,"score":0.2799000144004822},{"id":"https://openalex.org/C61326573","wikidata":"https://www.wikidata.org/wiki/Q1496376","display_name":"Gaussian process","level":3,"score":0.2752000093460083},{"id":"https://openalex.org/C96608239","wikidata":"https://www.wikidata.org/wiki/Q1199823","display_name":"Statistical power","level":2,"score":0.2653999924659729},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.2621000111103058},{"id":"https://openalex.org/C2778348673","wikidata":"https://www.wikidata.org/wiki/Q739302","display_name":"Production (economics)","level":2,"score":0.2606000006198883},{"id":"https://openalex.org/C168167062","wikidata":"https://www.wikidata.org/wiki/Q1117970","display_name":"Component (thermodynamics)","level":2,"score":0.26010000705718994},{"id":"https://openalex.org/C61224824","wikidata":"https://www.wikidata.org/wiki/Q2260434","display_name":"Mixture model","level":2,"score":0.2587999999523163},{"id":"https://openalex.org/C87007009","wikidata":"https://www.wikidata.org/wiki/Q210832","display_name":"Statistical hypothesis testing","level":2,"score":0.257099986076355},{"id":"https://openalex.org/C70437156","wikidata":"https://www.wikidata.org/wiki/Q7228652","display_name":"Pooling","level":2,"score":0.251800000667572},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/3797114","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3797114","pdf_url":null,"source":{"id":"https://openalex.org/S4404663975","display_name":"Proceedings of the ACM on software engineering.","issn_l":"2994-970X","issn":["2994-970X"],"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the ACM on Software Engineering","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2509.17314","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2509.17314","pdf_url":"https://arxiv.org/pdf/2509.17314","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2509.17314","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2509.17314","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2509.17314","is_oa":true,"landing_page_url":"https://arxiv.org/abs/2509.17314","pdf_url":"https://arxiv.org/pdf/2509.17314","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1859597740","display_name":null,"funder_award_id":"RS-2021-NR060080","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G8743049482","display_name":null,"funder_award_id":"CAEP-26-Feldt","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"},{"id":"https://openalex.org/G967999616","display_name":null,"funder_award_id":"RS-2023-00208998","funder_id":"https://openalex.org/F4320322120","funder_display_name":"National Research Foundation of Korea"}],"funders":[{"id":"https://openalex.org/F4320322120","display_name":"National Research Foundation of Korea","ror":"https://ror.org/013aysd81"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415254167.pdf","grobid_xml":"https://content.openalex.org/works/W4415254167.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Software":[0],"increasingly":[1],"relies":[2],"on":[3,24,38,48,134,177],"the":[4,76,79,126,138,213,235,238,249],"emergent":[5],"capabilities":[6],"of":[7,78,109,148,169,181,204,237,252,260],"Large":[8],"Language":[9],"Models":[10],"(LLMs),":[11],"from":[12,101,226,255],"natural":[13],"language":[14],"understanding":[15],"to":[16,63,123,197,231,257,263],"program":[17],"analysis":[18],"and":[19,29,51,158,206],"generation.":[20],"Yet":[21],"testing":[22],"them":[23],"specific":[25,114],"tasks":[26,157],"remains":[27],"difficult":[28],"costly:":[30],"many":[31],"prompts":[32],"lack":[33],"ground":[34],"truths,":[35],"forcing":[36],"reliance":[37],"human":[39,131],"judgments,":[40],"while":[41],"existing":[42],"test":[43,68,93,242],"adequacy":[44,94,209,223],"measures":[45,210],"typically":[46],"rely":[47],"output":[49],"uncertainty":[50,199],"thus":[52],"are":[53,176],"only":[54,179],"available":[55],"after":[56,171],"full":[57],"inference.":[58],"A":[59],"key":[60],"challenge":[61],"is":[62,70],"assess":[64],"how":[65],"useful":[66],"a":[67,72,90,106,113,118,167],"input":[69,98],"in":[71],"way":[73],"that":[74,96,175,212,222],"reflects":[75],"demands":[77],"task,":[80,115],"ideally":[81],"before":[82],"even":[83],"generating":[84,187],"any":[85,188],"output.":[86],"We":[87],"introduce":[88],"Clotho,":[89],"task-specific,":[91],"pre-generation":[92],"measure":[95],"estimates":[97],"difficulty":[99],"directly":[100],"LLM":[102,193],"hidden":[103],"states.":[104],"Given":[105],"large":[107],"pool":[108],"unlabelled":[110],"inputs":[111,144,243,254],"for":[112,130,244],"Clotho":[116,162,205,247],"uses":[117],"Gaussian":[119],"Mixture":[120],"Model":[121],"(GMM)":[122],"adaptively":[124],"sample":[125],"most":[127],"informative":[128],"cases":[129],"labelling.":[132],"Based":[133],"this":[135],"reference":[136,173],"set":[137],"GMM":[139],"can":[140,163],"then":[141],"rank":[142],"unseen":[143],"by":[145],"their":[146],"likelihood":[147],"failure.":[149],"In":[150],"our":[151],"empirical":[152],"evaluation":[153],"across":[154],"eight":[155],"benchmark":[156],"three":[159],"open-weight":[160,227],"LLMs,":[161],"predict":[164],"failures":[165],"with":[166],"ROC-AUC":[168],"0.716,":[170],"labelling":[172],"sets":[174],"average":[178,250],"5.4%":[180],"inputs.":[182],"It":[183],"does":[184],"so":[185],"without":[186],"outputs,":[189],"thereby":[190],"significantly":[191],"reducing":[192],"execution":[194],"costs":[195],"compared":[196,262],"output-based":[198],"or":[200],"confidence":[201],"measures.":[202],"Comparison":[203],"these":[207],"post-generation":[208],"shows":[211],"two":[214],"approaches":[215],"complement":[216],"each":[217],"other.":[218],"Crucially,":[219],"we":[220],"show":[221],"scores":[224],"learnt":[225],"LLMs":[228],"transfer":[229],"effectively":[230],"proprietary":[232,245],"models,":[233,246],"extending":[234],"applicability":[236],"approach.":[239],"When":[240],"prioritising":[241],"increases":[248],"number":[251],"failing":[253],"18.7":[256],"42.5":[258],"out":[259],"100,":[261],"random":[264],"prioritisation.":[265]},"counts_by_year":[{"year":2026,"cited_by_count":2}],"updated_date":"2026-08-25T07:29:55.448023","created_date":"2025-10-16T00:00:00"}