{"id":"https://openalex.org/W4392736863","doi":"https://doi.org/10.48550/arxiv.2403.05680","title":"How Well Do Multi-modal LLMs Interpret CT Scans? An Auto-Evaluation Framework for Analyses","display_name":"How Well Do Multi-modal LLMs Interpret CT Scans? An Auto-Evaluation Framework for Analyses","publication_year":2024,"publication_date":"2024-03-08","ids":{"openalex":"https://openalex.org/W4392736863","doi":"https://doi.org/10.48550/arxiv.2403.05680"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2403.05680","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.05680","pdf_url":"https://arxiv.org/pdf/2403.05680","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2403.05680","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101700453","display_name":"Qingqing Zhu","orcid":"https://orcid.org/0000-0001-7326-0098"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Qingqing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084982293","display_name":"Benjamin Hou","orcid":"https://orcid.org/0000-0003-3968-1707"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hou, Benjamin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5048201234","display_name":"Tejas Sudharshan Mathai","orcid":"https://orcid.org/0009-0005-8440-1702"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mathai, Tejas S.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075266423","display_name":"Pritam Mukherjee","orcid":"https://orcid.org/0000-0002-9975-9994"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mukherjee, Pritam","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100611571","display_name":"Qiao Jin","orcid":"https://orcid.org/0000-0002-1268-7239"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jin, Qiao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110999110","display_name":"Xiuying Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Xiuying","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5087031397","display_name":"Zhizheng Wang","orcid":"https://orcid.org/0000-0003-2584-136X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Zhizheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079582829","display_name":"Ruida Cheng","orcid":"https://orcid.org/0000-0003-4395-7331"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Ruida","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5016047550","display_name":"Ronald M. Summers","orcid":"https://orcid.org/0000-0001-8081-7376"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Summers, Ronald M.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5083081872","display_name":"Zhiyong Lu","orcid":"https://orcid.org/0000-0001-9998-916X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lu, Zhiyong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.953000009059906,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.953000009059906,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.410078763961792},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.390569806098938},{"id":"https://openalex.org/keywords/environmental-science","display_name":"Environmental science","score":0.3758494257926941},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.34028926491737366}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.410078763961792},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.390569806098938},{"id":"https://openalex.org/C39432304","wikidata":"https://www.wikidata.org/wiki/Q188847","display_name":"Environmental science","level":0,"score":0.3758494257926941},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.34028926491737366}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2403.05680","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.05680","pdf_url":"https://arxiv.org/pdf/2403.05680","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2403.05680","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2403.05680","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2403.05680","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.05680","pdf_url":"https://arxiv.org/pdf/2403.05680","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4392736863.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2058170566","https://openalex.org/W2755342338","https://openalex.org/W2772917594","https://openalex.org/W2775347418","https://openalex.org/W2166024367","https://openalex.org/W3116076068","https://openalex.org/W2229312674","https://openalex.org/W2951359407","https://openalex.org/W2079911747","https://openalex.org/W1969923398"],"abstract_inverted_index":{"Automatically":[0],"interpreting":[1],"CT":[2],"scans":[3],"can":[4],"ease":[5],"the":[6,17,45,160],"workload":[7],"of":[8,19,47,90,96],"radiologists.":[9],"However,":[10],"this":[11,32,169],"is":[12],"challenging":[13],"mainly":[14],"due":[15,156],"to":[16,30,120,122,127,157,186,193,200],"scarcity":[18],"adequate":[20],"datasets":[21],"and":[22,60,94,106,117,142,174,195],"reference":[23],"standards":[24],"for":[25,65,153,163],"evaluation.":[26],"This":[27,42],"study":[28],"aims":[29],"bridge":[31],"gap":[33],"by":[34,133],"introducing":[35],"a":[36,70,100,129],"novel":[37],"evaluation":[38],"framework,":[39],"named":[40],"``GPTRadScore''.":[41],"framework":[43],"assesses":[44],"capabilities":[46],"multi-modal":[48],"LLMs,":[49],"such":[50,113],"as":[51,114],"GPT-4":[52],"with":[53,81,103],"Vision":[54,145],"(GPT-4V),":[55],"Gemini":[56,143],"Pro":[57,144],"Vision,":[58],"LLaVA-Med,":[59],"RadFM,":[61],"in":[62,88,159,177],"generating":[63],"descriptions":[64,80],"prospectively-identified":[66],"findings.":[67],"By":[68],"employing":[69],"decomposition":[71],"technique":[72],"based":[73],"on":[74],"GPT-4,":[75],"GPTRadScore":[76],"compares":[77],"these":[78,165],"generated":[79],"gold-standard":[82],"report":[83],"sentences,":[84],"analyzing":[85],"their":[86,148],"accuracy":[87,179,182,190,197],"terms":[89],"body":[91,188],"part,":[92],"location,":[93],"type":[95,196],"finding.":[97],"Evaluations":[98],"demonstrated":[99],"high":[101],"correlation":[102],"clinician":[104],"assessments":[105],"highlighted":[107],"its":[108],"potential":[109],"over":[110],"traditional":[111],"metrics,":[112],"BLEU,":[115],"METEOR,":[116],"ROUGE.":[118],"Furthermore,":[119],"contribute":[121],"future":[123],"studies,":[124],"we":[125,137],"plan":[126],"release":[128],"benchmark":[130],"dataset":[131,161],"annotated":[132],"clinicians.":[134],"Using":[135],"GPTRadScore,":[136],"found":[138],"that":[139],"while":[140],"GPT-4V":[141],"fare":[146],"better,":[147],"performance":[149],"revealed":[150],"significant":[151,178],"areas":[152],"improvement,":[154],"primarily":[155],"limitations":[158],"used":[162],"training":[164],"models.":[166],"To":[167],"demonstrate":[168],"potential,":[170],"RadFM":[171],"was":[172],"fine-tuned":[173],"it":[175],"resulted":[176],"improvements:":[180],"location":[181],"rose":[183],"from":[184,191,198],"3.41\\%":[185],"12.8\\%,":[187],"part":[189],"29.12\\%":[192],"53\\%,":[194],"9.24\\%":[199],"30\\%,":[201],"thereby":[202],"validating":[203],"our":[204],"hypothesis.":[205]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2024-03-13T00:00:00"}