{"id":"https://openalex.org/W4404345304","doi":"https://doi.org/10.48550/arxiv.2411.00646","title":"Phase Diagram of Vision Large Language Models Inference: A Perspective from Interaction across Image and Instruction","display_name":"Phase Diagram of Vision Large Language Models Inference: A Perspective from Interaction across Image and Instruction","publication_year":2024,"publication_date":"2024-11-01","ids":{"openalex":"https://openalex.org/W4404345304","doi":"https://doi.org/10.48550/arxiv.2411.00646"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2411.00646","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.00646","pdf_url":"https://arxiv.org/pdf/2411.00646","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2411.00646","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5114831771","display_name":"Houjing Wei","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei, Houjing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034981004","display_name":"Yuting Shi","orcid":"https://orcid.org/0000-0002-6733-0827"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shi, Yuting","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5028046901","display_name":"Naoya Inoue","orcid":"https://orcid.org/0000-0002-2961-6939"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Inoue, Naoya","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7827000021934509,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.7827000021934509,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6680999994277954,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.8360840678215027},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.733834981918335},{"id":"https://openalex.org/keywords/phase-diagram","display_name":"Phase diagram","score":0.5290398001670837},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5095891952514648},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.4934617578983307},{"id":"https://openalex.org/keywords/phase","display_name":"Phase (matter)","score":0.4730091392993927},{"id":"https://openalex.org/keywords/diagram","display_name":"Diagram","score":0.44613200426101685},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4148625135421753},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.3689945936203003},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3488582968711853},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.1869453489780426}],"concepts":[{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.8360840678215027},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.733834981918335},{"id":"https://openalex.org/C85906118","wikidata":"https://www.wikidata.org/wiki/Q186693","display_name":"Phase diagram","level":3,"score":0.5290398001670837},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5095891952514648},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.4934617578983307},{"id":"https://openalex.org/C44280652","wikidata":"https://www.wikidata.org/wiki/Q104837","display_name":"Phase (matter)","level":2,"score":0.4730091392993927},{"id":"https://openalex.org/C186399060","wikidata":"https://www.wikidata.org/wiki/Q959962","display_name":"Diagram","level":2,"score":0.44613200426101685},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4148625135421753},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3689945936203003},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3488582968711853},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.1869453489780426},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2411.00646","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.00646","pdf_url":"https://arxiv.org/pdf/2411.00646","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2411.00646","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.00646","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2411.00646","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.00646","pdf_url":"https://arxiv.org/pdf/2411.00646","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4404345304.pdf"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2030742537","https://openalex.org/W2055243143","https://openalex.org/W2075062329","https://openalex.org/W1978089009","https://openalex.org/W2030693616","https://openalex.org/W1998520214","https://openalex.org/W2332383162","https://openalex.org/W4205332007","https://openalex.org/W2514159064","https://openalex.org/W2022416154"],"abstract_inverted_index":{"Vision":[0],"Large":[1],"Language":[2],"Models":[3],"(VLLMs)":[4],"usually":[5],"take":[6],"input":[7],"as":[8],"a":[9,67,91,111,128],"concatenation":[10],"of":[11,32,37,59,71,76],"image":[12],"token":[13,17],"embeddings":[14,18],"and":[15,19,144],"text":[16],"conduct":[20],"causal":[21],"modeling.":[22],"However,":[23],"their":[24],"internal":[25],"behaviors":[26],"remain":[27],"underexplored,":[28],"raising":[29],"the":[30,52,55,74,150],"question":[31],"interaction":[33,43,107],"among":[34,54],"two":[35],"types":[36],"tokens.":[38],"To":[39],"investigate":[40],"such":[41],"multimodal":[42],"during":[44],"model":[45],"inference,":[46],"in":[47],"this":[48],"paper,":[49],"we":[50],"measure":[51],"contextualization":[53,86,102,122,140],"hidden":[56,145],"state":[57],"vectors":[58],"tokens":[60],"from":[61],"different":[62],"modalities.":[63,115,132],"Our":[64],"experiments":[65],"uncover":[66],"four-phase":[68],"inference":[69],"dynamics":[70],"VLLMs":[72],"against":[73],"depth":[75],"Transformer-based":[77],"LMs,":[78],"including":[79],"(I)":[80],"Alignment:":[81],"In":[82,98,119,136],"very":[83,137],"early":[84,99],"layers,":[85,100,121,139],"emerges":[87],"between":[88],"modalities,":[89],"suggesting":[90,110,127],"feature":[92],"space":[93],"alignment.":[94],"(II)":[95],"Intra-modal":[96],"Encoding:":[97,118],"intra-modal":[101],"is":[103,108,125,141],"enhanced":[104],"while":[105],"inter-modal":[106],"suppressed,":[109],"local":[112],"encoding":[113],"within":[114],"(III)":[116],"Inter-modal":[117],"later":[120],"across":[123,131],"modalities":[124],"enhanced,":[126],"deeper":[129],"fusion":[130],"(IV)":[133],"Output":[134],"Preparation:":[135],"late":[138],"reduced":[142],"globally,":[143],"states":[146],"are":[147],"aligned":[148],"towards":[149],"unembedding":[151],"space.":[152]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}