{"id":"https://openalex.org/W7077935078","doi":"https://doi.org/10.48550/arxiv.2508.18756","title":"UltraMemV2: Memory Networks Scaling to 120B Parameters with Superior Long-Context Learning","display_name":"UltraMemV2: Memory Networks Scaling to 120B Parameters with Superior Long-Context Learning","publication_year":2025,"publication_date":"2025-08-26","ids":{"openalex":"https://openalex.org/W7077935078","doi":"https://doi.org/10.48550/arxiv.2508.18756"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2508.18756","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.18756","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2508.18756","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Huang, Zihao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Bao, Yu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Bao, Yu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Min, Qiyang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Min, Qiyang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chen, Siyan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Siyan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Guo, Ran","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guo, Ran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Huang, Hongzhi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Hongzhi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhu, Defa","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Defa","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zeng, Yutao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zeng, Yutao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wu, Banggu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Banggu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zhou, Xun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Xun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Qiao, Siyuan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiao, Siyuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.7149999737739563,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.7149999737739563,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13067","display_name":"Geological Modeling and Analysis","score":0.0272000003606081,"subfield":{"id":"https://openalex.org/subfields/1906","display_name":"Geochemistry and Petrology"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14311","display_name":"Electrical and Electromagnetic Research","score":0.015300000086426735,"subfield":{"id":"https://openalex.org/subfields/3107","display_name":"Atomic and Molecular Physics, and Optics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computation","display_name":"Computation","score":0.6715999841690063},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4925999939441681},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.43149998784065247},{"id":"https://openalex.org/keywords/scaling","display_name":"Scaling","score":0.4226999878883362},{"id":"https://openalex.org/keywords/memory-architecture","display_name":"Memory architecture","score":0.32910001277923584},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.32829999923706055},{"id":"https://openalex.org/keywords/random-access-memory","display_name":"Random access memory","score":0.3077999949455261}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.692300021648407},{"id":"https://openalex.org/C45374587","wikidata":"https://www.wikidata.org/wiki/Q12525525","display_name":"Computation","level":2,"score":0.6715999841690063},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4925999939441681},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.43149998784065247},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.4226999878883362},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3375000059604645},{"id":"https://openalex.org/C2779602883","wikidata":"https://www.wikidata.org/wiki/Q15544750","display_name":"Memory architecture","level":2,"score":0.32910001277923584},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.32829999923706055},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.31610000133514404},{"id":"https://openalex.org/C2994168587","wikidata":"https://www.wikidata.org/wiki/Q5295","display_name":"Random access memory","level":2,"score":0.3077999949455261},{"id":"https://openalex.org/C12186640","wikidata":"https://www.wikidata.org/wiki/Q6815743","display_name":"Memory model","level":3,"score":0.3010999858379364},{"id":"https://openalex.org/C123657996","wikidata":"https://www.wikidata.org/wiki/Q12271","display_name":"Architecture","level":2,"score":0.29409998655319214},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.29100000858306885},{"id":"https://openalex.org/C2778915421","wikidata":"https://www.wikidata.org/wiki/Q3643177","display_name":"Performance improvement","level":2,"score":0.28529998660087585},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.2838999927043915},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2766000032424927},{"id":"https://openalex.org/C2777115002","wikidata":"https://www.wikidata.org/wiki/Q7168246","display_name":"Performance prediction","level":2,"score":0.27160000801086426},{"id":"https://openalex.org/C68043766","wikidata":"https://www.wikidata.org/wiki/Q267416","display_name":"Static random-access memory","level":2,"score":0.25540000200271606}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2508.18756","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.18756","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2508.18756","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.18756","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"While":[0],"Mixture":[1],"of":[2,13,45,52,137],"Experts":[3],"(MoE)":[4],"models":[5,116,161],"achieve":[6],"remarkable":[7],"efficiency":[8],"by":[9],"activating":[10],"only":[11,41],"subsets":[12],"parameters,":[14,170],"they":[15],"suffer":[16],"from":[17,92,167],"high":[18],"memory":[19,33,75,125],"access":[20],"costs":[21],"during":[22],"inference.":[23],"Memory-layer":[24],"architectures":[25,190],"offer":[26],"an":[27],"appealing":[28],"alternative":[29,201],"with":[30,84,113,135,160,194],"very":[31],"few":[32],"access,":[34],"but":[35,122],"previous":[36],"attempts":[37],"like":[38],"UltraMem":[39],"have":[40],"matched":[42],"the":[43],"performance":[44,66,111,131,180,192],"2-expert":[46],"MoE":[47,115,196],"models,":[48,197],"falling":[49],"significantly":[50,123],"short":[51],"state-of-the-art":[53,195],"8-expert":[54,114],"configurations.":[55],"We":[56,154],"present":[57],"UltraMemV2,":[58],"a":[59,199],"redesigned":[60],"memory-layer":[61,189],"architecture":[62],"that":[63,108,173],"closes":[64],"this":[65],"gap.":[67],"Our":[68,186],"approach":[69,157],"introduces":[70],"five":[71],"key":[72],"improvements:":[73],"integrating":[74],"layers":[76],"into":[77],"every":[78],"transformer":[79],"block,":[80],"simplifying":[81],"value":[82,90],"expansion":[83],"single":[85],"linear":[86],"projections,":[87],"adopting":[88],"FFN-based":[89],"processing":[91],"PEER,":[93],"implementing":[94],"principled":[95],"parameter":[96,184],"initialization,":[97],"and":[98,120,148,171],"rebalancing":[99],"memory-to-FFN":[100],"computation":[101,119],"ratios.":[102],"Through":[103],"extensive":[104],"evaluation,":[105],"we":[106],"demonstrate":[107],"UltraMemV2":[109,128],"achieves":[110],"parity":[112,193],"under":[117],"same":[118],"parameters":[121,166],"low":[124],"access.":[126],"Notably,":[127],"shows":[129],"superior":[130],"on":[132,140,145,151,179],"memory-intensive":[133],"tasks,":[134],"improvements":[136],"+1.6":[138],"points":[139,144,150],"long-context":[141],"memorization,":[142,147],"+6.2":[143],"multi-round":[146],"+7.9":[149],"in-context":[152],"learning.":[153],"validate":[155],"our":[156],"at":[158],"scale":[159],"up":[162],"to":[163,191],"2.5B":[164],"activated":[165],"120B":[168],"total":[169,182],"establish":[172],"activation":[174],"density":[175],"has":[176],"greater":[177],"impact":[178],"than":[181],"sparse":[183,204],"count.":[185],"work":[187],"brings":[188],"presenting":[198],"compelling":[200],"for":[202],"efficient":[203],"computation.":[205]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}