{"id":"https://openalex.org/W4408642247","doi":"https://doi.org/10.1002/spe.3421","title":"On the Performance and Memory Footprint of Distributed Training: An Empirical Study on Transformers","display_name":"On the Performance and Memory Footprint of Distributed Training: An Empirical Study on Transformers","publication_year":2025,"publication_date":"2025-03-18","ids":{"openalex":"https://openalex.org/W4408642247","doi":"https://doi.org/10.1002/spe.3421"},"language":"en","primary_location":{"id":"doi:10.1002/spe.3421","is_oa":true,"landing_page_url":"https://doi.org/10.1002/spe.3421","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/spe.3421","source":{"id":"https://openalex.org/S122199241","display_name":"Software Practice and Experience","issn_l":"0038-0644","issn":["0038-0644","1097-024X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software: Practice and Experience","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"bronze","oa_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/spe.3421","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102553721","display_name":"Zhengxian Lu","orcid":"https://orcid.org/0000-0003-0869-9728"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]},{"id":"https://openalex.org/I4210126530","display_name":"Data Assurance and Communication Security","ror":"https://ror.org/02z2gfm30","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210126530","https://openalex.org/I4210156404"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhengxian Lu","raw_affiliation_strings":["College of Computer Science Nankai University  Tianjin China","Tianjin Key Laboratory of Network and Data Security Technology  Tianjin China"],"raw_orcid":"https://orcid.org/0000-0003-0869-9728","affiliations":[{"raw_affiliation_string":"College of Computer Science Nankai University  Tianjin China","institution_ids":["https://openalex.org/I205237279"]},{"raw_affiliation_string":"Tianjin Key Laboratory of Network and Data Security Technology  Tianjin China","institution_ids":["https://openalex.org/I4210126530"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104211041","display_name":"Fangyu Wang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fangyu Wang","raw_affiliation_strings":["Zhejiang Lab  Zhejiang China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang Lab  Zhejiang China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100782604","display_name":"Zhiwei Xu","orcid":"https://orcid.org/0000-0002-8249-9997"},"institutions":[{"id":"https://openalex.org/I4210164952","display_name":"Tianjin haihe hospital","ror":"https://ror.org/05m762q77","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210164952"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhiwei Xu","raw_affiliation_strings":["Haihe Lab of ITAI  Tianjin China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Haihe Lab of ITAI  Tianjin China","institution_ids":["https://openalex.org/I4210164952"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066710918","display_name":"Fei Yang","orcid":"https://orcid.org/0000-0001-6751-1954"},"institutions":[{"id":"https://openalex.org/I4210123185","display_name":"Zhejiang Lab","ror":"https://ror.org/02m2h7991","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210123185"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Fei Yang","raw_affiliation_strings":["Zhejiang Lab  Zhejiang China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang Lab  Zhejiang China","institution_ids":["https://openalex.org/I4210123185"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100455376","display_name":"Tao Li","orcid":"https://orcid.org/0000-0003-1697-8022"},"institutions":[{"id":"https://openalex.org/I205237279","display_name":"Nankai University","ror":"https://ror.org/01y1kjr75","country_code":"CN","type":"education","lineage":["https://openalex.org/I205237279"]},{"id":"https://openalex.org/I4210164952","display_name":"Tianjin haihe hospital","ror":"https://ror.org/05m762q77","country_code":"CN","type":"healthcare","lineage":["https://openalex.org/I4210164952"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Tao Li","raw_affiliation_strings":["College of Computer Science Nankai University  Tianjin China","Haihe Lab of ITAI  Tianjin China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Computer Science Nankai University  Tianjin China","institution_ids":["https://openalex.org/I205237279"]},{"raw_affiliation_string":"Haihe Lab of ITAI  Tianjin China","institution_ids":["https://openalex.org/I4210164952"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5066710918","https://openalex.org/A5100455376"],"corresponding_institution_ids":["https://openalex.org/I205237279","https://openalex.org/I4210123185","https://openalex.org/I4210164952"],"apc_list":{"value":4650,"currency":"USD","value_usd":4650},"apc_paid":null,"fwci":1.7849,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":{"value":0.84402071,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"55","issue":"7","first_page":"1266","last_page":"1284"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9948999881744385,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.989799976348877,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10764","display_name":"Privacy-Preserving Technologies in Data","score":0.9890000224113464,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/memory-footprint","display_name":"Memory footprint","score":0.6781389713287354},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.5757634043693542},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5596814155578613},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.5448818206787109},{"id":"https://openalex.org/keywords/footprint","display_name":"Footprint","score":0.4663483500480652},{"id":"https://openalex.org/keywords/reliability-engineering","display_name":"Reliability engineering","score":0.3254474997520447},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.2371007204055786},{"id":"https://openalex.org/keywords/operating-system","display_name":"Operating system","score":0.2292023003101349},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.20708265900611877},{"id":"https://openalex.org/keywords/geology","display_name":"Geology","score":0.14522919058799744},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.09441393613815308},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.0559631884098053}],"concepts":[{"id":"https://openalex.org/C74912251","wikidata":"https://www.wikidata.org/wiki/Q6815727","display_name":"Memory footprint","level":2,"score":0.6781389713287354},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.5757634043693542},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5596814155578613},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.5448818206787109},{"id":"https://openalex.org/C132943942","wikidata":"https://www.wikidata.org/wiki/Q2562511","display_name":"Footprint","level":2,"score":0.4663483500480652},{"id":"https://openalex.org/C200601418","wikidata":"https://www.wikidata.org/wiki/Q2193887","display_name":"Reliability engineering","level":1,"score":0.3254474997520447},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.2371007204055786},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.2292023003101349},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.20708265900611877},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.14522919058799744},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.09441393613815308},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.0559631884098053},{"id":"https://openalex.org/C153294291","wikidata":"https://www.wikidata.org/wiki/Q25261","display_name":"Meteorology","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1002/spe.3421","is_oa":true,"landing_page_url":"https://doi.org/10.1002/spe.3421","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/spe.3421","source":{"id":"https://openalex.org/S122199241","display_name":"Software Practice and Experience","issn_l":"0038-0644","issn":["0038-0644","1097-024X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software: Practice and Experience","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1002/spe.3421","is_oa":true,"landing_page_url":"https://doi.org/10.1002/spe.3421","pdf_url":"https://onlinelibrary.wiley.com/doi/pdfdirect/10.1002/spe.3421","source":{"id":"https://openalex.org/S122199241","display_name":"Software Practice and Experience","issn_l":"0038-0644","issn":["0038-0644","1097-024X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320595","host_organization_name":"Wiley","host_organization_lineage":["https://openalex.org/P4310320595"],"host_organization_lineage_names":["Wiley"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Software: Practice and Experience","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4342606088","display_name":null,"funder_award_id":"23JCQNJC00010","funder_id":"https://openalex.org/F4320323993","funder_display_name":"Natural Science Foundation of Tianjin City"},{"id":"https://openalex.org/G6024644208","display_name":null,"funder_award_id":"62272248","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8000429783","display_name":null,"funder_award_id":"U22A6001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320323993","display_name":"Natural Science Foundation of Tianjin City","ror":null}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4408642247.pdf","grobid_xml":"https://content.openalex.org/works/W4408642247.grobid-xml"},"referenced_works_count":63,"referenced_works":["https://openalex.org/W639708223","https://openalex.org/W1522301498","https://openalex.org/W2002555321","https://openalex.org/W2060393849","https://openalex.org/W2108598243","https://openalex.org/W2168231600","https://openalex.org/W2194775991","https://openalex.org/W2515080096","https://openalex.org/W2613718673","https://openalex.org/W2618530766","https://openalex.org/W2734556049","https://openalex.org/W2734941459","https://openalex.org/W2736230459","https://openalex.org/W2773756426","https://openalex.org/W2799054028","https://openalex.org/W2883830791","https://openalex.org/W2896457183","https://openalex.org/W2901299405","https://openalex.org/W2923014074","https://openalex.org/W2963339410","https://openalex.org/W2963351145","https://openalex.org/W2963674387","https://openalex.org/W2964330541","https://openalex.org/W2966048283","https://openalex.org/W2969388332","https://openalex.org/W2970971581","https://openalex.org/W2973727699","https://openalex.org/W2979245724","https://openalex.org/W2991040477","https://openalex.org/W3008591352","https://openalex.org/W3012125688","https://openalex.org/W3042766772","https://openalex.org/W3081168214","https://openalex.org/W3086105743","https://openalex.org/W3093818819","https://openalex.org/W3119866685","https://openalex.org/W3129831491","https://openalex.org/W3157919170","https://openalex.org/W3168867926","https://openalex.org/W3175786688","https://openalex.org/W3204998121","https://openalex.org/W3210871626","https://openalex.org/W4205734798","https://openalex.org/W4205871220","https://openalex.org/W4225146526","https://openalex.org/W4226064176","https://openalex.org/W4236965008","https://openalex.org/W4239521792","https://openalex.org/W4241618768","https://openalex.org/W4244652618","https://openalex.org/W4287391717","https://openalex.org/W4295312788","https://openalex.org/W4311080316","https://openalex.org/W4322718191","https://openalex.org/W4385585365","https://openalex.org/W4385963839","https://openalex.org/W4386071687","https://openalex.org/W4386542773","https://openalex.org/W4392222465","https://openalex.org/W6684859321","https://openalex.org/W6756379755","https://openalex.org/W6769062451","https://openalex.org/W6796581206"],"related_works":["https://openalex.org/W230091440","https://openalex.org/W2233261550","https://openalex.org/W2810751659","https://openalex.org/W258997015","https://openalex.org/W2997094352","https://openalex.org/W3216976533","https://openalex.org/W4404133543","https://openalex.org/W4408151435","https://openalex.org/W3137434606","https://openalex.org/W4372263373"],"abstract_inverted_index":{"ABSTRACT":[0],"Background:":[1],"Transformer":[2],"models":[3],"have":[4],"emerged":[5],"as":[6,215],"potent":[7],"solutions":[8],"to":[9,53,69,84,94,128,224,243],"a":[10,100,135,153],"wide":[11],"array":[12],"of":[13,18,103,112,132,138,156,174,197,236],"multidisciplinary":[14],"challenges.":[15],"The":[16],"deployment":[17],"transformer":[19,70,113,204],"architectures":[20],"is":[21],"significantly":[22],"hindered":[23],"by":[24,98,182],"their":[25],"extensive":[26],"computational":[27,86],"and":[28,57,81,87,119,143,159,240],"memory":[29,88,221],"requirements,":[30],"necessitating":[31],"reliance":[32],"on":[33,77,147,210],"advanced":[34],"efficient":[35],"distributed":[36,50,79,110,141,165],"training":[37,111,166],"methodologies.":[38],"Motivation:":[39],"Prior":[40],"research":[41],"has":[42],"delved":[43],"into":[44],"the":[45,72,75,82,104,109,175,183,188,216,234],"performance":[46,105],"bottlenecks":[47,56,106],"associated":[48],"with":[49],"training,":[51],"aiming":[52],"unravel":[54],"these":[55,129],"suggest":[58,194],"optimization":[59],"directions.":[60],"However,":[61],"such":[62,214],"analyses":[63],"often":[64],"overlook":[65],"three":[66],"aspects":[67,131],"unique":[68,130],"models:":[71],"specialized":[73],"architecture,":[74],"dependency":[76],"various":[78,164],"strategies,":[80,142],"requirement":[83],"balance":[85],"overhead.":[89],"Method:":[90],"This":[91],"paper":[92],"aims":[93],"bridge":[95],"this":[96,148],"gap":[97],"offering":[99],"comprehensive":[101],"examination":[102],"inherent":[107],"in":[108,169],"models,":[114],"leveraging":[115],"both":[116],"theoretical":[117,157],"analysis":[118,155],"empirical":[120],"investigation.":[121],"We":[122],"propose":[123],"an":[124,195],"analytical":[125,149,184,189],"framework":[126],"tailored":[127],"transformers,":[133],"facilitating":[134],"holistic":[136],"evaluation":[137],"model":[139,226],"architectures,":[140],"resource":[144],"consumption.":[145],"Based":[146],"framework,":[150],"we":[151,207,232],"conduct":[152],"comparative":[154],"performances":[158],"further":[160,244],"systematically":[161],"explore":[162],"how":[163],"strategies":[167],"fare":[168],"real\u2010world":[170],"scenarios.":[171],"Results:":[172],"Most":[173],"experimental":[176],"results":[177],"can":[178],"be":[179],"well":[180],"explained":[181],"outcomes":[185],"derived":[186],"from":[187],"framework.":[190],"Notably,":[191],"our":[192],"findings":[193],"advantage":[196],"pipeline":[198,229],"parallelism":[199,202],"over":[200],"data":[201],"for":[203,218],"models.":[205],"Moreover,":[206],"shed":[208],"light":[209],"some":[211],"unexpected":[212],"outcomes,":[213],"potential":[217],"increased":[219],"total":[220],"overhead":[222],"due":[223],"suboptimal":[225],"partitioning":[227],"within":[228],"parallelism.":[230],"Additionally,":[231],"underscore":[233],"significance":[235],"communication":[237],"block":[238],"size":[239],"waiting":[241],"time":[242],"enhance":[245],"performance.":[246]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-08-21T09:56:20.448147","created_date":"2025-10-10T00:00:00"}