{"id":"https://openalex.org/W4391248519","doi":"https://doi.org/10.1109/lsp.2024.3358978","title":"Music Conditioned Generation for Human-Centric Video","display_name":"Music Conditioned Generation for Human-Centric Video","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4391248519","doi":"https://doi.org/10.1109/lsp.2024.3358978"},"language":"en","primary_location":{"id":"doi:10.1109/lsp.2024.3358978","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lsp.2024.3358978","pdf_url":null,"source":{"id":"https://openalex.org/S120629676","display_name":"IEEE Signal Processing Letters","issn_l":"1070-9908","issn":["1070-9908","1558-2361"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Signal Processing Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060664372","display_name":"Zimeng Zhao","orcid":"https://orcid.org/0000-0001-6570-0620"},"institutions":[{"id":"https://openalex.org/I1327237609","display_name":"Ministry of Education of the People's Republic of China","ror":"https://ror.org/01mv9t934","country_code":"CN","type":"government","lineage":["https://openalex.org/I1327237609","https://openalex.org/I4210127390"]},{"id":"https://openalex.org/I76569877","display_name":"Southeast University","ror":"https://ror.org/04ct4d772","country_code":"CN","type":"education","lineage":["https://openalex.org/I76569877"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zimeng Zhao","raw_affiliation_strings":["School of Automation, Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Southeast University, Nanjing, China","Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Nanjing, China","School of Automation, Southeast University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0001-6570-0620","affiliations":[{"raw_affiliation_string":"School of Automation, Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877"]},{"raw_affiliation_string":"Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Nanjing, China","institution_ids":["https://openalex.org/I1327237609"]},{"raw_affiliation_string":"School of Automation, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040805760","display_name":"Binghui Zuo","orcid":"https://orcid.org/0000-0002-4975-8205"},"institutions":[{"id":"https://openalex.org/I1327237609","display_name":"Ministry of Education of the People's Republic of China","ror":"https://ror.org/01mv9t934","country_code":"CN","type":"government","lineage":["https://openalex.org/I1327237609","https://openalex.org/I4210127390"]},{"id":"https://openalex.org/I76569877","display_name":"Southeast University","ror":"https://ror.org/04ct4d772","country_code":"CN","type":"education","lineage":["https://openalex.org/I76569877"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Binghui Zuo","raw_affiliation_strings":["School of Automation, Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Southeast University, Nanjing, China","Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Nanjing, China","School of Automation, Southeast University, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-4975-8205","affiliations":[{"raw_affiliation_string":"School of Automation, Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877"]},{"raw_affiliation_string":"Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Nanjing, China","institution_ids":["https://openalex.org/I1327237609"]},{"raw_affiliation_string":"School of Automation, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100758518","display_name":"Yangang Wang","orcid":"https://orcid.org/0000-0002-1325-9252"},"institutions":[{"id":"https://openalex.org/I1327237609","display_name":"Ministry of Education of the People's Republic of China","ror":"https://ror.org/01mv9t934","country_code":"CN","type":"government","lineage":["https://openalex.org/I1327237609","https://openalex.org/I4210127390"]},{"id":"https://openalex.org/I76569877","display_name":"Southeast University","ror":"https://ror.org/04ct4d772","country_code":"CN","type":"education","lineage":["https://openalex.org/I76569877"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yangang Wang","raw_affiliation_strings":["School of Automation, Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Southeast University, Nanjing, China","School of Automation, Southeast University, Nanjing, China","Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-1325-9252","affiliations":[{"raw_affiliation_string":"School of Automation, Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877"]},{"raw_affiliation_string":"School of Automation, Southeast University, Nanjing, China","institution_ids":["https://openalex.org/I76569877"]},{"raw_affiliation_string":"Key Laboratory of Measurement and Control of Complex Systems of Engineering, Ministry of Education, Nanjing, China","institution_ids":["https://openalex.org/I1327237609"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.2913,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.48660086,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"31","issue":null,"first_page":"506","last_page":"510"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9569000005722046,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12290","display_name":"Human Motion and Animation","score":0.9569000005722046,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11309","display_name":"Music and Audio Processing","score":0.9503999948501587,"subfield":{"id":"https://openalex.org/subfields/1711","display_name":"Signal Processing"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10812","display_name":"Human Pose and Action Recognition","score":0.9488000273704529,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6532502770423889},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.3374348282814026},{"id":"https://openalex.org/keywords/speech-recognition","display_name":"Speech recognition","score":0.33130693435668945}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6532502770423889},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.3374348282814026},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.33130693435668945}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lsp.2024.3358978","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lsp.2024.3358978","pdf_url":null,"source":{"id":"https://openalex.org/S120629676","display_name":"IEEE Signal Processing Letters","issn_l":"1070-9908","issn":["1070-9908","1558-2361"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Signal Processing Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G6981116056","display_name":null,"funder_award_id":"BK20220127","funder_id":"https://openalex.org/F4320322769","funder_display_name":"Natural Science Foundation of Jiangsu Province"},{"id":"https://openalex.org/G8375517783","display_name":null,"funder_award_id":"62076061","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322769","display_name":"Natural Science Foundation of Jiangsu Province","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":32,"referenced_works":["https://openalex.org/W1901129140","https://openalex.org/W2064675550","https://openalex.org/W2963470893","https://openalex.org/W2963524571","https://openalex.org/W2963876278","https://openalex.org/W3096831136","https://openalex.org/W3163720395","https://openalex.org/W3193427204","https://openalex.org/W3196616669","https://openalex.org/W3204221554","https://openalex.org/W3215495615","https://openalex.org/W4214722801","https://openalex.org/W4307323391","https://openalex.org/W4309044858","https://openalex.org/W4312445951","https://openalex.org/W4312822036","https://openalex.org/W4312933868","https://openalex.org/W4319300158","https://openalex.org/W4372260310","https://openalex.org/W4377010269","https://openalex.org/W4384304163","https://openalex.org/W4385489997","https://openalex.org/W4385571093","https://openalex.org/W4385756679","https://openalex.org/W4386075767","https://openalex.org/W6637373629","https://openalex.org/W6640963894","https://openalex.org/W6779823529","https://openalex.org/W6783713337","https://openalex.org/W6844305113","https://openalex.org/W6849109464","https://openalex.org/W6855932524"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2350741829","https://openalex.org/W2530322880","https://openalex.org/W1596801655","https://openalex.org/W2359140296"],"abstract_inverted_index":{"Music":[0],"and":[1,20,40,79,83,101,115,124,159],"human-centric":[2,32,44],"video":[3,33],"are":[4,47],"two":[5,14],"fundamental":[6],"signals":[7],"across":[8],"languages.":[9],"Correlation":[10],"analysis":[11],"between":[12],"the":[13,87,95,102,131,135,152,157],"is":[15,69,142],"currently":[16],"used":[17],"in":[18,28],"choreography":[19],"film":[21],"accompaniment.":[22],"This":[23],"letter":[24],"explores":[25],"this":[26,51,162],"correlation":[27],"a":[29,36,72,138],"new":[30],"task:":[31],"generation":[34,45,74,164],"from":[35],"start-end":[37],"image":[38,103],"pair":[39],"transitional":[41],"music.":[42],"Existing":[43],"methods":[46],"not":[48],"competent":[49],"for":[50],"task":[52],"because":[53],"they":[54],"require":[55],"frame-wise":[56,132],"pose":[57],"as":[58,144],"input":[59],"or":[60],"have":[61,155],"difficulty":[62],"handling":[63],"long-duration":[64],"videos.":[65,128],"Our":[66],"key":[67],"idea":[68],"to":[70],"build":[71],"temporal":[73],"framework":[75,119],"dominated":[76],"by":[77,81,93],"DDPM":[78],"assisted":[80],"VAE":[82,100],"GAN.":[84,107],"It":[85],"reduces":[86],"computational":[88],"cost":[89],"of":[90,99,106,134,161],"music-image":[91],"diffusion":[92],"utilizing":[94],"latent":[96],"space":[97],"compactness":[98],"translation":[104],"efficiency":[105],"To":[108,129],"produce":[109],"videos":[110],"with":[111,151],"both":[112],"long":[113],"duration":[114],"high":[116],"quality,":[117],"our":[118],"first":[120],"generates":[121,126],"small-scale":[122],"keyframes":[123],"then":[125],"high-resolution":[127],"strengthen":[130],"consistency":[133],"human":[136],"body,":[137],"frame-aligned":[139],"correspondence":[140],"map":[141],"adopted":[143],"an":[145],"intermediate":[146],"supervision.":[147],"Extensive":[148],"experiments":[149],"compared":[150],"SOTA":[153],"method":[154],"demonstrated":[156],"rationality":[158],"effectiveness":[160],"signal":[163],"framework.":[165]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}