{"id":"https://openalex.org/W4417514321","doi":"https://doi.org/10.48550/arxiv.2505.06855","title":"Joint Low-level and High-level Textual Representation Learning with Multiple Masking Strategies","display_name":"Joint Low-level and High-level Textual Representation Learning with Multiple Masking Strategies","publication_year":2025,"publication_date":"2025-05-11","ids":{"openalex":"https://openalex.org/W4417514321","doi":"https://doi.org/10.48550/arxiv.2505.06855"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2505.06855","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.06855","pdf_url":"https://arxiv.org/pdf/2505.06855","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2505.06855","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017897073","display_name":"Zhengmi Tang","orcid":"https://orcid.org/0000-0003-2011-8105"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tang, Zhengmi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068449854","display_name":"Yuto Mitsui","orcid":"https://orcid.org/0009-0005-2525-4837"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mitsui, Yuto","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009192524","display_name":"Tomo Miyazaki","orcid":"https://orcid.org/0000-0001-5205-0542"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Miyazaki, Tomo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5020830042","display_name":"Shinichiro Omachi","orcid":"https://orcid.org/0000-0001-7706-9995"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Omachi, Shinichiro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10601","display_name":"Handwritten Text Recognition Techniques","score":0.8942000269889832,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10601","display_name":"Handwritten Text Recognition Techniques","score":0.8942000269889832,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.012799999676644802,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.011599999852478504,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/masking","display_name":"Masking (illustration)","score":0.7924000024795532},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.5860999822616577},{"id":"https://openalex.org/keywords/joint","display_name":"Joint (building)","score":0.5735999941825867},{"id":"https://openalex.org/keywords/autoencoder","display_name":"Autoencoder","score":0.5461999773979187},{"id":"https://openalex.org/keywords/feature-learning","display_name":"Feature learning","score":0.5407999753952026},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.49639999866485596},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.48089998960494995},{"id":"https://openalex.org/keywords/domain","display_name":"Domain (mathematical analysis)","score":0.4300999939441681}],"concepts":[{"id":"https://openalex.org/C2777402240","wikidata":"https://www.wikidata.org/wiki/Q6783436","display_name":"Masking (illustration)","level":2,"score":0.7924000024795532},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7585999965667725},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6534000039100647},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5860999822616577},{"id":"https://openalex.org/C18555067","wikidata":"https://www.wikidata.org/wiki/Q8375051","display_name":"Joint (building)","level":2,"score":0.5735999941825867},{"id":"https://openalex.org/C101738243","wikidata":"https://www.wikidata.org/wiki/Q786435","display_name":"Autoencoder","level":3,"score":0.5461999773979187},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.5407999753952026},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.49639999866485596},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.48089998960494995},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.44209998846054077},{"id":"https://openalex.org/C36503486","wikidata":"https://www.wikidata.org/wiki/Q11235244","display_name":"Domain (mathematical analysis)","level":2,"score":0.4300999939441681},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.38749998807907104},{"id":"https://openalex.org/C197115733","wikidata":"https://www.wikidata.org/wiki/Q1003136","display_name":"Forcing (mathematics)","level":2,"score":0.35179999470710754},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.3418999910354614},{"id":"https://openalex.org/C22367795","wikidata":"https://www.wikidata.org/wiki/Q7625208","display_name":"Structured prediction","level":2,"score":0.336899995803833},{"id":"https://openalex.org/C28490314","wikidata":"https://www.wikidata.org/wiki/Q189436","display_name":"Speech recognition","level":1,"score":0.30570000410079956},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.2777000069618225},{"id":"https://openalex.org/C35639132","wikidata":"https://www.wikidata.org/wiki/Q7452468","display_name":"Sequence labeling","level":3,"score":0.2766000032424927},{"id":"https://openalex.org/C2778753569","wikidata":"https://www.wikidata.org/wiki/Q1960395","display_name":"Span (engineering)","level":2,"score":0.2703000009059906},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.26350000500679016},{"id":"https://openalex.org/C2779304628","wikidata":"https://www.wikidata.org/wiki/Q3503480","display_name":"Face (sociological concept)","level":2,"score":0.2502000033855438}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2505.06855","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.06855","pdf_url":"https://arxiv.org/pdf/2505.06855","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2505.06855","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2505.06855","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2505.06855","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2505.06855","pdf_url":"https://arxiv.org/pdf/2505.06855","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Most":[0],"existing":[1],"text":[2,65,108,174],"recognition":[3,109],"methods":[4,168],"are":[5],"trained":[6],"on":[7],"large-scale":[8],"synthetic":[9],"datasets":[10],"due":[11],"to":[12,127],"the":[13,71,95,107,115,125,147,165],"scarcity":[14],"of":[15],"labeled":[16],"real-world":[17,25,43],"datasets.":[18],"Synthetic":[19],"images,":[20],"however,":[21],"cannot":[22],"faithfully":[23],"reproduce":[24],"scenarios,":[26],"such":[27],"as":[28],"uneven":[29],"illumination,":[30],"irregular":[31],"layout,":[32],"occlusion,":[33],"and":[34,52,76,103,119,143,154,177],"degradation,":[35],"resulting":[36],"in":[37,106,169],"performance":[38],"disparities":[39],"when":[40],"handling":[41],"complex":[42],"images.":[44,66],"Recent":[45],"self-supervised":[46,167],"learning":[47,51],"techniques,":[48],"notably":[49],"contrastive":[50],"masked":[53],"image":[54,117],"modeling":[55],"(MIM),":[56],"narrow":[57],"this":[58],"domain":[59],"gap":[60],"by":[61],"exploiting":[62],"unlabeled":[63],"real":[64,161],"This":[67],"study":[68],"first":[69],"analyzes":[70],"original":[72],"Masked":[73],"AutoEncoder":[74],"(MAE)":[75],"observes":[77],"that":[78],"random":[79,101,140],"patch":[80],"masking":[81,105,145],"predominantly":[82],"captures":[83],"low-level":[84],"textural":[85],"features":[86],"but":[87],"misses":[88],"high-level":[89,96,155],"contextual":[90,97],"representations.":[91,157],"To":[92],"fully":[93],"exploit":[94],"representations,":[98],"we":[99],"introduce":[100],"blockwise":[102],"span":[104,144],"task.":[110],"These":[111],"strategies":[112],"can":[113],"mask":[114],"continuous":[116],"patches":[118],"completely":[120],"remove":[121],"some":[122],"characters,":[123],"forcing":[124],"model":[126],"infer":[128],"relationships":[129],"among":[130],"characters":[131],"within":[132],"a":[133],"word.":[134],"Our":[135],"Multi-Masking":[136],"Strategy":[137],"(MMS)":[138],"integrates":[139],"patch,":[141],"blockwise,":[142],"into":[146],"MIM":[148],"frame,":[149],"which":[150],"jointly":[151],"learns":[152],"low":[153],"textual":[156],"After":[158],"fine-tuning":[159],"with":[160],"data,":[162],"MMS":[163],"outperforms":[164],"state-of-the-art":[166],"various":[170],"text-related":[171],"tasks,":[172],"including":[173],"recognition,":[175],"segmentation,":[176],"text-image":[178],"super-resolution.":[179]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}