{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2025,7,23]],"date-time":"2025-07-23T12:27:01Z","timestamp":1753273621804,"version":"3.41.0"},"reference-count":80,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,4,26]],"date-time":"2025-04-26T00:00:00Z","timestamp":1745625600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,4,26]],"date-time":"2025-04-26T00:00:00Z","timestamp":1745625600000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.15223\/policy-037"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,4,26]]},"DOI":"10.1109\/icse55347.2025.00144","type":"proceedings-article","created":{"date-parts":[[2025,6,23]],"date-time":"2025-06-23T17:24:29Z","timestamp":1750699469000},"page":"463-475","source":"Crossref","is-referenced-by-count":1,"title":["SeeAction: Towards Reverse Engineering How-What-Where of HCI Actions from Screencasts for UI Automation"],"prefix":"10.1109","author":[{"given":"Dehai","family":"Zhao","sequence":"first","affiliation":[{"name":"CSIRO's Data61,Sydney,Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Zhenchang","family":"Xing","sequence":"additional","affiliation":[{"name":"CSIRO's Data61 & School of Computing, ANU,Sydney,Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Qinghua","family":"Lu","sequence":"additional","affiliation":[{"name":"CSIRO's Data61,Sydney,Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Xiwei","family":"Xu","sequence":"additional","affiliation":[{"name":"CSIRO's Data61,Sydney,Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Liming","family":"Zhu","sequence":"additional","affiliation":[{"name":"CSIRO's Data61 & School of CSE, UNSW,Sydney,Australia"}],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"263","reference":[{"key":"ref1","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330557"},{"key":"ref2","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2019.00022"},{"key":"ref3","doi-asserted-by":"publisher","DOI":"10.1145\/2884781.2884854"},{"key":"ref4","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00139"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338947"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00030"},{"key":"ref7","doi-asserted-by":"publisher","DOI":"10.1109\/ICST.2016.34"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213869"},{"key":"ref9","first-page":"297","article-title":"Roscript: a visual script driven truly non-intrusive robotic testing system for touch screen applications","volume-title":"Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering","author":"Qian"},{"volume-title":"Selenium","year":"2021","key":"ref10"},{"volume-title":"Appium","year":"2021","key":"ref11"},{"volume-title":"UIAutoma-tor","year":"2021","key":"ref12"},{"volume-title":"Netease","year":"2021","key":"ref14"},{"key":"ref15","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2013.6606553"},{"key":"ref16","doi-asserted-by":"publisher","DOI":"10.1109\/ISPASS.2015.7095807"},{"key":"ref17","doi-asserted-by":"publisher","DOI":"10.1109\/ASE.2015.90"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2018.2802916"},{"key":"ref19","doi-asserted-by":"publisher","DOI":"10.1145\/2814270.2814320"},{"volume-title":"Google","year":"2021","key":"ref20"},{"key":"ref22","doi-asserted-by":"publisher","DOI":"10.1145\/3106237.3117769"},{"volume-title":"A-mobile","key":"ref23"},{"key":"ref24","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.617"},{"key":"ref25","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2017.317"},{"key":"ref26","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE.2019.00049"},{"key":"ref27","first-page":"309","article-title":"Translating video recordings of mobile app usages into replayable scenarios","volume-title":"Proceedings of the ACM\/IEEE 42nd International Conference on Software Engineering","author":"Bernal-Cardenas"},{"key":"ref28","doi-asserted-by":"publisher","DOI":"10.1145\/3368089.3417940"},{"volume-title":"chatgpt","year":"2024","key":"ref29"},{"key":"ref30","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR52729.2023.00637"},{"journal-title":"Misar: A multi-modal instructional system with augmented reality","year":"2023","author":"Bi","key":"ref31"},{"key":"ref32","doi-asserted-by":"publisher","DOI":"10.1007\/978-3-031-02238-8"},{"key":"ref33","doi-asserted-by":"publisher","DOI":"10.1109\/ICCV.2015.510"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1145\/2380116.2380129"},{"volume-title":"Sikulix","year":"2021","key":"ref35"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.3115\/1073083.1073135"},{"key":"ref37","first-page":"65","article-title":"Meteor: An automatic metric for mt evaluation with improved correlation with human judgments","volume-title":"Proceedings of the acl workshop on intrinsic and extrinsic evaluation measures for machine translation and\/or summarization","author":"Banerjee"},{"key":"ref38","first-page":"74","article-title":"Rouge: A package for automatic evaluation of summaries","author":"Lin","year":"2004","journal-title":"Text summarization branches out"},{"key":"ref39","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7299087"},{"key":"ref40","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1212.0402"},{"key":"ref41","first-page":"1286","article-title":"Seenomaly: vision-based linting of gui animation effects against design-don\u2019t guidelines","volume-title":"2020 IEEE\/ACM 42nd International Conference on Software Engineering (ICSE)","author":"Zhao"},{"journal-title":"Ucf101: A dataset of 101 human actions classes from videos in the wild","year":"2012","author":"Soomro","key":"ref42"},{"journal-title":"Microsoft coco captions: Data collection and evaluation server","year":"2015","author":"Chen","key":"ref43"},{"volume-title":"Bugzilla","year":"2021","key":"ref44"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2015.7298935"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1023\/A:1007379606734"},{"volume-title":"GitHub","year":"2021","key":"ref47"},{"volume-title":"Google","year":"2021","key":"ref48"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/1357054.1357126"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1145\/1753326.1753554"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-015-9417-1"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/2884781.2884824"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1145\/371578.371593"},{"journal-title":"Extracting replayable interactions from videos of mobile app usage","year":"2022","author":"Chen","key":"ref55"},{"volume-title":"Robotium","year":"2021","key":"ref56"},{"volume-title":"Monkeyrunner","year":"2021","key":"ref57"},{"volume-title":"UI. Vision","year":"2021","key":"ref58"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.503"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.494"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3123266.3123448"},{"key":"ref62","doi-asserted-by":"publisher","DOI":"10.1007\/s11280-018-0531-z"},{"key":"ref63","doi-asserted-by":"publisher","DOI":"10.1109\/TMM.2017.2729019"},{"key":"ref64","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2017.111"},{"key":"ref65","doi-asserted-by":"publisher","DOI":"10.1145\/3220134.3220135"},{"key":"ref66","doi-asserted-by":"publisher","DOI":"10.1145\/3180155.3180240"},{"key":"ref67","first-page":"322","article-title":"Unblind your apps: Predicting natural-language labels for mobile gui components by deep learning","volume-title":"2020 IEEE\/ACM 42nd International Conference on Software Engineering (ICSE)","author":"Chen"},{"key":"ref68","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468604"},{"key":"ref69","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/2020.emnlp-main.443"},{"journal-title":"Spotlight: Mobile ui understanding using vision-language models with a focus","year":"2022","author":"Li","key":"ref70"},{"key":"ref71","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE48619.2023.00197"},{"key":"ref72","doi-asserted-by":"publisher","DOI":"10.1145\/3597503.3639163"},{"key":"ref73","doi-asserted-by":"publisher","DOI":"10.1109\/ICSE43902.2021.00091"},{"key":"ref74","first-page":"91","article-title":"Faster r-cnn: Towards real-time object detection with region proposal networks","volume":"28","author":"Ren","year":"2015","journal-title":"Advances in neural information processing systems"},{"key":"ref75","doi-asserted-by":"publisher","DOI":"10.1109\/CVPR.2016.91"},{"key":"ref76","doi-asserted-by":"publisher","DOI":"10.1145\/3293882.3330551"},{"key":"ref77","doi-asserted-by":"publisher","DOI":"10.1109\/ICMLA51294.2020.00200"},{"key":"ref78","first-page":"398","article-title":"Owl eyes: Spotting ui display issues via visual understanding","volume-title":"2020 35th IEEE\/ACM International Conference on Automated Software Engineering (ASE)","author":"Liu"},{"key":"ref79","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3473935"},{"key":"ref80","article-title":"Deep-GUI. Towards Platform-Independent UI Input Generation with Deep Reinforcement Learning","author":"YazdaniBanafsheDaragh","year":"2020","journal-title":"University of California, Irvine"},{"key":"ref81","doi-asserted-by":"publisher","DOI":"10.48550\/ARXIV.1706.03762"},{"journal-title":"Bert: Pre-training of deep bidirectional transformers for language understanding","year":"2018","author":"Devlin","key":"ref82"}],"event":{"name":"2025 IEEE\/ACM 47th International Conference on Software Engineering (ICSE)","start":{"date-parts":[[2025,4,26]]},"location":"Ottawa, ON, Canada","end":{"date-parts":[[2025,5,6]]}},"container-title":["2025 IEEE\/ACM 47th International Conference on Software Engineering (ICSE)"],"original-title":[],"link":[{"URL":"https:\/\/2.zoppoz.workers.dev:443\/http\/xplorestaging.ieee.org\/ielx8\/11029684\/11029718\/11029891.pdf?arnumber=11029891","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,24]],"date-time":"2025-06-24T06:20:46Z","timestamp":1750746046000},"score":1,"resource":{"primary":{"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/ieeexplore.ieee.org\/document\/11029891\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,4,26]]},"references-count":80,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.1109\/icse55347.2025.00144","relation":{},"subject":[],"published":{"date-parts":[[2025,4,26]]}}}