{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,18]],"date-time":"2026-07-18T02:39:28Z","timestamp":1784342368137,"version":"3.55.0"},"reference-count":65,"publisher":"IEEE","license":[{"start":{"date-parts":[[2025,11,16]],"date-time":"2025-11-16T00:00:00Z","timestamp":1763251200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.15223\/policy-029"},{"start":{"date-parts":[[2025,11,16]],"date-time":"2025-11-16T00:00:00Z","timestamp":1763251200000},"content-version":"stm-asf","delay-in-days":0,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.15223\/policy-037"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/100006190","name":"Research and Development","doi-asserted-by":"publisher","id":[{"id":"10.13039\/100006190","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"published-print":{"date-parts":[[2025,11,16]]},"DOI":"10.1109\/ase63991.2025.00278","type":"proceedings-article","created":{"date-parts":[[2026,1,28]],"date-time":"2026-01-28T20:54:38Z","timestamp":1769633678000},"page":"3369-3380","source":"Crossref","is-referenced-by-count":1,"title":["A Characterization Study of Bugs in LLM Agent Workflow Orchestration Frameworks"],"prefix":"10.1109","author":[{"given":"Ziluo","family":"Xue","sequence":"first","affiliation":[{"name":"Huazhong University of Science and Technology,Wuhan,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Yanjie","family":"Zhao","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology,Wuhan,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Shenao","family":"Wang","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology,Wuhan,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Kai","family":"Chen","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology,Wuhan,China"}],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Haoyu","family":"Wang","sequence":"additional","affiliation":[{"name":"Huazhong University of Science and Technology,Wuhan,China"}],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"263","reference":[{"key":"ref1","article-title":"GPT-4 technical report","volume-title":"CoRR","year":"2023"},{"key":"ref2","article-title":"Llama: Open and efficient foundation language models","volume-title":"CoRR","author":"Touvron","year":"2023"},{"key":"ref3","article-title":"Deepseek-v3 technical report","volume-title":"CoRR","year":"2024"},{"key":"ref4","first-page":"8048","article-title":"Large language model based multi-agents: A survey of progress and challenges","volume-title":"Proceedings of the Thirty-Third International Joint Conference on Artificial Intelligence, IJCAI 2024, Jeju, South Korea, August 3-9, 2024","author":"Guo"},{"key":"ref5","doi-asserted-by":"publisher","DOI":"10.1007\/s11704-024-40231-1"},{"key":"ref6","doi-asserted-by":"publisher","DOI":"10.1109\/tse.2024.3428972"},{"key":"ref7","article-title":"Generating images with multimodal language models","volume-title":"Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023","author":"Koh"},{"key":"ref8","doi-asserted-by":"publisher","DOI":"10.1609\/aaai.v38i17.29837"},{"key":"ref9","article-title":"Fincon: A synthesized LLM multi-agent system with conceptual verbal reinforcement for enhanced financial decision making","volume-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver, BC, Canada, December 10 - 15, 2024","author":"Yu"},{"key":"ref10","article-title":"EAI: emotional decision-making of llms in strategic games and ethical dilemmas","volume-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver, BC, Canada, December 10 - 15, 2024","author":"Mozikov"},{"key":"ref11","article-title":"Mdagents: An adaptive collaboration of llms for medical decision-making","volume-title":"Advances in Neural Information Processing Systems 38: Annual Conference on Neural Information Processing Systems 2024, NeurIPS 2024, Vancouver, BC, Canada, December 10 - 15, 2024","author":"Kim"},{"key":"ref12","article-title":"Awesome LLM Agent Frameworks"},{"key":"ref13","doi-asserted-by":"publisher","DOI":"10.1145\/3708531"},{"key":"ref14","article-title":"Sok: Understanding vulnerabilities in the large language model supply chain","volume-title":"CoRR","author":"Wang","year":"2025"},{"key":"ref15","article-title":"LangChain","author":"Chase","year":"2022"},{"key":"ref16","article-title":"LlamaIndex","author":"Liu","year":"2022"},{"key":"ref17","article-title":"Haystack: the end-to-end NLP framework for pragmatic builders","author":"Pietsch","year":"2019"},{"key":"ref18","doi-asserted-by":"publisher","DOI":"10.1016\/j.eswa.2024.126113"},{"key":"ref19","article-title":"Defining and detecting the defects of the large language model-based autonomous agents","volume-title":"CoRR","author":"Ning","year":"2024"},{"key":"ref20","article-title":"LangChain Documentation","author":"Chase","year":"2022"},{"key":"ref21","article-title":"LlamaIndex Documentation","author":"Liu","year":"2022"},{"key":"ref22","article-title":"Haystack Documentation","author":"Pietsch","year":"2019"},{"key":"ref23","article-title":"AutoGPT Documentation","author":"Gravitas"},{"key":"ref24","article-title":"AutoGen Documentation"},{"key":"ref25","article-title":"outlines Documentation","author":"ai"},{"key":"ref26","article-title":"FlowiseAI Documentation"},{"key":"ref27","article-title":"crewAI Documentation"},{"key":"ref28","article-title":"AutoGPT","author":"Gravitas","year":"2023"},{"key":"ref29","article-title":"AutoGen","year":"2023"},{"key":"ref30","article-title":"outlines","year":"2023"},{"key":"ref31","article-title":"FlowiseAI","year":"2023"},{"key":"ref32","article-title":"crewAI","year":"2023"},{"key":"ref33","article-title":"A systematic survey of prompt engineering in large language models: Techniques and applications","volume-title":"CoRR","author":"Sahoo","year":"2024"},{"key":"ref34","doi-asserted-by":"publisher","DOI":"10.1007\/978-981-99-7962-2_30"},{"key":"ref35","article-title":"Understanding the planning of llm agents: A survey","volume-title":"CoRR","author":"Huang","year":"2024"},{"key":"ref36","doi-asserted-by":"publisher","DOI":"10.1007\/s44336-024-00009-2"},{"key":"ref37","article-title":"LangChain Documentation of Chain Mechanism","author":"Chase","year":"2022"},{"key":"ref38","article-title":"Haystack Documentation of Pipeline","author":"Pietsch","year":"2019"},{"key":"ref39","article-title":"LangChain Documentation of Memory Module","author":"Chase","year":"2022"},{"key":"ref40","doi-asserted-by":"crossref","DOI":"10.21428\/e90189c8.43aeb06e","article-title":"Scalability and Performance Benchmarking of LangChain, LlamaIndex, and Haystack for Enterprise AI Customer Support Systems","volume-title":"IJGIS Fall of 2024 Conference","author":"Malviya"},{"key":"ref41","article-title":"GitHub Search API Documentation"},{"key":"ref42","doi-asserted-by":"publisher","DOI":"10.1109\/32.799955"},{"key":"ref43","doi-asserted-by":"publisher","DOI":"10.1177\/001316446002000104"},{"key":"ref44","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2010.31"},{"key":"ref45","doi-asserted-by":"publisher","DOI":"10.1145\/3363562"},{"key":"ref46","doi-asserted-by":"publisher","DOI":"10.1109\/ms.2008.130"},{"key":"ref47","doi-asserted-by":"publisher","DOI":"10.1109\/TSE.2006.38"},{"key":"ref48","doi-asserted-by":"publisher","DOI":"10.1145\/3213846.3213866"},{"key":"ref49","doi-asserted-by":"publisher","DOI":"10.1145\/3551349.3560427"},{"key":"ref50","doi-asserted-by":"publisher","DOI":"10.1145\/3587155"},{"key":"ref51","doi-asserted-by":"publisher","DOI":"10.1016\/j.infsof.2022.107004"},{"key":"ref52","doi-asserted-by":"publisher","DOI":"10.1145\/3338906.3338955"},{"key":"ref53","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380395"},{"key":"ref54","doi-asserted-by":"publisher","DOI":"10.1145\/3468264.3468591"},{"key":"ref55","doi-asserted-by":"publisher","DOI":"10.1016\/j.jss.2021.110935"},{"key":"ref56","doi-asserted-by":"publisher","DOI":"10.1109\/issre.2012.22"},{"key":"ref57","doi-asserted-by":"publisher","DOI":"10.1109\/apsec.2017.41"},{"key":"ref58","doi-asserted-by":"publisher","DOI":"10.1007\/s10664-023-10400-0"},{"key":"ref59","doi-asserted-by":"publisher","DOI":"10.1109\/msr.2017.59"},{"key":"ref60","doi-asserted-by":"publisher","DOI":"10.1145\/3624743"},{"key":"ref61","doi-asserted-by":"publisher","DOI":"10.1145\/3377811.3380397"},{"key":"ref62","article-title":"Breaking agents: Compromising autonomous LLM agents through malfunction amplification","volume-title":"CoRR","author":"Zhang","year":"2024"},{"key":"ref63","doi-asserted-by":"crossref","DOI":"10.1145\/3711896.3736561","article-title":"A survey on trustworthy LLM agents: Threats and countermeasures","volume-title":"CoRR","author":"Yu","year":"2025"},{"key":"ref64","article-title":"Why do multi-agent LLM systems fail?","volume-title":"CoRR","author":"Cemri","year":"2025"},{"key":"ref65","article-title":"Which agent causes task failures and when? on automated failure attribution of llm multi-agent systems","author":"Zhang","year":"2025"}],"event":{"name":"2025 40th IEEE\/ACM International Conference on Automated Software Engineering (ASE)","location":"Seoul, Korea, Republic of","start":{"date-parts":[[2025,11,16]]},"end":{"date-parts":[[2025,11,20]]}},"container-title":["2025 40th IEEE\/ACM International Conference on Automated Software Engineering (ASE)"],"original-title":[],"link":[{"URL":"https:\/\/2.zoppoz.workers.dev:443\/http\/xplorestaging.ieee.org\/ielx8\/11334056\/11334198\/11334467.pdf?arnumber=11334467","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2026,1,29]],"date-time":"2026-01-29T09:48:20Z","timestamp":1769680100000},"score":1,"resource":{"primary":{"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/ieeexplore.ieee.org\/document\/11334467\/"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2025,11,16]]},"references-count":65,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.1109\/ase63991.2025.00278","relation":{},"subject":[],"published":{"date-parts":[[2025,11,16]]}}}