{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,28]],"date-time":"2026-04-28T23:03:33Z","timestamp":1777417413081,"version":"3.51.4"},"reference-count":42,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2018,11,1]],"date-time":"2018-11-01T00:00:00Z","timestamp":1541030400000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/www.elsevier.com\/tdm\/userlicense\/1.0\/"}],"funder":[{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61432008"],"award-info":[{"award-number":["61432008"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61503178"],"award-info":[{"award-number":["61503178"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]},{"DOI":"10.13039\/501100001809","name":"National Natural Science Foundation of China","doi-asserted-by":"publisher","award":["61321491"],"award-info":[{"award-number":["61321491"]}],"id":[{"id":"10.13039\/501100001809","id-type":"DOI","asserted-by":"publisher"}]}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Knowledge-Based Systems"],"published-print":{"date-parts":[[2018,11]]},"DOI":"10.1016\/j.knosys.2018.07.010","type":"journal-article","created":{"date-parts":[[2018,7,5]],"date-time":"2018-07-05T14:53:19Z","timestamp":1530802399000},"page":"244-258","update-policy":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":31,"special_numbering":"C","title":["Active learning with confidence-based answers for crowdsourcing labeling tasks"],"prefix":"10.1016","volume":"159","author":[{"given":"Jinhua","family":"Song","sequence":"first","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Hao","family":"Wang","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"ORCID":"https:\/\/2.zoppoz.workers.dev:443\/https\/orcid.org\/0000-0002-2488-1813","authenticated-orcid":false,"given":"Yang","family":"Gao","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]},{"given":"Bo","family":"An","sequence":"additional","affiliation":[],"role":[{"role":"author","vocabulary":"crossref"}]}],"member":"78","reference":[{"issue":"1","key":"10.1016\/j.knosys.2018.07.010_bib0001","doi-asserted-by":"crossref","first-page":"142","DOI":"10.1109\/TPAMI.2015.2437384","article-title":"Region-based convolutional networks for accurate object detection and segmentation","volume":"38","author":"Girshick","year":"2016","journal-title":"IEEE Trans. Pattern Anal. Mach. Intell."},{"issue":"Supplement C","key":"10.1016\/j.knosys.2018.07.010_bib0002","doi-asserted-by":"crossref","first-page":"69","DOI":"10.1016\/j.knosys.2017.09.032","article-title":"Active and adaptive ensemble learning for online activity recognition from data streams","volume":"138","author":"Krawczyk","year":"2017","journal-title":"Knowl. Based Syst."},{"key":"10.1016\/j.knosys.2018.07.010_bib0003","series-title":"Proceedings of the 11th IEEE International Symposium on Intelligent Systems and Informatics","first-page":"191","article-title":"Active learning enhanced semi-automatic annotation tool for aspect-based sentiment analysis","author":"Smatana","year":"2013"},{"key":"10.1016\/j.knosys.2018.07.010_bib0004","series-title":"Proceedings of the 2014 International Working Conference on Advanced Visual Interfaces","first-page":"405","article-title":"User interface design for crowdsourcing systems","author":"Rahmanian","year":"2014"},{"key":"10.1016\/j.knosys.2018.07.010_bib0005","series-title":"Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing","first-page":"254","article-title":"Cheap and fast - but is it good? evaluating non-expert annotations for natural language tasks","author":"Snow","year":"2008"},{"issue":"2","key":"10.1016\/j.knosys.2018.07.010_bib0006","doi-asserted-by":"crossref","first-page":"249","DOI":"10.1007\/s10115-012-0507-8","article-title":"A survey on instance selection for active learning","volume":"35","author":"Fu","year":"2013","journal-title":"Knowl. Inf. Syst."},{"key":"10.1016\/j.knosys.2018.07.010_bib0007","series-title":"Computer Sciences Technical Report 1648","article-title":"Active Learning Literature Survey","author":"Settles","year":"2010"},{"issue":"Supplement C","key":"10.1016\/j.knosys.2018.07.010_bib0008","doi-asserted-by":"crossref","first-page":"179","DOI":"10.1016\/j.knosys.2017.06.023","article-title":"A combination of active learning and self-learning for named entity recognition on twitter using conditional random fields","volume":"132","author":"Tran","year":"2017","journal-title":"Knowl. Based Syst."},{"issue":"Supplement C","key":"10.1016\/j.knosys.2018.07.010_bib0009","doi-asserted-by":"crossref","first-page":"96","DOI":"10.1016\/j.knosys.2016.06.003","article-title":"Noise filtering to improve data and model quality for crowdsourcing","volume":"107","author":"Li","year":"2016","journal-title":"Knowl. Based Syst."},{"key":"10.1016\/j.knosys.2018.07.010_bib0010","series-title":"Proceedings of the 15th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","first-page":"259","article-title":"Efficiently learning the accuracy of labeling sources for selective sampling","author":"Donmez","year":"2009"},{"key":"10.1016\/j.knosys.2018.07.010_bib0011","series-title":"Proceedings of the 28th International Conference on Machine Learning","first-page":"1161","article-title":"Active learning from crowds","author":"Yan","year":"2011"},{"key":"10.1016\/j.knosys.2018.07.010_bib0012","series-title":"Advances in Neural Information Processing Systems 28: 29th Annual Conference on Neural Information Processing Systems","first-page":"703","article-title":"Active learning from weak and strong labelers","author":"Zhang","year":"2015"},{"key":"10.1016\/j.knosys.2018.07.010_bib0013","first-page":"1","article-title":"Statistical decision making for optimal budget allocation in crowd labeling","volume":"16","author":"Chen","year":"2015","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.knosys.2018.07.010_bib0014","series-title":"Proceedings of the 17th International Conference on Artificial Intelligence and Statistics","first-page":"832","article-title":"Sequential crowdsourced labeling as an epsilon-greedy exploration in a markov decision process","author":"Raykar","year":"2014"},{"key":"10.1016\/j.knosys.2018.07.010_bib0015","series-title":"Proceedings of the 30th AAAI Conference on Artificial Intelligence","first-page":"1845","article-title":"Re-active learning: Active learning with relabeling","author":"Lin","year":"2016"},{"key":"10.1016\/j.knosys.2018.07.010_bib0016","series-title":"Proceedings of the 14th ACM SIGKDD International Conference on Knowledge Discovery and Data Mining","first-page":"614","article-title":"Get another label? improving data quality and data mining using multiple, noisy labelers","author":"Sheng","year":"2008"},{"issue":"5","key":"10.1016\/j.knosys.2018.07.010_bib0017","first-page":"1081","article-title":"Active learning with imbalanced multiple noisy labeling","volume":"45","author":"Zhang","year":"2015","journal-title":"IEEE Trans. Cybern."},{"issue":"2","key":"10.1016\/j.knosys.2018.07.010_bib0018","doi-asserted-by":"crossref","first-page":"402","DOI":"10.1007\/s10618-013-0306-1","article-title":"Repeated labeling using multiple noisy labelers","volume":"28","author":"Ipeirotis","year":"2014","journal-title":"Data Min. Knowl. Discov."},{"key":"10.1016\/j.knosys.2018.07.010_bib0019","series-title":"Proceedings of the 24th International Joint Conference on Artificial Intelligence","first-page":"1061","article-title":"Active learning from crowds with unsure option","author":"Zhong","year":"2015"},{"key":"10.1016\/j.knosys.2018.07.010_bib0020","series-title":"Proceedings of the 28th AAAI Conference on Artificial Intelligence","first-page":"1809","article-title":"Active learning for crowdsourcing using knowledge transfer","author":"Fang","year":"2014"},{"key":"10.1016\/j.knosys.2018.07.010_bib0021","series-title":"Proceedings of the 17th ACM Conference on Information and Knowledge Management","first-page":"619","article-title":"Proactive learning: Cost-sensitive active learning with multiple imperfect oracles","author":"Donmez","year":"2008"},{"key":"10.1016\/j.knosys.2018.07.010_bib0022","series-title":"Proceedings of The 10th IEEE International Conference on Data Mining","first-page":"639","article-title":"Active learning from multiple noisy labelers with varied costs","author":"Zheng","year":"2010"},{"key":"10.1016\/j.knosys.2018.07.010_bib0023","series-title":"Proceedings of the 19th European Conference on Artificial Intelligence","first-page":"145","article-title":"Using crowdsourcing and active learning to track sentiment in online media","author":"Brew","year":"2010"},{"key":"10.1016\/j.knosys.2018.07.010_bib0024","series-title":"Proceedings of the 2011 Conference on Empirical Methods in Natural Language Processing","first-page":"1546","article-title":"Active learning with amazon mechanical turk","author":"Laws","year":"2011"},{"key":"10.1016\/j.knosys.2018.07.010_bib0025","series-title":"2011 IEEE 3rd International Conference on Privacy, Security, Risk and Trust and 2011 IEEE 3rd International Conference on Social Computing","first-page":"728","article-title":"Incremental relabeling for active learning with noisy crowdsourced annotations","author":"Zhao","year":"2011"},{"issue":"2","key":"10.1016\/j.knosys.2018.07.010_bib0026","doi-asserted-by":"crossref","first-page":"125","DOI":"10.14778\/2735471.2735474","article-title":"Scaling up crowd-sourcing to very large datasets: a case for active learning","volume":"8","author":"Mozafari","year":"2014","journal-title":"Proc. VLDB Endowment"},{"key":"10.1016\/j.knosys.2018.07.010_bib0027","series-title":"Proceedings of the 30th International Conference on Machine Learning","first-page":"534","article-title":"Adaptive task assignment for crowdsourced classification","author":"Ho","year":"2013"},{"issue":"7","key":"10.1016\/j.knosys.2018.07.010_bib0028","doi-asserted-by":"crossref","first-page":"799","DOI":"10.1080\/0266476042000214501","article-title":"Beta regression for modelling rates and proportions","volume":"31","author":"Ferrari","year":"2004","journal-title":"J. Appl. Stat."},{"issue":"2","key":"10.1016\/j.knosys.2018.07.010_bib0029","doi-asserted-by":"crossref","first-page":"960","DOI":"10.1016\/j.csda.2005.10.002","article-title":"Improved point and interval estimation for a beta regression model","volume":"51","author":"Ospina","year":"2006","journal-title":"Comput. Stat. Data Anal."},{"key":"10.1016\/j.knosys.2018.07.010_bib0030","series-title":"Numerical Recipes in C: The Art of Scientific Computing","author":"Press","year":"1992"},{"key":"10.1016\/j.knosys.2018.07.010_bib0031","series-title":"An Introduction to the Bootstrap","author":"Efron","year":"1994"},{"key":"10.1016\/j.knosys.2018.07.010_bib0032","series-title":"Bootstrap Methods and Their Application","author":"Davison","year":"2013"},{"issue":"301","key":"10.1016\/j.knosys.2018.07.010_bib0033","doi-asserted-by":"crossref","first-page":"13","DOI":"10.1080\/01621459.1963.10500830","article-title":"Probability inequalities for sums of bounded random variables","volume":"58","author":"Hoeffding","year":"1963","journal-title":"J. Am. Stat. Assoc."},{"key":"10.1016\/j.knosys.2018.07.010_bib0034","series-title":"Proceedings of the 20th International Conference on Machine Learning","first-page":"59","article-title":"Incorporating diversity in active learning with support vector machines","author":"Brinker","year":"2003"},{"issue":"2","key":"10.1016\/j.knosys.2018.07.010_bib0035","doi-asserted-by":"crossref","first-page":"345","DOI":"10.3233\/IDA-150720","article-title":"A batch-mode active learning SVM method based on semi-supervised clustering","volume":"19","author":"Fu","year":"2015","journal-title":"Intell. Data Anal."},{"key":"10.1016\/j.knosys.2018.07.010_bib0036","series-title":"Proceedings of the 15th International Conference on World Wide Web","first-page":"633","article-title":"Large-scale text categorization by batch mode active learning","author":"Hoi","year":"2006"},{"key":"10.1016\/j.knosys.2018.07.010_bib0037","doi-asserted-by":"crossref","first-page":"88","DOI":"10.1016\/j.neucom.2015.11.062","article-title":"A batch-mode active learning framework by querying discriminative and representative samples for hyperspectral image classification","volume":"179","author":"Wang","year":"2016","journal-title":"Neurocomputing"},{"key":"10.1016\/j.knosys.2018.07.010_bib0038","unstructured":"D. Dheeru, E. Karra Taniskidou, UCI machine learning repository, 2017."},{"issue":"1","key":"10.1016\/j.knosys.2018.07.010_bib0039","doi-asserted-by":"crossref","first-page":"5","DOI":"10.1023\/A:1010933404324","article-title":"Random forests","volume":"45","author":"Breiman","year":"2001","journal-title":"Mach Learn"},{"key":"10.1016\/j.knosys.2018.07.010_bib0040","series-title":"Data Mining: Practical Machine Learning Tools and Techniques","author":"Witten","year":"2011"},{"issue":"1","key":"10.1016\/j.knosys.2018.07.010_bib0041","first-page":"3133","article-title":"Do we need hundreds of classifiers to solve real world classification problems?","volume":"15","author":"Delgado","year":"2014","journal-title":"J. Mach. Learn. Res."},{"key":"10.1016\/j.knosys.2018.07.010_bib0042","unstructured":"FG-NET, https:\/\/2.zoppoz.workers.dev:443\/http\/www-prima.inrialpes.fr\/FGnet\/, 2014."}],"container-title":["Knowledge-Based Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/api.elsevier.com\/content\/article\/PII:S0950705118303460?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/api.elsevier.com\/content\/article\/PII:S0950705118303460?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2019,10,20]],"date-time":"2019-10-20T07:32:19Z","timestamp":1571556739000},"score":1,"resource":{"primary":{"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/linkinghub.elsevier.com\/retrieve\/pii\/S0950705118303460"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2018,11]]},"references-count":42,"alternative-id":["S0950705118303460"],"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.1016\/j.knosys.2018.07.010","relation":{},"ISSN":["0950-7051"],"issn-type":[{"value":"0950-7051","type":"print"}],"subject":[],"published":{"date-parts":[[2018,11]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Active learning with confidence-based answers for crowdsourcing labeling tasks","name":"articletitle","label":"Article Title"},{"value":"Knowledge-Based Systems","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.1016\/j.knosys.2018.07.010","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2018 Elsevier B.V. All rights reserved.","name":"copyright","label":"Copyright"}]}}