{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,7,29]],"date-time":"2026-07-29T04:00:47Z","timestamp":1785297647116,"version":"3.55.0"},"reference-count":53,"publisher":"Elsevier BV","license":[{"start":{"date-parts":[[2019,2,1]],"date-time":"2019-02-01T00:00:00Z","timestamp":1548979200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/www.elsevier.com\/tdm\/userlicense\/1.0\/"},{"start":{"date-parts":[[2019,2,1]],"date-time":"2019-02-01T00:00:00Z","timestamp":1548979200000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/www.elsevier.com\/legal\/tdmrep-license"}],"content-domain":{"domain":["elsevier.com","sciencedirect.com"],"crossmark-restriction":true},"short-container-title":["Information and Software Technology"],"published-print":{"date-parts":[[2019,2]]},"DOI":"10.1016\/j.infsof.2018.10.006","type":"journal-article","created":{"date-parts":[[2018,10,9]],"date-time":"2018-10-09T06:16:22Z","timestamp":1539065782000},"page":"201-230","update-policy":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.1016\/elsevier_cm_policy","source":"Crossref","is-referenced-by-count":258,"special_numbering":"C","title":["Identifying, categorizing and mitigating threats to validity in software engineering secondary studies"],"prefix":"10.1016","volume":"106","author":[{"given":"Apostolos","family":"Ampatzoglou","sequence":"first","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"ORCID":"https:\/\/2.zoppoz.workers.dev:443\/https\/orcid.org\/0000-0003-4248-3752","authenticated-orcid":false,"given":"Stamatia","family":"Bibi","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Paris","family":"Avgeriou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Marijn","family":"Verbeek","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]},{"given":"Alexander","family":"Chatzigeorgiou","sequence":"additional","affiliation":[],"role":[{"vocabulary":"crossref","role":"author"}]}],"member":"78","reference":[{"issue":"4","key":"10.1016\/j.infsof.2018.10.006_bib0001","doi-asserted-by":"crossref","first-page":"283","DOI":"10.1177\/0193841X16665199","article-title":"External validity: the next step for systematic reviews?","volume":"41","author":"Avellar","year":"2017","journal-title":"Eval. Rev."},{"key":"10.1016\/j.infsof.2018.10.006_bib0002","series-title":"2014 IEEE Fourth International Workshop on","first-page":"9","article-title":"Systematic reviews in requirements engineering: a tertiary study\u201d, Empirical Requirements Engineering (EmpiRE)","author":"Bano","year":"2014"},{"issue":"1\u20132","key":"10.1016\/j.infsof.2018.10.006_bib0003","doi-asserted-by":"crossref","first-page":"171","DOI":"10.1016\/0951-8320(91)90053-A","article-title":"Paradigms for experimentation and empirical studies in software engineering","volume":"32","author":"Basili","year":"1991","journal-title":"Reliab. Eng. Syst. Saf."},{"key":"10.1016\/j.infsof.2018.10.006_bib0004","series-title":"8th ACM\/IEEE International Symposium on Empirical Software Engineering and Measurement (ESEM)","article-title":"Towards a semantic knowledge base on threats to validity and control actions in controlled experiments","author":"Biffl","year":"2014"},{"key":"10.1016\/j.infsof.2018.10.006_bib0005","doi-asserted-by":"crossref","first-page":"62","DOI":"10.1016\/j.infsof.2017.10.017","article-title":"Reporting systematic reviews: some lessons from a tertiary study","volume":"95","author":"Budgen","year":"2018","journal-title":"Inf. Softw. Technol."},{"issue":"6","key":"10.1016\/j.infsof.2018.10.006_bib0006","doi-asserted-by":"crossref","first-page":"1051","DOI":"10.1016\/j.jss.2007.08.028","article-title":"An analysis of research topics in software engineering \u2013 2006","volume":"81","author":"Cai","year":"2008","journal-title":"J. Syst. Softw."},{"issue":"2","key":"10.1016\/j.infsof.2018.10.006_bib0007","first-page":"1","article-title":"The Database of Abstracts of Reviews of Effects (DARE)","volume":"6","year":"2002","journal-title":"Effect. Matters"},{"key":"10.1016\/j.infsof.2018.10.006_bib0008","article-title":"Quasi-experimentation: design & analysis issues for field settings","author":"Cook","year":"1979","journal-title":"Boston"},{"issue":"5","key":"10.1016\/j.infsof.2018.10.006_bib0009","doi-asserted-by":"crossref","first-page":"440","DOI":"10.1016\/j.infsof.2011.01.004","article-title":"Research synthesis in software engineering: a tertiary study","volume":"53","author":"Cruzes","year":"2011","journal-title":"Inf. Softw. Technol."},{"issue":"6","key":"10.1016\/j.infsof.2018.10.006_bib0010","doi-asserted-by":"crossref","first-page":"377","DOI":"10.1136\/jech.52.6.377","article-title":"The feasibility of creating a checklist for the assessment of the methodological quality both of randomised and non-randomised studies of health care interventions","volume":"52","author":"Downs","year":"1998","journal-title":"Journal of Epidemiology & Community Health"},{"issue":"8","key":"10.1016\/j.infsof.2018.10.006_bib0011","doi-asserted-by":"crossref","first-page":"e3081","DOI":"10.1371\/journal.pone.0003081","article-title":"\"Systematic review of the empirical evidence of study publication bias and outcome reporting bias","volume":"3","author":"Dwan","year":"2008","journal-title":"PLoS One"},{"key":"10.1016\/j.infsof.2018.10.006_bib0012","series-title":"Proceedings of the 22nd Int. Conf. on Software Engineering and Knowledge Engineering (SEKE)","first-page":"374","article-title":"Validity Threats in Empirical Software Engineering Research - An Initial Survey","author":"Feldt","year":"2010"},{"issue":"3","key":"10.1016\/j.infsof.2018.10.006_bib0013","doi-asserted-by":"crossref","first-page":"282","DOI":"10.1109\/TSE.2013.56","article-title":"Variability in software systems - A systematic literature review","volume":"40","author":"Galster","year":"2014","journal-title":"IEEE Trans. Softw. Eng."},{"key":"10.1016\/j.infsof.2018.10.006_bib0014","series-title":"Introduction to Action research: Social Research For Social Change","author":"Greenwood","year":"2007"},{"key":"10.1016\/j.infsof.2018.10.006_bib0015","series-title":"Proceedings of the 17th International Conference on Evaluation and Assessment in Software Engineering (EASE '13)","article-title":"A tertiary study: experiences of conducting systematic literature reviews in software engineering","author":"Imtiaz","year":"2013"},{"issue":"1","key":"10.1016\/j.infsof.2018.10.006_bib0016","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/0197-2456(95)00134-4","article-title":"Assessing the quality of reports of randomized clinical trials: is blinding necessary?","volume":"17","author":"Jadad","year":"1999","journal-title":"Controlled Clin. Trials"},{"key":"10.1016\/j.infsof.2018.10.006_bib0017","series-title":"Proceedings of the International Symposium on Empirical Software Engineering","first-page":"10","article-title":"Reporting guidelines for controlled experiments in software engineering","author":"Jedlitschka","year":"2005"},{"key":"10.1016\/j.infsof.2018.10.006_bib0018","series-title":"Guidelines For Performing Systematic Literature Reviews in Software Engineering","author":"Kitchenham","year":"2007"},{"issue":"8","key":"10.1016\/j.infsof.2018.10.006_bib0019","doi-asserted-by":"crossref","first-page":"792","DOI":"10.1016\/j.infsof.2010.03.006","article-title":"Systematic literature reviews in software engineering \u2013 A tertiary study","volume":"52","author":"Kitchenham","year":"2010","journal-title":"Inf. Softw. Technol."},{"issue":"1","key":"10.1016\/j.infsof.2018.10.006_bib0020","doi-asserted-by":"crossref","first-page":"7","DOI":"10.1016\/j.infsof.2008.09.009","article-title":"Systematic literature reviews in software engineering \u2013 a systematic literature review","volume":"51","author":"Kitchenham","year":"2009","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2018.10.006_bib0021","series-title":"Evaluation and Assessment in Software Engineering (EASE '10)","first-page":"1","article-title":"The value of mapping studies \u2013 A participant-observer case study","author":"Kitchenham","year":"2010"},{"key":"10.1016\/j.infsof.2018.10.006_bib0022","series-title":"Proceedings of the 26th International Conference on Software Engineering (ICSE '04)","first-page":"273","article-title":"Evidence-based software engineering","author":"Kitchenham","year":"2004"},{"key":"10.1016\/j.infsof.2018.10.006_bib0023","doi-asserted-by":"crossref","unstructured":"B. Kitchenham et\u00a0al. \"The impact of limited search procedures for systematic literature reviews: a participant-observer case study\", Proc. Third Int'l Symp. Empirical Software Eng. and Measurement, pp. 336\u2013345","DOI":"10.1109\/ESEM.2009.5314238"},{"issue":"12","key":"10.1016\/j.infsof.2018.10.006_bib0024","doi-asserted-by":"crossref","first-page":"2049","DOI":"10.1016\/j.infsof.2013.07.010","article-title":"A systematic review of systematic review process research in software engineering","volume":"55","author":"Kitchenham","year":"2013","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2018.10.006_bib0025","series-title":"Evidence Base Software Engineering and Systematic Reviews","author":"Kitchenham","year":"2015"},{"issue":"3","key":"10.1016\/j.infsof.2018.10.006_bib0026","doi-asserted-by":"crossref","first-page":"311","DOI":"10.1007\/s10664-005-1290-x","article-title":"Studying software engineers: data collection techniques for software field studies","volume":"10","author":"Lethbridge","year":"2005","journal-title":"Empirical Softw. Eng."},{"key":"10.1016\/j.infsof.2018.10.006_bib0027","series-title":"Naturalistic Inquiry","author":"Lincoln","year":"1985"},{"key":"10.1016\/j.infsof.2018.10.006_bib0028","series-title":"Global Software Engineering (ICGSE '12)","article-title":"Systematic literature reviews in distributed software development: a tertiary study","author":"Marques","year":"2012"},{"issue":"3","key":"10.1016\/j.infsof.2018.10.006_bib0029","doi-asserted-by":"crossref","first-page":"279","DOI":"10.17763\/haer.62.3.8323320856251826","article-title":"Understanding and validity in qualitative research","volume":"62","author":"Maxwell","year":"1992","journal-title":"Harvard educational review"},{"key":"10.1016\/j.infsof.2018.10.006_bib0030","article-title":"Taxonomy development in information systems: developing a taxonomy of mobile applications","author":"Nickerson","year":"2009","journal-title":"European Conference in Information Systems"},{"issue":"1","key":"10.1016\/j.infsof.2018.10.006_bib0031","doi-asserted-by":"crossref","DOI":"10.1186\/2046-4053-4-1","article-title":"Preferred reporting items for systematic review and meta-analysis protocols (PRISMA-P) 2015 statement","volume":"54","author":"Moher","year":"2015","journal-title":"Systematic Reviews"},{"key":"10.1016\/j.infsof.2018.10.006_bib0032","series-title":"Proceedings of the conference on the future of Software engineering","first-page":"345","article-title":"Empirical studies of software engineering: a roadmap","author":"Perry","year":"2000"},{"key":"10.1016\/j.infsof.2018.10.006_bib0033","series-title":"In the proceedings of Evaluation and Assessment in Software Engineering","first-page":"68","article-title":"Systematic mapping studies in software engineering","volume":"8","author":"Petersen","year":"2008"},{"key":"10.1016\/j.infsof.2018.10.006_bib0034","doi-asserted-by":"crossref","first-page":"1","DOI":"10.1016\/j.infsof.2015.03.007","article-title":"Guidelines for conducting systematic mapping studies in software engineering: an update","volume":"64","author":"Petersen","year":"2015","journal-title":"Information and Software Technology"},{"key":"10.1016\/j.infsof.2018.10.006_bib0035","series-title":"Proceedings of the Eighth International Conference on Software Process and Product Measurement (IWSM-MENSURA), 2013","first-page":"81","article-title":"Worldviews, research methods, and their relationship to validity in empirical software engineering research","author":"Petersen","year":"2013"},{"key":"10.1016\/j.infsof.2018.10.006_bib0036","series-title":"SIGSOFT Softw. Eng. Notes 26, 6 November 2001","article-title":"Principles of survey research: part 1: turning lemons into lemonade","author":"Pfleeger","year":"2001"},{"key":"10.1016\/j.infsof.2018.10.006_bib0037","doi-asserted-by":"crossref","first-page":"61","DOI":"10.1007\/s11292-007-9046-9","article-title":"Publication bias as a threat to the validity of meta-analytic results","author":"Rothstein","year":"2008","journal-title":"J. Exp. Criminol."},{"issue":"2","key":"10.1016\/j.infsof.2018.10.006_bib0038","doi-asserted-by":"crossref","first-page":"131","DOI":"10.1007\/s10664-008-9102-8","article-title":"Guidelines for conducting and reporting case study research in software engineering","volume":"14","author":"Runeson","year":"2009","journal-title":"Emp. Softw. Eng."},{"issue":"1","key":"10.1016\/j.infsof.2018.10.006_bib0039","doi-asserted-by":"crossref","first-page":"10","DOI":"10.1186\/1471-2288-7-10","article-title":"Development of AMSTAR: a measurement tool to assess the methodological quality of systematic reviews","volume":"7","author":"Shea","year":"2007","journal-title":"BMC Med. Res. Methodol."},{"key":"10.1016\/j.infsof.2018.10.006_bib0040","series-title":"37th IEEE International Conference on Software Engineering (ICSE)","first-page":"9","article-title":"Views on internal and external validity in empirical software engineering","author":"Siegmund","year":"2015"},{"issue":"9","key":"10.1016\/j.infsof.2018.10.006_bib0041","doi-asserted-by":"crossref","first-page":"899","DOI":"10.1016\/j.infsof.2011.04.004","article-title":"Six years of systematic literature reviews in software engineering: an updated tertiary study","volume":"53","author":"da Silva","year":"2011","journal-title":"Inf. Softw. Technol."},{"key":"10.1016\/j.infsof.2018.10.006_bib0042","series-title":"Proceedings of 2007 Future of Software Engineering","first-page":"358","article-title":"The future of empirical methods in software engineering research","author":"Sjoberg","year":"2007"},{"issue":"2","key":"10.1016\/j.infsof.2018.10.006_bib0043","doi-asserted-by":"crossref","first-page":"201","DOI":"10.1353\/etc.2012.0009","article-title":"\"Evaluating the validity of systematic reviews to indentify empirically supported treatments","volume":"35","author":"Slocum","year":"2012","journal-title":"Educ. Treat. Child."},{"key":"10.1016\/j.infsof.2018.10.006_bib0044","first-page":"163","article-title":"Empirical methods in software engineering research","author":"Tichy","year":"2007","journal-title":"ICSE Companion"},{"issue":"12","key":"10.1016\/j.infsof.2018.10.006_bib0045","doi-asserted-by":"crossref","first-page":"1235","DOI":"10.1016\/S0895-4356(98)00131-0","article-title":"The Delphi list: a criteria list for quality assessment of randomized clinical trials for conducting systematic reviews developed by Delphi consensus","volume":"51","author":"Verhagen","year":"1998","journal-title":"J. Clin. Epidemiol."},{"key":"10.1016\/j.infsof.2018.10.006_bib0046","series-title":"Evaluation and Assessment in Software Engineering (EASE '12)","article-title":"Systematic literature reviews in global software development: a tertiary study","author":"Verner","year":"2012"},{"key":"10.1016\/j.infsof.2018.10.006_bib0047","doi-asserted-by":"crossref","unstructured":"C. Wohlin, M. Host, P. Runeson, M. Ohlsson, B. Regnell, and A. Wesslen, \u201cExperimentation in software engineering: an introduction\u201d, Kluwer Academic Publishers, 2000","DOI":"10.1007\/978-1-4615-4625-2"},{"issue":"10","key":"10.1016\/j.infsof.2018.10.006_bib0048","doi-asserted-by":"crossref","first-page":"2594","DOI":"10.1016\/j.jss.2013.04.076","article-title":"On the reliability of mapping studies in software engineering","volume":"86","author":"Wohlin","year":"2013","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.infsof.2018.10.006_bib0049","series-title":"Proceedings of the 18th International Conference on Evaluation and Assessment in Software Engineering (EASE '14)","article-title":"Guidelines for snowballing in systematic literature studies and a replication in software engineering","author":"Wohlin","year":"2014"},{"issue":"1","key":"10.1016\/j.infsof.2018.10.006_bib0050","doi-asserted-by":"crossref","first-page":"162","DOI":"10.1016\/j.jss.2010.09.036","article-title":"An assessment of systems and software engineering scholars and institutions (2003\u20132007 and 2004\u20132008)","volume":"84","author":"Wong","year":"2011","journal-title":"J. Syst. Softw."},{"key":"10.1016\/j.infsof.2018.10.006_bib0051","series-title":"Case Study research: Design and Methods","author":"Yin","year":"2009"},{"issue":"1","key":"10.1016\/j.infsof.2018.10.006_bib0052","doi-asserted-by":"crossref","first-page":"44","DOI":"10.1016\/S0749-3797(99)00122-1","article-title":"Data collection instrument and procedure for systematic reviews in the Guide to Community Preventive Services","volume":"18","author":"Zaza","year":"2000","journal-title":"Am. J. Prevent. Med."},{"key":"10.1016\/j.infsof.2018.10.006_bib0053","series-title":"23rd Asia-Pacific Software Engineering Conference (APSEC)","first-page":"153","article-title":"A map of threats to validity of systematic literature reviews in software engineering","author":"Zhou","year":"2016"}],"container-title":["Information and Software Technology"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/api.elsevier.com\/content\/article\/PII:S0950584918302106?httpAccept=text\/xml","content-type":"text\/xml","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/api.elsevier.com\/content\/article\/PII:S0950584918302106?httpAccept=text\/plain","content-type":"text\/plain","content-version":"vor","intended-application":"text-mining"}],"deposited":{"date-parts":[[2025,10,16]],"date-time":"2025-10-16T03:27:14Z","timestamp":1760585234000},"score":1,"resource":{"primary":{"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/linkinghub.elsevier.com\/retrieve\/pii\/S0950584918302106"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2019,2]]},"references-count":53,"alternative-id":["S0950584918302106"],"URL":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.1016\/j.infsof.2018.10.006","relation":{},"ISSN":["0950-5849"],"issn-type":[{"value":"0950-5849","type":"print"}],"subject":[],"published":{"date-parts":[[2019,2]]},"assertion":[{"value":"Elsevier","name":"publisher","label":"This article is maintained by"},{"value":"Identifying, categorizing and mitigating threats to validity in software engineering secondary studies","name":"articletitle","label":"Article Title"},{"value":"Information and Software Technology","name":"journaltitle","label":"Journal Title"},{"value":"https:\/\/2.zoppoz.workers.dev:443\/https\/doi.org\/10.1016\/j.infsof.2018.10.006","name":"articlelink","label":"CrossRef DOI link to publisher maintained version"},{"value":"article","name":"content_type","label":"Content Type"},{"value":"\u00a9 2018 Elsevier B.V. All rights reserved.","name":"copyright","label":"Copyright"}]}}