{"id":"https://openalex.org/W3093896776","doi":"https://doi.org/10.1162/coli_r_00388","title":"<i>Statistical Significance Testing for Natural Language Processing</i>. By Rotem Dror, Lotem Peled-Cohen, Segev Shlomov, and Roi Reichart (Technion Israel Institute of Technology)). Morgan &amp; Claypool (Synthesis Lectures on Human Language Technologies, edited by Graeme Hirst, volume 45), 2020, xx+98 pp; paperback, ISBN 978-1-68173-795-9, $49.95; ebook, ISBN 978-1-68173-796-6, $39.96; hardcover, $69.95; doi:10.2200/S00994ED1V01Y202002HLT045","display_name":"<i>Statistical Significance Testing for Natural Language Processing</i>. By Rotem Dror, Lotem Peled-Cohen, Segev Shlomov, and Roi Reichart (Technion Israel Institute of Technology)). Morgan &amp; Claypool (Synthesis Lectures on Human Language Technologies, edited by Graeme Hirst, volume 45), 2020, xx+98 pp; paperback, ISBN 978-1-68173-795-9, $49.95; ebook, ISBN 978-1-68173-796-6, $39.96; hardcover, $69.95; doi:10.2200/S00994ED1V01Y202002HLT045","publication_year":2020,"publication_date":"2020-10-20","ids":{"openalex":"https://openalex.org/W3093896776","doi":"https://doi.org/10.1162/coli_r_00388","mag":"3093896776"},"language":"en","primary_location":{"id":"doi:10.1162/coli_r_00388","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_r_00388","pdf_url":"https://direct.mit.edu/coli/article-pdf/46/4/905/1888268/coli_r_00388.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"type":"book-review","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://direct.mit.edu/coli/article-pdf/46/4/905/1888268/coli_r_00388.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061992028","display_name":"Edwin Simpson","orcid":"https://orcid.org/0000-0002-6447-1552"},"institutions":[{"id":"https://openalex.org/I137377929","display_name":"East Sussex County Council","ror":"https://ror.org/0236s9n59","country_code":"GB","type":"government","lineage":["https://openalex.org/I137377929"]},{"id":"https://openalex.org/I36234482","display_name":"University of Bristol","ror":"https://ror.org/0524sp257","country_code":"GB","type":"education","lineage":["https://openalex.org/I36234482"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Edwin D. Simpson","raw_affiliation_strings":["University of Bristol","Edwin D. Simpson is a Lecturer in the Department of\n                            Computer Science, University of Bristol, UK. His research focuses on\n                            natural language processing (NLP), with particular interests in applying\n                            interactive machine learning and Bayesian techniques to NLP problems\n                            such as argumentation, crowdsourced annotation and text ranking. His\n                            e-mail address is "],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Bristol","institution_ids":["https://openalex.org/I36234482"]},{"raw_affiliation_string":"Edwin D. Simpson is a Lecturer in the Department of\n                            Computer Science, University of Bristol, UK. His research focuses on\n                            natural language processing (NLP), with particular interests in applying\n                            interactive machine learning and Bayesian techniques to NLP problems\n                            such as argumentation, crowdsourced annotation and text ranking. His\n                            e-mail address is ","institution_ids":["https://openalex.org/I137377929"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5061992028"],"corresponding_institution_ids":["https://openalex.org/I137377929","https://openalex.org/I36234482"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"46","issue":"4","first_page":"905","last_page":"908"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9969000220298767,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11719","display_name":"Data Quality and Management","score":0.9937999844551086,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T13083","display_name":"Advanced Text Analysis Techniques","score":0.9922000169754028,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5553402900695801},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45310020446777344},{"id":"https://openalex.org/keywords/natural-language-processing","display_name":"Natural language processing","score":0.35558784008026123}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5553402900695801},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45310020446777344},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.35558784008026123}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1162/coli_r_00388","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_r_00388","pdf_url":"https://direct.mit.edu/coli/article-pdf/46/4/905/1888268/coli_r_00388.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:8e252141f0284cd4976165c0c9308ecd","is_oa":false,"landing_page_url":"https://doaj.org/article/8e252141f0284cd4976165c0c9308ecd","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Computational Linguistics, Vol 46, Iss 4, Pp 905-908 (2021)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1162/coli_r_00388","is_oa":true,"landing_page_url":"https://doi.org/10.1162/coli_r_00388","pdf_url":"https://direct.mit.edu/coli/article-pdf/46/4/905/1888268/coli_r_00388.pdf","source":{"id":"https://openalex.org/S155526855","display_name":"Computational Linguistics","issn_l":"0891-2017","issn":["0891-2017","1530-9312"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310320244","host_organization_name":"Association for Computational Linguistics","host_organization_lineage":["https://openalex.org/P4310320244"],"host_organization_lineage_names":["Association for Computational Linguistics"],"type":"journal"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Computational Linguistics","raw_type":"journal-article"},"sustainable_development_goals":[{"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4","score":0.7799999713897705}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3093896776.pdf","grobid_xml":"https://content.openalex.org/works/W3093896776.grobid-xml"},"referenced_works_count":10,"referenced_works":["https://openalex.org/W1607035479","https://openalex.org/W2059676361","https://openalex.org/W2121044470","https://openalex.org/W2144600658","https://openalex.org/W2292427613","https://openalex.org/W2794509261","https://openalex.org/W2924950986","https://openalex.org/W2953183526","https://openalex.org/W4254419114","https://openalex.org/W4256170606"],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W2358668433","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W2382290278","https://openalex.org/W2350741829","https://openalex.org/W2130043461","https://openalex.org/W3192589309"],"abstract_inverted_index":{"Like":[0],"any":[1,609,1379],"other":[2],"science,":[3],"research":[4,210,500,519,883,1289,1417],"in":[5,197,298,331,556,581,650,701,745,834,850,856,1063,1265,1364],"natural":[6],"language":[7],"processing":[8],"(NLP)":[9],"depends":[10],"on":[11,185,219,436,527,930,1164,1188,1228],"the":[12,99,146,157,180,198,208,220,229,234,236,246,255,262,269,274,293,299,305,325,333,384,397,410,439,443,461,485,502,531,560,570,663,676,687,694,704,710,723,728,748,760,820,830,851,881,909,931,942,956,961,972,1016,1033,1036,1046,1058,1071,1099,1134,1143,1177,1195,1209,1229,1235,1240,1251,1255,1261,1268,1284,1301,1344,1365,1370,1394,1402],"ability":[13,368],"to":[14,32,67,70,105,124,155,167,286,324,343,366,369,376,391,446,473,483,488,507,529,607,616,671,689,708,758,785,815,865,888,893,948,1056,1129,1141,1193,1207,1214,1242,1253,1313,1323,1328,1343,1353,1372,1429],"draw":[15,913],"correct":[16,398],"conclusions":[17,910],"from":[18,626,914,1320],"experiments.":[19],"A":[20,1112,1121],"key":[21],"tool":[22],"for":[23,178,215,409,450,512,562,730,802,808,848,854,966,1048,1092,1160,1175,1425],"this":[24,79,232,536,548,618,628,827,984,1150,1311,1382,1419],"is":[25,233,264,335,401,455,481,535,707,743,756,844,928,1139,1337,1387,1411],"statistical":[26,290,613,739,890,1303],"significance":[27,68,87,150,169,186,195,213,291,312,437,505,614,645,788,842,868,891,1037,1094,1126,1347,1398,1409],"testing:":[28],"We":[29],"use":[30,102,1057,1208,1354],"it":[31,153,480,1062,1367,1386,1389,1431],"judge":[33],"whether":[34],"a":[35,46,107,111,176,239,259,279,288,363,371,393,402,429,433,456,489,563,732,753,763,766,786,791,803,841,862,904,915,920,967,996,1010,1049,1084,1157,1185,1223,1314,1351,1422],"result":[36,109,263,334,373],"provides":[37,182,721,1222,1421],"meaningful,":[38],"generalizable":[39],"findings":[40],"or":[41,89,770,773,1018,1181,1400],"should":[42],"be":[43,287,296,317,389,470,946],"taken":[44],"with":[45,194,226,282,432,460,662,870,1361],"pinch":[47],"of":[48,149,160,200,212,222,231,272,276,309,327,362,383,425,458,504,538,612,699,727,747,826,832,885,906,919,958,974,1012,1021,1035,1086,1101,1136,1146,1197,1225,1291,1305,1317,1346,1397,1415],"salt.":[49],"When":[50],"comparing":[51,223],"new":[52,189,475,1158,1321],"methods":[53,190,1333],"against":[54],"others,":[55],"performance":[56,1022],"metrics":[57,750],"often":[58,81,1081],"differ":[59],"by":[60,162,252,954,1074],"only":[61],"small":[62],"amounts,":[63],"so":[64,550],"researchers":[65,119,144,1426],"turn":[66],"tests":[69,88,196,349,426,449,506,615,646,700,1127],"show":[71,1205],"that":[72,126,191,238,341,350,395,427,468,544,569,680,794,1283,1330,1388],"improved":[73],"models":[74,1079],"are":[75,129,422,635,648,811,838,1080,1179],"genuinely":[76],"better.":[77],"Unfortunately,":[78],"reasoning":[80],"fails":[82],"because":[83],"we":[84,101,248,911],"choose":[85,392],"inappropriate":[86],"carry":[90,447],"them":[91],"out":[92,448,717,814],"incorrectly,":[93],"making":[94,152],"their":[95,127,718,987],"outcomes":[96,1266],"meaningless.":[97],"Or,":[98],"test":[100,114,161,315,364,394,682,714,799,917],"may":[103],"fail":[104],"indicate":[106],"significant":[108,372,1137,1147],"when":[110,1176,1297,1355],"more":[112,498,574,672,817,950],"appropriate":[113,311,821],"would":[115],"find":[116,484],"one.":[117],"NLP":[118,143,165,413,452,491,702,736,782,1078,1244,1318,1416],"must":[120,295,316,388],"avoid":[121,133,377],"these":[122,521,1218],"pitfalls":[123],"ensure":[125,866,1329],"evaluations":[128],"sound":[130],"and":[131,136,203,206,320,374,386,406,476,511,541,552,566,593,598,600,638,642,656,678,715,738,903,937,978,992,1068,1132,1168,1173,1190,1234,1247,1271,1280,1300,1349,1418,1433],"ultimately":[132],"wasting":[134],"time":[135],"money":[137],"through":[138,145],"incorrect":[139,352],"conclusions.This":[140],"book":[141,181,217,303,440,549,729,1026,1312,1336,1383,1420],"guides":[142],"whole":[147],"process":[148,944,1241],"testing,":[151,187,438,590,1167],"easy":[154,482],"select":[156],"right":[158,300],"kind":[159],"matching":[163],"canonical":[164],"tasks":[166,414,737],"specific":[168,451,787,804,1243],"testing":[170,214,513,869,892,1076,1348,1399,1410],"procedures.":[171],"As":[172],"well":[173],"as":[174,243,245,355,630,633,1002,1015,1045,1149,1295,1339,1350],"being":[175],"handbook":[177],"researchers,":[179,1319],"theoretical":[183,577],"background":[184,435,539,578,1227],"includes":[188],"solve":[192],"problems":[193,211,1279,1296],"world":[199],"deep":[201,508,894,921],"learning":[202],"multidataset":[204],"benchmarks,":[205],"describes":[207,304,829],"open":[209,518,1278],"NLP.The":[216],"focuses":[218],"task":[221,742,810],"one":[224,247,1200,1376],"algorithm":[225,1111,1114],"another.":[227],"At":[228],"core":[230,586],"p-value,":[235],"probability":[237],"difference":[240],"at":[241,1381,1384],"least":[242],"extreme":[244],"observed":[249],"could":[250,623,1377],"occur":[251],"chance.":[253],"If":[254,1375],"p-value":[256,294],"falls":[257],"below":[258],"predetermined":[260],"threshold,":[261,285],"declared":[265,337],"significant.":[266,338],"Leaving":[267],"aside":[268],"fundamental":[270],"limitation":[271],"turning":[273],"validity":[275,405,597],"results":[277,1138,1259],"into":[278,501,654],"binary":[280],"question":[281,884],"an":[283,310,1030,1064,1341,1412],"arbitrary":[284],"valid":[289,319],"test,":[292,768,772,789],"computed":[297],"way.":[301],"The":[302,314,360,493,576,602,659,823,859,1039,1052,1153,1335],"two":[306,495,1103,1170,1275],"crucial":[307],"properties":[308],"test:":[313],"both":[318,474,1340],"powerful.":[321],"Validity":[322],"refers":[323,365],"avoidance":[326],"type":[328,344,378,591,594],"1":[329,345,558,592],"errors,":[330,596],"which":[332,584,798,835,927,1088,1118,1216],"incorrectly":[336],"Common":[339],"mistakes":[340],"lead":[342],"errors":[346,1072],"include":[347],"deploying":[348],"make":[351],"assumptions,":[353],"such":[354,632,1001,1014,1029,1294],"independence":[356],"between":[357,404,734,765,1267],"data":[358,385,852],"points.":[359],"power":[361,991,1304],"its":[367,1292],"detect":[370],"therefore":[375],"2":[379,595],"errors.":[380],"Here,":[381,479],"knowledge":[382,611],"experiment":[387],"used":[390],"makes":[396,547],"assumptions.":[399],"There":[400,454],"trade-off":[403],"power,":[407,599],"but":[408,986],"most":[411,724],"common":[412,735],"(language":[415],"modeling,":[416],"sequence":[417],"labeling,":[418],"translation,":[419],"etc.),":[420],"there":[421],"clear":[423,525],"choices":[424],"provide":[428,524,816,949,995],"good":[430],"balance.Beginning":[431],"detailed":[434,1226],"then":[441,668,752,779,963,970,1155,1204],"shows":[442,1027],"reader":[444,603,688,761,1252],"how":[445,528,886,941,1028,1055,1107,1206],"tasks.":[453],"mix":[457,537],"styles,":[459],"first":[462],"four":[463],"chapters":[464,496,523,572,1276],"providing":[465],"reference":[466,542,1352],"material":[467,486,540,579],"will":[469],"extremely":[471],"useful":[472],"experienced":[477,621],"researchers.":[478],"related":[487],"given":[490],"task.":[492,805],"next":[494],"discuss":[497,1169,1393],"recent":[499],"application":[503],"neural":[509,922],"networks":[510],"across":[514,1023,1083],"multiple":[515],"datasets.":[516],"Alongside":[517],"questions,":[520],"later":[522,571,1236],"guidelines":[526,543],"apply":[530,889,1254],"proposed":[532,1059,1230,1269],"methods.":[533,976],"It":[534],"I":[545,1308],"believe":[546],"compelling":[551],"nicely":[553],"self-contained.The":[554],"introduction":[555,1342],"Chapter":[557,582,651,873,878,1096,1220],"motivates":[559],"need":[561,606,801],"comprehensive":[564],"textbook":[565],"outlines":[567],"challenges":[568,1100],"address":[573],"deeply.":[575],"begins":[580,661],"2,":[583],"introduces":[585,880,964],"concepts,":[587],"including":[588],"hypothesis":[589],"p-values.":[601],"does":[604,696,993,1110,1120,1390],"not":[605,994,1391],"have":[608,981],"prior":[610],"follow":[617],"part.":[619],"However,":[620],"readers":[622,796],"still":[624],"benefit":[625],"reading":[627],"chapter,":[629,926],"concepts":[631],"p-values":[634],"widely":[636],"misunderstood":[637],"misused":[639],"(Amrhein,":[640],"Greenland,":[641],"McShane":[643],"2019).The":[644],"themselves":[647],"introduced":[649,757],"3,":[652],"categorized":[653],"parametric":[655,767],"nonparametric":[657,775],"tests.":[658,740,1038],"chapter":[660,695,828,860,962,1053,1151],"intuitively":[664],"simple":[665,863],"paired":[666],"z-test,":[667],"builds":[669],"up":[670],"commonly-applied":[673],"techniques,":[674],"showing":[675,1282],"connections":[677],"assumptions":[679,1034],"each":[681,691,713,781,809,1075,1130],"makes.":[683],"Step-by-step":[684],"algorithms":[685],"help":[686,1250],"implement":[690],"test.":[692,776],"Although":[693],"cite":[697],"uses":[698],"research,":[703],"main":[705],"purpose":[706],"present":[709,1156,1277],"theory":[711],"behind":[712],"point":[716],"differences.Chapter":[719],"4":[720,874],"perhaps":[722],"handy":[725],"part":[726,825,1414],"reference:":[731],"correspondence":[733],"Each":[741],"discussed":[744],"terms":[746],"evaluation":[749,783],"used,":[751],"decision":[754],"tree":[755],"guide":[759],"toward":[762],"choice":[764],"bootstrap":[769],"randomization":[771],"sampling-free":[774],"Section":[777],"4.3":[778],"links":[780],"measure":[784],"presenting":[790],"large":[792,871],"table":[793],"helps":[795],"identify":[797,1215],"they":[800,1331],"Particular":[806],"considerations":[807],"also":[812,1368],"pointed":[813],"detail":[818],"about":[819],"options.":[822],"final":[824,1274],"issue":[831],"p-hacking,":[833],"dataset":[836,1131],"sizes":[837],"increased":[839],"until":[840],"threshold":[843],"reached":[845],"without":[846],"consideration":[847],"biases":[849],"(discussed,":[853],"example,":[855],"Hofmann":[857],"[2015]).":[858],"proposes":[861],"solution":[864],"robust":[867],"datasets.Where":[872],"presents":[875,1089],"well-established":[876],"methods,":[877],"5":[879],"current":[882,975],"best":[887],"learning.":[895],"Non-convex":[896],"loss":[897],"functions,":[898],"stochastic":[899,1043],"optimization,":[900],"random":[901],"initialization,":[902],"multitude":[905],"hyperparameters":[907],"limit":[908],"can":[912,945],"single":[916],"run":[918],"network":[923],"(DNN).":[924],"This":[925,1025],"based":[929,1163,1187],"authors\u2019":[932],"ACL":[933],"paper":[934],"(Dror,":[935],"Shlomov,":[936],"Reichart":[938],"2019),":[939],"explains":[940,1054],"comparison":[943,968],"overhauled":[947],"meaningful":[951],"evaluations.":[952],"Beginning":[953],"explaining":[955],"difficulties":[957],"evaluating":[959,1356],"DNNs,":[960],"criteria":[965],"framework,":[969,1233],"discusses":[971,1098],"limitations":[973],"Reimers":[977],"Gurevych":[979],"(2018)":[980],"previously":[982],"tackled":[983],"problem,":[985],"approach":[988,1031,1270],"has":[989,1286],"limited":[990,1302],"confidence":[997],"score.":[998],"Other":[999],"works,":[1000],"Clark":[1003],"et":[1004],"al.":[1005],"(2011),":[1006],"compare":[1007,1332],"DNNs":[1008],"using":[1009],"collection":[1011],"statistics,":[1013],"mean":[1017],"standard":[1019,1093,1125,1272],"deviation":[1020],"runs.":[1024],"violates":[1032],"authors":[1040,1154],"propose":[1041],"almost":[1042],"dominance":[1044],"basis":[1047],"better":[1050],"alternative.":[1051],"method,":[1060],"evaluates":[1061],"empirical":[1065,1258],"case":[1066,1245],"study,":[1067],"finally":[1069],"analyzes":[1070],"made":[1073],"approach.Large":[1077],"tested":[1082],"range":[1085,1316],"datasets,":[1087],"another":[1090],"problem":[1091],"testing.":[1095],"6":[1097,1221],"assessing":[1102],"questions:":[1104],"(1)":[1105],"On":[1106,1117],"many":[1108,1287],"datasets":[1109,1119,1178,1198,1217],"outperform":[1113,1122],"B?":[1115,1123],"(2)":[1116],"Applying":[1124],"individually":[1128],"counting":[1133],"number":[1135,1145,1196],"likely":[1140],"overestimate":[1142],"total":[1144],"results,":[1148],"explains.":[1152],"framework":[1159],"replicability":[1161,1231,1306],"analysis,":[1162],"partial":[1165],"conjunction":[1166],"variants":[1171],"(Bonferroni":[1172],"Fisher)":[1174],"independent":[1180],"dependent.":[1182],"They":[1183],"introduce":[1184],"method":[1186,1201],"Benjamini":[1189],"Heller":[1191],"(2008)":[1192],"count":[1194],"where":[1199],"outperforms":[1202],"another,":[1203],"Holm":[1210],"procedure":[1211],"(Holm":[1212],"1979)":[1213],"are.":[1219],"lot":[1224],"analysis":[1232],"sections":[1237],"again":[1238],"link":[1239],"studies,":[1246],"step-by-step":[1248],"summaries":[1249],"methodology.":[1256],"Extensive":[1257],"illustrate":[1260],"very":[1262],"substantial":[1263],"differences":[1264],"procedures.The":[1273],"conclude,":[1281],"topic":[1285,1345],"interesting":[1288],"questions":[1290],"own,":[1293],"performing":[1298],"cross-validation,":[1299],"analysis.Overall,":[1307],"highly":[1309],"recommend":[1310],"wide":[1315],"students":[1322],"seasoned":[1324],"experts":[1325],"who":[1326,1427],"wish":[1327,1428],"effectively.":[1334],"excellent":[1338],"your":[1357],"results.":[1358],"For":[1359,1406],"anyone":[1360],"further":[1362],"interest":[1363],"topic,":[1366],"points":[1369],"way":[1371],"future":[1373],"work.":[1374],"level":[1378],"criticism":[1380],"all,":[1385],"deeply":[1392],"basic":[1395],"flaws":[1396],"what":[1401],"alternatives":[1403],"might":[1404],"be.":[1405],"now,":[1407],"though,":[1408],"integral":[1413],"great":[1423],"resource":[1424],"perform":[1430],"correctly":[1432],"painlessly.":[1434]},"counts_by_year":[],"updated_date":"2026-07-14T23:27:15.235271","created_date":"2020-10-29T00:00:00"}