{"id":"https://openalex.org/W2114226921","doi":"https://doi.org/10.1109/hotos.1999.798385","title":"The case for application-specific benchmarking","display_name":"The case for application-specific benchmarking","publication_year":2003,"publication_date":"2003-01-20","ids":{"openalex":"https://openalex.org/W2114226921","doi":"https://doi.org/10.1109/hotos.1999.798385","mag":"2114226921"},"language":"en","primary_location":{"id":"doi:10.1109/hotos.1999.798385","is_oa":false,"landing_page_url":"https://doi.org/10.1109/hotos.1999.798385","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Seventh Workshop on Hot Topics in Operating Systems","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5054751297","display_name":"Margo Seltzer","orcid":"https://orcid.org/0000-0002-2165-4658"},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"M. Seltzer","raw_affiliation_strings":["Harvard University, USA","Harvard University , MA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University, USA","institution_ids":["https://openalex.org/I2801851002"]},{"raw_affiliation_string":"Harvard University , MA, USA","institution_ids":["https://openalex.org/I2801851002"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5073506848","display_name":"D. Krinsky","orcid":null},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"D. Krinsky","raw_affiliation_strings":["Harvard University, USA","Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University, USA","institution_ids":["https://openalex.org/I2801851002"]},{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I2801851002"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5109479101","display_name":"Keith Smith","orcid":null},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"K. Smith","raw_affiliation_strings":["Harvard University, USA","Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University, USA","institution_ids":["https://openalex.org/I2801851002"]},{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I2801851002"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100759721","display_name":"Xiaolan Zhang","orcid":"https://orcid.org/0000-0002-6343-3116"},"institutions":[{"id":"https://openalex.org/I2801851002","display_name":"Harvard University Press","ror":"https://ror.org/006v7bf86","country_code":"US","type":"other","lineage":["https://openalex.org/I136199984","https://openalex.org/I2801851002"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xiaolan Zhang","raw_affiliation_strings":["Harvard University, USA","Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University, USA","institution_ids":["https://openalex.org/I2801851002"]},{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I2801851002"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I2801851002"],"apc_list":null,"apc_paid":null,"fwci":2.7978,"has_fulltext":false,"cited_by_count":101,"citation_normalized_percentile":{"value":0.92243332,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"102","last_page":"107"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12127","display_name":"Software System Performance and Reliability","score":0.9993000030517578,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10101","display_name":"Cloud Computing and Resource Management","score":0.9320999979972839,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.9103999733924866,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.968454122543335},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.8430168032646179},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.8305961489677429},{"id":"https://openalex.org/keywords/trace","display_name":"TRACE (psycholinguistics)","score":0.6838287115097046},{"id":"https://openalex.org/keywords/spec#","display_name":"Spec#","score":0.5424706339836121},{"id":"https://openalex.org/keywords/range","display_name":"Range (aeronautics)","score":0.5237526893615723},{"id":"https://openalex.org/keywords/software","display_name":"Software","score":0.5110419988632202},{"id":"https://openalex.org/keywords/software-engineering","display_name":"Software engineering","score":0.37138062715530396},{"id":"https://openalex.org/keywords/programming-language","display_name":"Programming language","score":0.20369544625282288}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.968454122543335},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.8430168032646179},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.8305961489677429},{"id":"https://openalex.org/C75291252","wikidata":"https://www.wikidata.org/wiki/Q1315756","display_name":"TRACE (psycholinguistics)","level":2,"score":0.6838287115097046},{"id":"https://openalex.org/C2778565505","wikidata":"https://www.wikidata.org/wiki/Q2207566","display_name":"Spec#","level":2,"score":0.5424706339836121},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.5237526893615723},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.5110419988632202},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.37138062715530396},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.20369544625282288},{"id":"https://openalex.org/C159985019","wikidata":"https://www.wikidata.org/wiki/Q181790","display_name":"Composite material","level":1,"score":0.0},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.0},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C162853370","wikidata":"https://www.wikidata.org/wiki/Q39809","display_name":"Marketing","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C192562407","wikidata":"https://www.wikidata.org/wiki/Q228736","display_name":"Materials science","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":5,"locations":[{"id":"doi:10.1109/hotos.1999.798385","is_oa":false,"landing_page_url":"https://doi.org/10.1109/hotos.1999.798385","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Seventh Workshop on Hot Topics in Operating Systems","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.133.7989","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.133.7989","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.cse.nd.edu/~cpoellab/teaching/cse598a/appbench.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.138.3577","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.138.3577","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://eecs.harvard.edu/~margo/papers/hotos99-2/paper.pdf","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.45.9439","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.45.9439","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.eecs.harvard.edu/~keith/papers/hotos99.ps.gz","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.71.2349","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.71.2349","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.eecs.harvard.edu/~keith/papers/hotos99.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W200780396","https://openalex.org/W1571812940","https://openalex.org/W1577928699","https://openalex.org/W1823414211","https://openalex.org/W1887863533","https://openalex.org/W1937459776","https://openalex.org/W2005373714","https://openalex.org/W2019436626","https://openalex.org/W2031771926","https://openalex.org/W2050634022","https://openalex.org/W2058519873","https://openalex.org/W2062512647","https://openalex.org/W2105321788","https://openalex.org/W2127599594","https://openalex.org/W4315751360","https://openalex.org/W6608222036","https://openalex.org/W6634338661","https://openalex.org/W6639487200","https://openalex.org/W6675903601","https://openalex.org/W6678975077"],"related_works":["https://openalex.org/W2142488884","https://openalex.org/W1838930658","https://openalex.org/W2373662841","https://openalex.org/W4252566889","https://openalex.org/W3082894236","https://openalex.org/W2066278930","https://openalex.org/W2378459399","https://openalex.org/W2098005539","https://openalex.org/W2011488966","https://openalex.org/W2107668421"],"abstract_inverted_index":{"Most":[0],"performance":[1,59],"analysis":[2],"today":[3],"uses":[4],"either":[5],"microbenchmarks":[6],"or":[7,75],"standard":[8],"macrobenchmarks":[9],"(e.g.":[10],"SPEC,":[11],"LADDIS,":[12],"the":[13,17,63,92,113],"Andrew":[14],"benchmark).":[15],"However,":[16],"results":[18,38],"of":[19,66,73],"such":[20,109],"benchmarks":[21],"provide":[22],"little":[23],"information":[24],"to":[25,56,83],"indicate":[26],"how":[27],"well":[28],"a":[29,34,67,71,103],"particular":[30,35,68],"system":[31,94],"will":[32],"handle":[33],"application.":[36],"Such":[37],"are,":[39],"at":[40,44],"best,":[41],"useless":[42],"and,":[43],"worst,":[45],"misleading.":[46],"In":[47],"this":[48],"paper":[49],"we":[50],"argue":[51,107],"for":[52],"an":[53,96],"application-directed":[54],"approach":[55],"benchmarking,":[57],"using":[58,87,99],"metrics":[60],"that":[61,89,108],"reflect":[62],"expected":[64],"behavior":[65],"application":[69],"across":[70],"range":[72],"hardware":[74],"software":[76],"platforms.":[77],"We":[78,106],"present":[79],"three":[80],"different":[81],"approaches":[82],"application-specific":[84],"measurement,":[85],"one":[86,98],"vectors":[88],"characterize":[90],"both":[91],"underlying":[93],"and":[95,102],"application,":[97],"trace-driven":[100],"techniques,":[101],"hybrid":[104],"approach.":[105],"techniques":[110],"should":[111],"become":[112],"new":[114],"standard.":[115]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":3},{"year":2021,"cited_by_count":2},{"year":2020,"cited_by_count":2},{"year":2019,"cited_by_count":4},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":3},{"year":2015,"cited_by_count":6},{"year":2014,"cited_by_count":12},{"year":2013,"cited_by_count":6},{"year":2012,"cited_by_count":4}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}