{"id":"https://openalex.org/W7162487684","doi":"https://doi.org/10.48550/arxiv.2605.27355","title":"Alignment Tampering: How Reinforcement Learning from Human Feedback Is Exploited to Optimize Misaligned Biases","display_name":"Alignment Tampering: How Reinforcement Learning from Human Feedback Is Exploited to Optimize Misaligned Biases","publication_year":2026,"publication_date":"2026-05-26","ids":{"openalex":"https://openalex.org/W7162487684","doi":"https://doi.org/10.48550/arxiv.2605.27355"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.27355","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27355","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.27355","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137095957","display_name":"Dongyoon Hahm","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hahm, Dongyoon","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137121033","display_name":"Dylan Hadfield-Menell","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hadfield-Menell, Dylan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137093010","display_name":"Kimin Lee","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lee, Kimin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.30230000615119934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.30230000615119934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10664","display_name":"Sentiment Analysis and Opinion Mining","score":0.10490000247955322,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0731000006198883,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7074999809265137},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.5985000133514404},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5756999850273132},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.5465999841690063},{"id":"https://openalex.org/keywords/vulnerability","display_name":"Vulnerability (computing)","score":0.47769999504089355},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.3472999930381775}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7871000170707703},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7074999809265137},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.5985000133514404},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5756999850273132},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5633000135421753},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5465999841690063},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5034999847412109},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.47769999504089355},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.3472999930381775},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.3463999927043915},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3370000123977661},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.3133000135421753},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.27459999918937683},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.2705000042915344}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.27355","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27355","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.27355","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.27355","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/5","display_name":"Gender equality","score":0.5305537581443787}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Reinforcement":[0],"Learning":[1],"from":[2,46,56,112,141],"Human":[3],"Feedback":[4],"(RLHF)":[5],"is":[6,74],"the":[7,30,35,57,115,182],"standard":[8],"method":[9],"to":[10,40,63,83,144,163,184],"align":[11],"Large":[12],"Language":[13],"Models":[14],"(LLMs)":[15],"with":[16,95],"human":[17],"preferences.":[18],"In":[19],"this":[20,119,186],"work,":[21],"we":[22],"introduce":[23],"alignment":[24,33,85,166],"tampering,":[25],"a":[26],"potential":[27],"vulnerability":[28],"where":[29],"LLM":[31,91],"undergoing":[32],"influences":[34],"preference":[36,52,106],"dataset,":[37],"causing":[38],"RLHF":[39,161,179],"amplify":[41,131],"undesired":[42],"behaviors.":[43],"This":[44],"arises":[45],"core":[47],"limitations":[48,79],"of":[49,177],"RLHF:":[50],"(1)":[51],"datasets":[53],"are":[54],"constructed":[55],"LLM's":[58],"own":[59],"outputs,":[60],"allowing":[61],"it":[62],"influence":[64],"them,":[65],"and":[66,114,150,180],"(2)":[67],"pairwise":[68],"comparisons":[69],"only":[70],"indicate":[71],"which":[72],"response":[73,170],"better,":[75],"not":[76,109],"why.":[77],"These":[78,172],"can":[80,130],"be":[81],"exploited":[82],"cause":[84],"tampering.":[86],"For":[87],"example,":[88],"if":[89],"an":[90],"generates":[92],"biased":[93],"responses":[94],"higher":[96],"quality,":[97],"annotators":[98],"will":[99],"prefer":[100],"them":[101],"based":[102],"on":[103],"quality.":[104,171],"However,":[105],"labels":[107],"do":[108],"distinguish":[110],"quality":[111],"bias,":[113],"reward":[116],"model":[117],"inherits":[118],"limitation.":[120],"Optimizing":[121],"such":[122],"rewards":[123],"through":[124],"reinforcement":[125],"learning":[126],"or":[127],"best-of-N":[128],"sampling":[129],"misaligned":[132],"biases.":[133],"Our":[134],"experiments":[135],"demonstrate":[136],"amplification":[137],"across":[138],"diverse":[139],"biases:":[140],"keyword":[142],"bias":[143],"propaganda":[145],"(e.g.,":[146],"sexism),":[147],"brand":[148],"promotion,":[149],"instrumental":[151],"goal-seeking.":[152],"Mitigation":[153],"remains":[154],"challenging,":[155],"as":[156],"existing":[157],"techniques":[158],"for":[159],"robust":[160],"fail":[162],"fully":[164],"resolve":[165],"tampering":[167],"without":[168],"sacrificing":[169],"findings":[173],"reveal":[174],"structural":[175],"vulnerabilities":[176],"current":[178],"emphasize":[181],"need":[183],"prevent":[185],"vulnerability.":[187],"Project":[188],"page:":[189],"https://alignment-tampering.github.io/":[190]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-28T00:00:00"}