{"id":"https://openalex.org/W7117538756","doi":"https://doi.org/10.1145/3733799.3762982","title":"Defending Against Prompt Injection With a Few DefensiveTokens","display_name":"Defending Against Prompt Injection With a Few DefensiveTokens","publication_year":2025,"publication_date":"2025-10-13","ids":{"openalex":"https://openalex.org/W7117538756","doi":"https://doi.org/10.1145/3733799.3762982"},"language":null,"primary_location":{"id":"doi:10.1145/3733799.3762982","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3733799.3762982","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM Workshop on Artificial Intelligence and Security","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://doi.org/10.1145/3733799.3762982","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101888837","display_name":"Sizhe Chen","orcid":"https://orcid.org/0000-0003-3274-6926"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Sizhe Chen","raw_affiliation_strings":["UC Berkeley, Berkeley, CA, USA"],"raw_orcid":"https://orcid.org/0000-0003-3274-6926","affiliations":[{"raw_affiliation_string":"UC Berkeley, Berkeley, CA, USA","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yizhu Wang","orcid":"https://orcid.org/0009-0001-9626-0290"},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yizhu Wang","raw_affiliation_strings":["UC Berkeley, Berkeley, CA, USA"],"raw_orcid":"https://orcid.org/0009-0001-9626-0290","affiliations":[{"raw_affiliation_string":"UC Berkeley, Berkeley, CA, USA","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5034257647","display_name":"Nicholas Carlini","orcid":"https://orcid.org/0000-0002-1463-3461"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Nicholas Carlini","raw_affiliation_strings":["Google, Mountain View, CA, USA and Anthropic, San Francisco, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-1463-3461","affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA and Anthropic, San Francisco, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040461332","display_name":"Chawin Sitawarin","orcid":"https://orcid.org/0000-0002-4949-9661"},"institutions":[{"id":"https://openalex.org/I1291425158","display_name":"Google (United States)","ror":"https://ror.org/00njsd438","country_code":"US","type":"company","lineage":["https://openalex.org/I1291425158","https://openalex.org/I4210128969"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chawin Sitawarin","raw_affiliation_strings":["Google, Mountain View, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-4949-9661","affiliations":[{"raw_affiliation_string":"Google, Mountain View, CA, USA","institution_ids":["https://openalex.org/I1291425158"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5121568632","display_name":"David Wagner","orcid":null},"institutions":[{"id":"https://openalex.org/I134446601","display_name":"Berkeley College","ror":"https://ror.org/02xewxa75","country_code":"US","type":"education","lineage":["https://openalex.org/I134446601"]},{"id":"https://openalex.org/I95457486","display_name":"University of California, Berkeley","ror":"https://ror.org/01an7q238","country_code":"US","type":"education","lineage":["https://openalex.org/I95457486"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"David Wagner","raw_affiliation_strings":["UC Berkeley, Berkeley, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-9944-9232","affiliations":[{"raw_affiliation_string":"UC Berkeley, Berkeley, CA, USA","institution_ids":["https://openalex.org/I134446601","https://openalex.org/I95457486"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":14.9135,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.99210202,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"242","last_page":"252"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.7524999976158142,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.7524999976158142,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11424","display_name":"Security and Verification in Computing","score":0.1371999979019165,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.014399999752640724,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/append","display_name":"Append","score":0.694100022315979},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6313999891281128},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.617900013923645},{"id":"https://openalex.org/keywords/computer-security-model","display_name":"Computer security model","score":0.5489000082015991},{"id":"https://openalex.org/keywords/security-testing","display_name":"Security testing","score":0.460099995136261},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.42980000376701355},{"id":"https://openalex.org/keywords/data-security","display_name":"Data security","score":0.37119999527931213}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7426999807357788},{"id":"https://openalex.org/C2777998813","wikidata":"https://www.wikidata.org/wiki/Q16869124","display_name":"Append","level":2,"score":0.694100022315979},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6313999891281128},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.6241000294685364},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.617900013923645},{"id":"https://openalex.org/C121822524","wikidata":"https://www.wikidata.org/wiki/Q5157582","display_name":"Computer security model","level":2,"score":0.5489000082015991},{"id":"https://openalex.org/C195518309","wikidata":"https://www.wikidata.org/wiki/Q13424265","display_name":"Security testing","level":5,"score":0.460099995136261},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.42980000376701355},{"id":"https://openalex.org/C10511746","wikidata":"https://www.wikidata.org/wiki/Q899388","display_name":"Data security","level":3,"score":0.37119999527931213},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.3490999937057495},{"id":"https://openalex.org/C2779639559","wikidata":"https://www.wikidata.org/wiki/Q7661178","display_name":"Symbolic execution","level":3,"score":0.3285999894142151},{"id":"https://openalex.org/C2777267654","wikidata":"https://www.wikidata.org/wiki/Q3519023","display_name":"Test (biology)","level":2,"score":0.2985000014305115},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C114869243","wikidata":"https://www.wikidata.org/wiki/Q133735","display_name":"Security through obscurity","level":5,"score":0.296099990606308},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.2912999987602234},{"id":"https://openalex.org/C56288433","wikidata":"https://www.wikidata.org/wiki/Q58673","display_name":"Data manipulation language","level":2,"score":0.2863999903202057},{"id":"https://openalex.org/C2988963302","wikidata":"https://www.wikidata.org/wiki/Q629206","display_name":"Program code","level":2,"score":0.27559998631477356},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.2574000060558319}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3733799.3762982","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3733799.3762982","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM Workshop on Artificial Intelligence and Security","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.1145/3733799.3762982","is_oa":true,"landing_page_url":"https://doi.org/10.1145/3733799.3762982","pdf_url":null,"source":null,"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 18th ACM Workshop on Artificial Intelligence and Security","raw_type":"proceedings-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","score":0.668560266494751,"display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G6360409883","display_name":"AI Institute for Agent-based Cyber Threat Intelligence and Operation","funder_award_id":"2229876","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320315389","display_name":"Open Philanthropy Project","ror":"https://ror.org/004d1k391"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":13,"referenced_works":["https://openalex.org/W3174770825","https://openalex.org/W4205991051","https://openalex.org/W4386768656","https://openalex.org/W4388886073","https://openalex.org/W4389520756","https://openalex.org/W4402670146","https://openalex.org/W4404612289","https://openalex.org/W4408612787","https://openalex.org/W4409150456","https://openalex.org/W4410609227","https://openalex.org/W4411113002","https://openalex.org/W4411337880","https://openalex.org/W4416549384"],"related_works":[],"abstract_inverted_index":{"When":[0],"large":[1],"language":[2],"model":[3,89],"(LLM)":[4],"systems":[5],"interact":[6],"with":[7,44,100,138],"external":[8],"data":[9,29],"to":[10,38,67,105,135],"perform":[11],"complex":[12],"tasks,":[13],"a":[14,21,74,97,128,139,150,179],"new":[15],"attack,":[16],"namely":[17],"prompt":[18,101],"injection,":[19],"becomes":[20],"significant":[22],"threat.":[23],"By":[24],"injecting":[25],"instructions":[26],"into":[27],"the":[28,32,34,40,50,54,88,132,157,161,176,183],"accessed":[30],"by":[31,49,92],"system,":[33,55],"attacker":[35],"is":[36,147,165,195],"able":[37],"override":[39],"initial":[41],"user":[42],"task":[43,47],"an":[45],"arbitrary":[46],"directed":[48],"attacker.":[51],"To":[52],"secure":[53],"test-time":[56,98],"defenses,":[57],"e.g.,":[58],"defensive":[59],"prompting,":[60],"have":[61],"been":[62],"proposed":[63],"for":[64,119],"system":[65,124,159],"developers":[66,125,152],"attain":[68],"security":[69,137,146,189],"only":[70],"when":[71],"needed":[72],"in":[73],"flexible":[75,180],"manner.":[76],"However,":[77],"they":[78],"are":[79,109,117],"much":[80],"less":[81,148],"effective":[82],"than":[83],"training-time":[84,106],"defenses":[85],"that":[86],"change":[87],"parameters.":[90],"Motivated":[91],"this,":[93],"we":[94],"propose":[95],"DefensiveToken,":[96],"defense":[99],"injection":[102],"robustness":[103],"comparable":[104],"alternatives.":[107],"DefensiveTokens":[108,130],"newly":[110],"inserted":[111],"as":[112,163],"special":[113],"tokens,":[114],"whose":[115],"embeddings":[116],"optimized":[118],"security.":[120],"In":[121,143],"security-sensitive":[122],"cases,":[123],"can":[126,153],"append":[127],"few":[129],"before":[131],"LLM":[133,158],"input":[134],"achieve":[136],"minimal":[140],"utility":[141,186],"drop.":[142],"scenarios":[144],"where":[145],"of":[149],"concern,":[151],"simply":[154],"skip":[155],"DefensiveTokens;":[156],"remains":[160],"same":[162],"there":[164],"no":[166],"defense,":[167],"generating":[168],"high-quality":[169],"responses.":[170],"Thus,":[171],"DefensiveTokens,":[172],"if":[173],"released":[174],"alongside":[175],"model,":[177],"allow":[178],"switch":[181],"between":[182],"state-of-the-art":[184],"(SOTA)":[185],"and":[187],"almost-SOTA":[188],"at":[190],"test":[191],"time.":[192],"The":[193],"code":[194],"available":[196],"here.":[197]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":1}],"updated_date":"2026-08-02T14:50:37.381335","created_date":"2025-12-30T00:00:00"}