@@ -42,11 +42,86 @@ pub enum BinOpToken {
4242 Shr ,
4343}
4444
45+ // This type must not implement `Hash` due to the unusual `PartialEq` impl below.
46+ #[ derive( Copy , Clone , Debug , Encodable , Decodable , HashStable_Generic ) ]
47+ pub enum InvisibleOrigin {
48+ // From the expansion of a metavariable in a declarative macro.
49+ MetaVar ( MetaVarKind ) ,
50+
51+ // Converted from `proc_macro::Delimiter` in
52+ // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro.
53+ ProcMacro ,
54+
55+ // Converted from `TokenKind::Interpolated` in
56+ // `TokenStream::flatten_token`. Treated similarly to `ProcMacro`.
57+ FlattenToken ,
58+ }
59+
60+ impl PartialEq for InvisibleOrigin {
61+ #[ inline]
62+ fn eq ( & self , _other : & InvisibleOrigin ) -> bool {
63+ // When we had AST-based nonterminals we couldn't compare them, and the
64+ // old `Nonterminal` type had an `eq` that always returned false,
65+ // resulting in this restriction:
66+ // https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment
67+ // This `eq` emulates that behaviour. We could consider lifting this
68+ // restriction now but there are still cases involving invisible
69+ // delimiters that make it harder than it first appears.
70+ false
71+ }
72+ }
73+
74+ /// Annoyingly similar to `NonterminalKind`, but the slight differences are important.
75+ #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable , Hash , HashStable_Generic ) ]
76+ pub enum MetaVarKind {
77+ Item ,
78+ Block ,
79+ Stmt ,
80+ Pat ( NtPatKind ) ,
81+ Expr {
82+ kind : NtExprKind ,
83+ // This field is needed for `Token::can_begin_literal_maybe_minus`.
84+ can_begin_literal_maybe_minus : bool ,
85+ // This field is needed for `Token::can_begin_string_literal`.
86+ can_begin_string_literal : bool ,
87+ } ,
88+ Ty ,
89+ Ident ,
90+ Lifetime ,
91+ Literal ,
92+ Meta ,
93+ Path ,
94+ Vis ,
95+ TT ,
96+ }
97+
98+ impl fmt:: Display for MetaVarKind {
99+ fn fmt ( & self , f : & mut fmt:: Formatter < ' _ > ) -> fmt:: Result {
100+ let sym = match self {
101+ MetaVarKind :: Item => sym:: item,
102+ MetaVarKind :: Block => sym:: block,
103+ MetaVarKind :: Stmt => sym:: stmt,
104+ MetaVarKind :: Pat ( PatParam { inferred : true } | PatWithOr ) => sym:: pat,
105+ MetaVarKind :: Pat ( PatParam { inferred : false } ) => sym:: pat_param,
106+ MetaVarKind :: Expr { kind : Expr2021 { inferred : true } | Expr , .. } => sym:: expr,
107+ MetaVarKind :: Expr { kind : Expr2021 { inferred : false } , .. } => sym:: expr_2021,
108+ MetaVarKind :: Ty => sym:: ty,
109+ MetaVarKind :: Ident => sym:: ident,
110+ MetaVarKind :: Lifetime => sym:: lifetime,
111+ MetaVarKind :: Literal => sym:: literal,
112+ MetaVarKind :: Meta => sym:: meta,
113+ MetaVarKind :: Path => sym:: path,
114+ MetaVarKind :: Vis => sym:: vis,
115+ MetaVarKind :: TT => sym:: tt,
116+ } ;
117+ write ! ( f, "{sym}" )
118+ }
119+ }
120+
45121/// Describes how a sequence of token trees is delimited.
46122/// Cannot use `proc_macro::Delimiter` directly because this
47123/// structure should implement some additional traits.
48- #[ derive( Copy , Clone , Debug , PartialEq , Eq ) ]
49- #[ derive( Encodable , Decodable , Hash , HashStable_Generic ) ]
124+ #[ derive( Copy , Clone , Debug , PartialEq , Encodable , Decodable , HashStable_Generic ) ]
50125pub enum Delimiter {
51126 /// `( ... )`
52127Parenthesis ,
@@ -59,7 +134,34 @@ pub enum Delimiter {
59134/// "macro variable" `$var`. It is important to preserve operator priorities in cases like
60135/// `$var * 3` where `$var` is `1 + 2`.
61136/// Invisible delimiters might not survive roundtrip of a token stream through a string.
62- Invisible ,
137+ Invisible ( InvisibleOrigin ) ,
138+ }
139+
140+ impl Delimiter {
141+ // Should the parser skip these delimiters? Only happens for certain kinds
142+ // of invisible delimiters. Ideally this function will eventually disappear
143+ // and no invisible delimiters will be skipped.
144+ #[ inline]
145+ pub fn skip ( & self ) -> bool {
146+ match self {
147+ Delimiter :: Parenthesis | Delimiter :: Bracket | Delimiter :: Brace => false ,
148+ Delimiter :: Invisible ( InvisibleOrigin :: MetaVar ( _) ) => false ,
149+ Delimiter :: Invisible ( InvisibleOrigin :: FlattenToken | InvisibleOrigin :: ProcMacro ) => {
150+ true
151+ }
152+ }
153+ }
154+
155+ // This exists because `InvisibleOrigin`s should be compared. It is only used for assertions.
156+ pub fn eq_ignoring_invisible_origin ( & self , other : & Delimiter ) -> bool {
157+ match ( self , other) {
158+ ( Delimiter :: Parenthesis , Delimiter :: Parenthesis ) => true ,
159+ ( Delimiter :: Brace , Delimiter :: Brace ) => true ,
160+ ( Delimiter :: Bracket , Delimiter :: Bracket ) => true ,
161+ ( Delimiter :: Invisible ( _) , Delimiter :: Invisible ( _) ) => true ,
162+ _ => false ,
163+ }
164+ }
63165}
64166
65167// Note that the suffix is *not* considered when deciding the `LitKind` in this
@@ -896,7 +998,7 @@ impl PartialEq<TokenKind> for Token {
896998 }
897999}
8981000
899- #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable ) ]
1001+ #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable , Hash , HashStable_Generic ) ]
9001002pub enum NtPatKind {
9011003 // Matches or-patterns. Was written using `pat` in edition 2021 or later.
9021004 PatWithOr ,
@@ -906,7 +1008,7 @@ pub enum NtPatKind {
9061008 PatParam { inferred : bool } ,
9071009}
9081010
909- #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable ) ]
1011+ #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable , Hash , HashStable_Generic ) ]
9101012pub enum NtExprKind {
9111013 // Matches expressions using the post-edition 2024. Was written using
9121014 // `expr` in edition 2024 or later.
@@ -933,7 +1035,7 @@ pub enum Nonterminal {
9331035 NtVis ( P < ast:: Visibility > ) ,
9341036}
9351037
936- #[ derive( Debug , Copy , Clone , PartialEq , Encodable , Decodable ) ]
1038+ #[ derive( Debug , Copy , Clone , PartialEq , Eq , Encodable , Decodable , Hash , HashStable_Generic ) ]
9371039pub enum NonterminalKind {
9381040 Item ,
9391041 Block ,
0 commit comments