Skip to content

Commit 33ab6f9

Browse files
progvalprogval
committed
Fix handling of (lack of) spaces after a keyword.
1 parent 7176eab commit 33ab6f9

File tree

5 files changed

+101
-74
lines changed

5 files changed

+101
-74
lines changed

‎src/bytes.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ named!(escapedchar<StrSpan, Option<u8>>,
1111
| char!('\\') => { |_| Some(b'\\') }
1212
| char!('\'') => { |_| Some(b'\'') }
1313
| char!('"') => { |_| Some(b'"') }
14-
| char!('b') => { |_| Some(b'\x07') } // BEL
14+
| char!('a') => { |_| Some(b'\x07') } // BEL
15+
| char!('b') => { |_| Some(b'\x08') } // BS
1516
| char!('f') => { |_| Some(b'\x0c') } // FF
1617
| char!('n') => { |_| Some(b'\n') }
1718
| char!('r') => { |_| Some(b'\r') }

‎src/expressions.rs

+37-40
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@ use std::marker::PhantomData;
33
use nom::{IResult, Err, Context, ErrorKind};
44
//use unicode_names;
55

6-
use helpers;
7-
use helpers::{StrSpan, name};
8-
use helpers::{AreNewlinesSpaces, NewlinesAreSpaces};
6+
use helpers::*;
97
use functions::varargslist;
108
use bytes::bytes;
119
use strings::string;
@@ -44,9 +42,9 @@ named!(pub test<StrSpan, Box<Expression>>,
4442
| do_parse!(
4543
left: call!(Self::or_test) >>
4644
right: opt!(do_parse!(
47-
ws3!(tag!("if")) >>
45+
ws3!(keyword!("if")) >>
4846
cond: call!(Self::or_test) >>
49-
ws3!(tag!("else")) >>
47+
ws3!(keyword!("else")) >>
5048
right: call!(Self::test) >> (
5149
(cond, right)
5250
)
@@ -70,23 +68,22 @@ named!(test_nocond<StrSpan, Box<Expression>>,
7068

7169
// lambdef: 'lambda' [varargslist] ':' test
7270
named!(lambdef<StrSpan, Box<Expression>>,
73-
do_parse!(
74-
tag!("lambda") >>
75-
args: opt!(preceded!(space_sep!(), varargslist)) >>
71+
ws3!(do_parse!(
72+
keyword!("lambda") >>
73+
args: opt!(varargslist) >>
7674
spaces!() >>
7775
char!(':') >>
7876
spaces!() >>
7977
code: call!(Self::test) >> (
8078
Box::new(Expression::Lambdef(args.unwrap_or_default(), code))
8179
)
82-
)
80+
))
8381
);
8482

8583
// lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
8684
named!(lambdef_nocond<StrSpan, Box<Expression>>,
8785
do_parse!(
88-
tag!("lambda") >>
89-
space_sep!() >>
86+
keyword!("lambda") >>
9087
args: opt!(varargslist) >>
9188
char!(':') >>
9289
code: call!(Self::test_nocond) >> (
@@ -129,18 +126,18 @@ impl<ANS: AreNewlinesSpaces> ExpressionParser<ANS> {
129126

130127
// or_test: and_test ('or' and_test)*
131128
bop!(or_test, Self::and_test, alt!(
132-
tuple!(tag!("or"), space_sep!()) => { |_| Bop::Or }
129+
keyword!("or") => { |_| Bop::Or }
133130
));
134131

135132
// and_test: not_test ('and' not_test)*
136133
bop!(and_test, Self::not_test, alt!(
137-
tuple!(tag!("and"), space_sep!()) => { |_| Bop::And }
134+
keyword!("and") => { |_| Bop::And }
138135
));
139136

140137
// not_test: 'not' not_test | comparison
141138
named!(not_test<StrSpan, Box<Expression>>,
142139
alt!(
143-
preceded!(tuple!(tag!("not"), space_sep!()), call!(Self::comparison)) => { |e| Box::new(Expression::Uop(Uop::Not, e)) }
140+
preceded!(tuple!(keyword!("not"), spaces!()), call!(Self::comparison)) => { |e| Box::new(Expression::Uop(Uop::Not, e)) }
144141
| call!(Self::comparison)
145142
)
146143
);
@@ -155,8 +152,8 @@ bop!(comparison, Self::expr, alt!(
155152
| char!('>') => { |_| Bop::Gt }
156153
| tag!("!=") => { |_| Bop::Neq }
157154
| tag!("in") => { |_| Bop::In }
158-
| tuple!(tag!("not"), space_sep!(), tag!("in"), space_sep!()) => { |_| Bop::NotIn }
159-
| tuple!(tag!("is"), space_sep!(), tag!("not"), space_sep!()) => { |_| Bop::IsNot }
155+
| tuple!(tag!("not"), space_sep!(), keyword!("in")) => { |_| Bop::NotIn }
156+
| tuple!(tag!("is"), space_sep!(), keyword!("not")) => { |_| Bop::IsNot }
160157
| tuple!(tag!("is"), space_sep!()) => { |_| Bop::Is }
161158
));
162159

@@ -259,9 +256,9 @@ named!(atom_expr<StrSpan, Box<Expression>>,
259256
named!(atom<StrSpan, Box<Expression>>,
260257
map!(alt!(
261258
tag!("...") => { |_| Expression::Ellipsis }
262-
| tag!("None") => { |_| Expression::None }
263-
| tag!("True") => { |_| Expression::True }
264-
| tag!("False") => { |_| Expression::False }
259+
| keyword!("None") => { |_| Expression::None }
260+
| keyword!("True") => { |_| Expression::True }
261+
| keyword!("False") => { |_| Expression::False }
265262
| separated_nonempty_list!(spaces!(), string) => { |s| Expression::String(s) }
266263
| separated_nonempty_list!(spaces!(), bytes) => { |v| {
267264
let mut v2 = Vec::new();
@@ -270,12 +267,12 @@ named!(atom<StrSpan, Box<Expression>>,
270267
}}
271268
| number
272269
| name => { |n| Expression::Name(n) }
273-
| ws3!(tuple!(char!('['), ws4!(opt!(char!(' '))), char!(']'))) => { |_| Expression::ListLiteral(vec![]) }
274-
| ws3!(tuple!(char!('{'), ws4!(opt!(char!(' '))), char!('}'))) => { |_| Expression::DictLiteral(vec![]) }
275-
| ws3!(tuple!(char!('('), ws4!(opt!(char!(' '))), char!(')'))) => { |_| Expression::TupleLiteral(vec![]) }
276-
| ws3!(delimited!(char!('{'), ws4!(map!(
270+
| tuple!(char!('['), ws4!(opt!(char!(' '))), char!(']')) => { |_| Expression::ListLiteral(vec![]) }
271+
| tuple!(char!('{'), ws4!(opt!(char!(' '))), char!('}')) => { |_| Expression::DictLiteral(vec![]) }
272+
| tuple!(char!('('), ws4!(opt!(char!(' '))), char!(')')) => { |_| Expression::TupleLiteral(vec![]) }
273+
| delimited!(char!('{'), ws4!(map!(
277274
call!(ExpressionParser::<NewlinesAreSpaces>::dictorsetmaker), |e:Box<_>| *e
278-
)), char!('}')))
275+
)), char!('}'))
279276
| map_opt!(ws3!(delimited!(char!('('), ws4!(
280277
call!(ExpressionParser::<NewlinesAreSpaces>::testlist_comp)
281278
), char!(')'))), |ret| {
@@ -290,12 +287,12 @@ named!(atom<StrSpan, Box<Expression>>,
290287
TestlistCompReturn::Single(SetItem::Star(_)) => None,
291288
}
292289
})
293-
| ws3!(delimited!(char!('('), ws4!(
290+
| delimited!(char!('('), ws4!(
294291
call!(ExpressionParser::<NewlinesAreSpaces>::yield_expr)
295-
), char!(')')))
296-
| ws3!(delimited!(char!('['), ws4!(
292+
), char!(')'))
293+
| delimited!(char!('['), ws4!(
297294
call!(ExpressionParser::<NewlinesAreSpaces>::testlist_comp)
298-
), char!(']'))) => { |ret| {
295+
), char!(']')) => { |ret| {
299296
match ret {
300297
TestlistCompReturn::Comp(e, comp) => Expression::ListComp(e, comp),
301298
TestlistCompReturn::Lit(v) => Expression::ListLiteral(v),
@@ -428,7 +425,7 @@ named_args!(dictmaker(item1: DictItem) <StrSpan, Box<Expression>>,
428425
v.insert(0, item1.clone()); // FIXME: do not clone
429426
Box::new(Expression::DictLiteral(v))
430427
}}
431-
| preceded!(peek!(tuple!(tag!("for"), call!(helpers::space_sep))), call!(Self::comp_for)) => { |comp| {
428+
| preceded!(peek!(keyword!("for")), call!(Self::comp_for)) => { |comp| {
432429
Box::new(Expression::DictComp(Box::new(item1.clone()), comp)) // FIXME: do not clone
433430
}}
434431
)),
@@ -570,11 +567,11 @@ named!(comp_for<StrSpan, Vec<ComprehensionChunk>>,
570567
named_args!(comp_for2(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionChunk>>,
571568
do_parse!(
572569
async: map!(opt!(terminated!(tag!("async"), space_sep!())), |o| o.is_some()) >>
573-
tag!("for") >>
570+
keyword!("for") >>
574571
spaces!() >>
575572
item: call!(Self::exprlist) >>
576573
spaces!() >>
577-
tag!("in") >>
574+
keyword!("in") >>
578575
spaces!() >>
579576
iterator: map!(call!(Self::or_test), |e| *e) >>
580577
spaces!() >>
@@ -587,7 +584,7 @@ named_args!(comp_for2(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionC
587584
// comp_if: 'if' test_nocond [comp_iter]
588585
named_args!(comp_if(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionChunk>>,
589586
do_parse!(
590-
tag!("if") >>
587+
keyword!("if") >>
591588
spaces!() >>
592589
cond: map!(call!(Self::test_nocond), |e| *e) >>
593590
spaces!() >>
@@ -601,14 +598,14 @@ named_args!(comp_if(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionChu
601598
// yield_expr: 'yield' [yield_arg]
602599
// yield_arg: 'from' test | testlist
603600
named!(pub yield_expr<StrSpan, Expression>,
604-
preceded!(
605-
tag!("yield"),
606-
alt!(
607-
preceded!(tuple!(space_sep!(), tag!("from"), space_sep!()), call!(Self::test)) => { |e| Expression::YieldFrom(e) }
608-
| preceded!(space_sep!(), call!(Self::testlist)) => { |e| Expression::Yield(e) }
609-
| ws3!(tag!("")) => { |_| Expression::Yield(Vec::new()) }
610-
)
611-
)
601+
ws3!(preceded!(
602+
keyword!("yield"),
603+
ws3!(alt!(
604+
preceded!(ws3!(keyword!("from")), call!(Self::test)) => { |e| Expression::YieldFrom(e) }
605+
| call!(Self::testlist) => { |e| Expression::Yield(e) }
606+
| tag!("") => { |_| Expression::Yield(Vec::new()) }
607+
))
608+
))
612609
);
613610

614611
} // End ExpressionParser

‎src/helpers.rs

+8
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,14 @@ named!(pub name<StrSpan, String>,
131131
)
132132
);
133133

134+
named!(pub word_end<StrSpan, ()>,
135+
not!(verify!(peek!(::nom::anychar), |c| UnicodeXID::is_xid_continue(c)))
136+
);
137+
138+
macro_rules! keyword {
139+
($i:expr, $kw:expr) => { terminated!($i, tag!($kw), word_end) }
140+
}
141+
134142
named!(pub newline<StrSpan, ()>,
135143
map!(
136144
many1!(

0 commit comments

Comments
 (0)