python_parser/
lib.rs

1//! A Python parser based on nom, plus some utilities.
2//!
3//! # Panics
4//!
5//! Never (except stack overflows).
6//!
7//! # Numbers
8//!
9//! Python's integer literals may be arbitrary large. This is supported
10//! thanks to the `num_bigint` crate.
11//! Disable the `bigint` feature to fall back to `u64`.
12//!
13//! # String encoding
14//!
15//! `ast::PyString`s are WTF8-encoded if the `wtf8` feature is enabled
16//! (the default) allowing full support for Python's string literals.
17//!
18//! If that feature is disabled, they default to regular Rust strings.
19//! Note that without the `wtf8` feature, some valid string
20//! literals will be badly parsed (missing characters).
21//!
22//! # Python version support
23//!
24//! Currently supports Python 3.7's syntax (and Python 3.8 up to
25//! [2018-09-22](http://github.com/python/cpython/commit/fd97d1f1af910a6222ea12aec42c456b64f9aee4)).
26//!
27//! # Example
28//!
29//! ```
30//! use python_parser::ast::*;
31//! let code = "print(2 + 3, fd=sys.stderr)";
32//! let ast = python_parser::file_input(python_parser::make_strspan(code))
33//!           .unwrap()
34//!           .1;
35//! assert_eq!(ast,
36//!     vec![
37//!         Statement::Assignment(
38//!             vec![
39//!                 Expression::Call(
40//!                     Box::new(Expression::Name("print".to_string())),
41//!                     vec![
42//!                         Argument::Positional(
43//!                             Expression::Bop(
44//!                                 Bop::Add,
45//!                                 Box::new(Expression::Int(2u32.into())),
46//!                                 Box::new(Expression::Int(3u32.into())),
47//!                             )
48//!                         ),
49//!                         Argument::Keyword(
50//!                             "fd".to_string(),
51//!                             Expression::Attribute(
52//!                                 Box::new(Expression::Name("sys".to_string())),
53//!                                 "stderr".to_string(),
54//!                             )
55//!                         ),
56//!                     ]
57//!                 ),
58//!             ],
59//!             vec![],
60//!         )
61//!     ]
62//! );
63//! ```
64
65#![recursion_limit = "128"]
66
67#[macro_use]
68extern crate nom;
69extern crate nom_locate;
70
71#[cfg(test)]
72#[macro_use]
73extern crate pretty_assertions;
74
75extern crate unicode_xid;
76
77#[cfg(feature = "unicode-names")]
78extern crate unicode_names2;
79
80#[cfg(feature = "bigint")]
81extern crate num_bigint;
82#[cfg(feature = "bigint")]
83extern crate num_traits;
84
85#[cfg(feature = "wtf8")]
86extern crate wtf8;
87
88#[macro_use]
89mod helpers;
90#[macro_use]
91mod expressions;
92#[macro_use]
93mod statements;
94pub mod ast;
95mod bytes;
96pub mod errors;
97mod functions;
98mod numbers;
99mod strings;
100pub mod visitors;
101
102use ast::*;
103use expressions::*;
104use helpers::*;
105use statements::*;
106
107pub use helpers::make_strspan;
108
109// single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
110named_attr!(#[doc = "Parses a single interactive statement, like in the REPL."],
111pub parse_single_input <StrSpan, Vec<Statement>>,
112  alt!(
113    newline => { |_| Vec::new() }
114  | call!(statement, 0) => { |stmts| stmts }
115  )
116);
117
118// file_input: (NEWLINE | stmt)* ENDMARKER
119named_attr!(#[doc = "Parses a module or sequence of commands."],
120pub file_input <StrSpan, Vec<Statement>>,
121  fold_many0!(
122    alt!(
123      newline => { |_| None }
124    | eof!() => { |_| None }
125    | call!(statement, 0) => { |s| Some(s) }
126    ),
127    Vec::new(),
128    |acc: Vec<_>, item| { let mut acc = acc; if let Some(s) = item { acc.extend(s); } acc }
129  )
130);
131
132// eval_input: testlist NEWLINE* ENDMARKER
133named_attr!(#[doc = "Parses the input of eval()."],
134pub eval_input <StrSpan, Vec<Expression>>,
135  terminated!(ws_nonl!(call!(ExpressionParser::<NewlinesAreNotSpaces>::testlist)), many0!(newline))
136);
137
138// encoding_decl: NAME
139// TODO
140
141#[cfg(test)]
142mod tests {
143    use super::*;
144    use helpers::{assert_parse_eq, make_strspan};
145
146    #[test]
147    fn foo() {
148        assert_parse_eq(newline(make_strspan("\n")), Ok((make_strspan(""), ())));
149        assert_parse_eq(
150            parse_single_input(make_strspan("del foo")),
151            Ok((
152                make_strspan(""),
153                vec![Statement::Del(vec![Expression::Name("foo".to_string())])],
154            )),
155        );
156        assert_parse_eq(
157            parse_single_input(make_strspan("del foo, bar")),
158            Ok((
159                make_strspan(""),
160                vec![Statement::Del(vec![
161                    Expression::Name("foo".to_string()),
162                    Expression::Name("bar".to_string()),
163                ])],
164            )),
165        );
166        assert_parse_eq(
167            parse_single_input(make_strspan("del foo; del bar")),
168            Ok((
169                make_strspan(""),
170                vec![
171                    Statement::Del(vec![Expression::Name("foo".to_string())]),
172                    Statement::Del(vec![Expression::Name("bar".to_string())]),
173                ],
174            )),
175        );
176        assert_parse_eq(
177            parse_single_input(make_strspan("del foo ;del bar")),
178            Ok((
179                make_strspan(""),
180                vec![
181                    Statement::Del(vec![Expression::Name("foo".to_string())]),
182                    Statement::Del(vec![Expression::Name("bar".to_string())]),
183                ],
184            )),
185        );
186    }
187}
python_parser/lib.rs

python_parser/
lib.rs
