Fix parsing and printing of escape sequences in strings and bytes.

Author: progval

src/bytes.rs

@@ -27,7 +27,7 @@ named!(escapedchar<StrSpan, Option<u8>>,
       }
     | preceded!(char!('x'), tuple!(one_of!("0123456789abcdefABCDEF"), one_of!("0123456789abcdefABCDEF"))) => { |(c1, c2): (char, char)|
         match (c1.to_digit(16), c2.to_digit(16)) {
-            (Some(d1), Some(d2)) => Some(((d1 << 8) + d2) as u8),
+            (Some(d1), Some(d2)) => Some(((d1 << 4) + d2) as u8),
             _ => unreachable!(),
         }
       }

src/expressions.rs

@@ -642,6 +642,10 @@ mod tests {
         assert_parse_eq(atom(make_strspan(r#"r'fo \' o' "#)), Ok((make_strspan(" "),
             Box::new(Expression::String(vec![PyString { prefix: "r".to_string(), content: "fo \\' o".to_string() }])))
         ));
+
+        assert_parse_eq(atom(make_strspan(r#"'\x8a'"#)), Ok((make_strspan(""),
+            Box::new(Expression::String(vec![new_pystring("\u{8a}")])))
+        ));
     }
 
     #[test]

src/main.rs

@@ -15,6 +15,7 @@ fn main() {
         let mut content = String::new();
         file.read_to_string(&mut content).expect("Could not read file");
         let (rest, ast) = file_input(make_strspan(&content)).unwrap();
+        //println!("{:?}", ast);
         let output = format_module(&ast);
         if rest.fragment.0.len() > 0 {
             println!("\nUnparsed: {:?}\n\n", rest.fragment.0)

src/strings.rs

@@ -26,7 +26,7 @@ named!(escapedchar<StrSpan, Option<char>>,
       }
     | preceded!(char!('x'), tuple!(one_of!("0123456789abcdefABCDEF"), one_of!("0123456789abcdefABCDEF"))) => { |(c1, c2): (char, char)|
         match (c1.to_digit(16), c2.to_digit(16)) {
-            (Some(d1), Some(d2)) => ::std::char::from_u32((d1 << 8) + d2),
+            (Some(d1), Some(d2)) => ::std::char::from_u32((d1 << 4) + d2),
             _ => unreachable!(),
         }
       }
@@ -37,15 +37,15 @@ named!(escapedchar<StrSpan, Option<char>>,
     | preceded!(char!('u'), count!(one_of!("0123456789abcdefABCDEF"), 4)) => { |v: Vec<char>| {
         let it: Vec<u32> = v.iter().map(|c| c.to_digit(16).unwrap()).collect();
         if let [d1, d2, d3, d4] = &it[..] {
-            ::std::char::from_u32((d1 << 24) + (d2 << 16) + (d3 << 8) + d4)
+            ::std::char::from_u32((d1 << 12) + (d2 << 8) + (d3 << 4) + d4)
         }
         else { unreachable!() }
       }}
     | preceded!(char!('U'), count!(one_of!("0123456789abcdefABCDEF"), 8)) => { |v: Vec<char>| {
         let it: Vec<u32> = v.iter().map(|c| c.to_digit(16).unwrap()).collect();
         if let [d1, d2, d3, d4, d5, d6, d7, d8] = &it[..] {
-            ::std::char::from_u32((d1 << 56) + (d2 << 48) + (d3 << 40) + (d4 << 32) +
-                                  (d5 << 24) + (d6 << 16) + (d7 << 8) + d8)
+            ::std::char::from_u32((d1 << 28) + (d2 << 24) + (d3 << 20) + (d4 << 16) +
+                                  (d5 << 12) + (d6 << 8) + (d7 << 4) + d8)
         }
         else { unreachable!() }
       }}

src/visitors/printer.rs

@@ -499,14 +499,25 @@ fn format_expr(e: &Expression) -> String {
                     '\t' => "\\t".to_string(),
                     '\\' => "\\\\".to_string(),
                     '"' => "\\\"".to_string(),
-                    c if c.is_ascii() => c.to_string(),
-                    c if (c as u32) <= 0xffff => format!("\\u{:04}", c as u32),
-                    c => format!("\\U{:08}", c as u32),
+                    '\x20'...'\x7e' => c.to_string(),
+                    '\x00'...'\x1f' | '\x7f' | '\u{80}'...'\u{ff}' => format!("\\x{:02x}", c as u8),
+                    '\u{100}'...'\u{ffff}' => format!("\\u{:04x}", c as u16),
+                    '\u{10000}'...'\u{10ffff}' => format!("\\U{:08x}", c as u32),
+                    _ => unreachable!(),
                 }).collect::<Vec<_>>()[..].concat())
             ))
         },
-        Expression::Bytes(ref b) => {
-            format!("bytes({:?})", b) // FIXME: that's cheating
+        Expression::Bytes(ref content) => {
+            format!("b\"{}\"", content.iter().map(|b| match b {
+                b'\r' => "\\r".to_string(),
+                b'\n' => "\\n".to_string(),
+                b'\t' => "\\t".to_string(),
+                b'\\' => "\\\\".to_string(),
+                b'"' => "\\\"".to_string(),
+                0x20...0x7e => (*b as char).to_string(),
+                0x00...0x1f | 0x7f | 0x80...0xff => format!("\\x{:02x}", b),
+                _ => unreachable!(), // waiting for https://github.com/rust-lang/rust/pull/50912
+            }).collect::<Vec<_>>()[..].concat())
         },
 
         Expression::DictLiteral(ref v) =>