Have parse_stream handle NUL bytes

Author: kddnewton

ext/prism/extension.c

@@ -856,8 +856,8 @@ parse_stream_fgets(char *string, int size, void *stream) {
         return NULL;
     }
 
-    const char *cstr = StringValueCStr(line);
-    size_t length = strlen(cstr);
+    const char *cstr = RSTRING_PTR(line);
+    long length = RSTRING_LEN(line);
 
     memcpy(string, cstr, length);
     string[length] = '\0';

src/prism.c

@@ -21696,18 +21696,21 @@ pm_parse_stream_read(pm_buffer_t *buffer, void *stream, pm_parse_stream_fgets_t
 #define LINE_SIZE 4096
     char line[LINE_SIZE];
 
-    while (fgets(line, LINE_SIZE, stream) != NULL) {
-        size_t length = strlen(line);
+    while (memset(line, '\n', LINE_SIZE), fgets(line, LINE_SIZE, stream) != NULL) {
+        size_t length = LINE_SIZE;
+        while (length > 0 && line[length - 1] == '\n') length--;
 
-        if (length == LINE_SIZE && line[length - 1] != '\n') {
+        if (length == LINE_SIZE) {
             // If we read a line that is the maximum size and it doesn't end
             // with a newline, then we'll just append it to the buffer and
             // continue reading.
+            length--;
             pm_buffer_append_string(buffer, line, length);
             continue;
         }
 
         // Append the line to the buffer.
+        length--;
         pm_buffer_append_string(buffer, line, length);
 
         // Check if the line matches the __END__ marker. If it does, then stop

test/prism/api/parse_stream_test.rb

@@ -69,5 +69,13 @@ def test_false___END___in_heredoc
       assert result.success?
       assert_equal 4, result.value.statements.body.length
     end
+
+    def test_nul_bytes
+      io = StringIO.new("1 # \0\0\0 \n2 # \0\0\0\n3")
+      result = Prism.parse_stream(io)
+
+      assert result.success?
+      assert_equal 3, result.value.statements.body.length
+    end
   end
 end