Rename YARP filepaths to prism filepaths

4 files changed, 585 insertions, 0 deletions

diff --git a/prism/templates/src/node.c.erb b/prism/templates/src/node.c.erb
new file mode 100644
index 0000000000..f21144f169
--- /dev/null
+++ b/prism/templates/src/node.c.erb
@@ -0,0 +1,157 @@
+#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
+#include "yarp/node.h"
+
+// Clear the node but preserves the location.
+void yp_node_clear(yp_node_t *node) {
+    yp_location_t location = node->location;
+    memset(node, 0, sizeof(yp_node_t));
+    node->location = location;
+}
+
+static void
+yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize);
+
+// Calculate the size of the node list in bytes.
+static size_t
+yp_node_list_memsize(yp_node_list_t *node_list, yp_memsize_t *memsize) {
+    size_t size = sizeof(yp_node_list_t) + (node_list->capacity * sizeof(yp_node_t *));
+    for (size_t index = 0; index < node_list->size; index++) {
+        yp_node_memsize_node(node_list->nodes[index], memsize);
+    }
+    return size;
+}
+
+// Append a new node onto the end of the node list.
+void
+yp_node_list_append(yp_node_list_t *list, yp_node_t *node) {
+    if (list->size == list->capacity) {
+        list->capacity = list->capacity == 0 ? 4 : list->capacity * 2;
+        list->nodes = (yp_node_t **) realloc(list->nodes, sizeof(yp_node_t *) * list->capacity);
+    }
+    list->nodes[list->size++] = node;
+}
+
+YP_EXPORTED_FUNCTION void
+yp_node_destroy(yp_parser_t *parser, yp_node_t *node);
+
+// Deallocate the inner memory of a list of nodes. The parser argument is not
+// used, but is here for the future possibility of pre-allocating memory pools.
+static void
+yp_node_list_free(yp_parser_t *parser, yp_node_list_t *list) {
+    if (list->capacity > 0) {
+        for (size_t index = 0; index < list->size; index++) {
+            yp_node_destroy(parser, list->nodes[index]);
+        }
+        free(list->nodes);
+    }
+}
+
+// Deallocate the space for a yp_node_t. Similarly to yp_node_alloc, we're not
+// using the parser argument, but it's there to allow for the future possibility
+// of pre-allocating larger memory pools.
+YP_EXPORTED_FUNCTION void
+yp_node_destroy(yp_parser_t *parser, yp_node_t *node) {
+    switch (YP_NODE_TYPE(node)) {
+        <%- nodes.each do |node| -%>
+#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
+        case <%= node.type %>: {
+            <%- if node.fields.any? { |field| ![YARP::LocationField, YARP::OptionalLocationField, YARP::UInt32Field, YARP::FlagsField, YARP::ConstantField, YARP::OptionalConstantField].include?(field.class) } -%>
+            yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node;
+            <%- end -%>
+            <%- node.fields.each do |field| -%>
+            <%- case field -%>
+            <%- when YARP::LocationField, YARP::OptionalLocationField, YARP::UInt32Field, YARP::FlagsField, YARP::ConstantField, YARP::OptionalConstantField -%>
+            <%- when YARP::NodeField -%>
+            yp_node_destroy(parser, (yp_node_t *)cast-><%= field.name %>);
+            <%- when YARP::OptionalNodeField -%>
+            if (cast-><%= field.name %> != NULL) {
+                yp_node_destroy(parser, (yp_node_t *)cast-><%= field.name %>);
+            }
+            <%- when YARP::StringField -%>
+            yp_string_free(&cast-><%= field.name %>);
+            <%- when YARP::NodeListField -%>
+            yp_node_list_free(parser, &cast-><%= field.name %>);
+            <%- when YARP::ConstantListField -%>
+            yp_constant_id_list_free(&cast-><%= field.name %>);
+            <%- else -%>
+            <%- raise -%>
+            <%- end -%>
+            <%- end -%>
+            break;
+        }
+        <%- end -%>
+#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
+        default:
+            assert(false && "unreachable");
+            break;
+    }
+    free(node);
+}
+
+static void
+yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize) {
+    memsize->node_count++;
+
+    switch (YP_NODE_TYPE(node)) {
+        // We do not calculate memsize of a ScopeNode
+        // as it should never be generated
+        case YP_SCOPE_NODE:
+            return;
+        <%- nodes.each do |node| -%>
+#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
+        case <%= node.type %>: {
+            yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node;
+            memsize->memsize += sizeof(*cast);
+            <%- if node.fields.any? { |f| f.is_a?(YARP::NodeListField) } -%>
+            // Node lists will add in their own sizes below.
+            memsize->memsize -= sizeof(yp_node_list_t) * <%= node.fields.count { |f| f.is_a?(YARP::NodeListField) } %>;
+            <%- end -%>
+            <%- if node.fields.any? { |f| f.is_a?(YARP::ConstantListField) } -%>
+            // Constant id lists will add in their own sizes below.
+            memsize->memsize -= sizeof(yp_constant_id_list_t) * <%= node.fields.count { |f| f.is_a?(YARP::ConstantListField) } %>;
+            <%- end -%>
+            <%- node.fields.each do |field| -%>
+            <%- case field -%>
+            <%- when YARP::ConstantField, YARP::OptionalConstantField, YARP::UInt32Field, YARP::FlagsField, YARP::LocationField, YARP::OptionalLocationField -%>
+            <%- when YARP::NodeField -%>
+            yp_node_memsize_node((yp_node_t *)cast-><%= field.name %>, memsize);
+            <%- when YARP::OptionalNodeField -%>
+            if (cast-><%= field.name %> != NULL) {
+                yp_node_memsize_node((yp_node_t *)cast-><%= field.name %>, memsize);
+            }
+            <%- when YARP::StringField -%>
+            memsize->memsize += yp_string_memsize(&cast-><%= field.name %>);
+            <%- when YARP::NodeListField -%>
+            memsize->memsize += yp_node_list_memsize(&cast-><%= field.name %>, memsize);
+            <%- when YARP::ConstantListField -%>
+            memsize->memsize += yp_constant_id_list_memsize(&cast-><%= field.name %>);
+            <%- else -%>
+            <%- raise -%>
+            <%- end -%>
+            <%- end -%>
+            break;
+        }
+        <%- end -%>
+#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
+    }
+}
+
+// Calculates the memory footprint of a given node.
+YP_EXPORTED_FUNCTION void
+yp_node_memsize(yp_node_t *node, yp_memsize_t *memsize) {
+    *memsize = (yp_memsize_t) { .memsize = 0, .node_count = 0 };
+    yp_node_memsize_node(node, memsize);
+}
+
+// Returns a string representation of the given node type.
+YP_EXPORTED_FUNCTION const char *
+yp_node_type_to_str(yp_node_type_t node_type)
+{
+    switch (node_type) {
+<%- nodes.each do |node| -%>
+        case <%= node.type %>:
+            return "<%= node.type %>";
+<%- end -%>
+    }
+    return "";
+}
diff --git a/prism/templates/src/prettyprint.c.erb b/prism/templates/src/prettyprint.c.erb
new file mode 100644
index 0000000000..796d8ccafb
--- /dev/null
+++ b/prism/templates/src/prettyprint.c.erb
@@ -0,0 +1,111 @@
+#include "yarp/defines.h"
+
+#include <stdio.h>
+
+#include "yarp/ast.h"
+#include "yarp/parser.h"
+#include "yarp/util/yp_buffer.h"
+
+static void
+prettyprint_location(yp_buffer_t *buffer, yp_parser_t *parser, yp_location_t *location) {
+    char printed[] = "[0000-0000]";
+    snprintf(printed, sizeof(printed), "[%04ld-%04ld]", (long int)(location->start - parser->start), (long int)(location->end - parser->start));
+    yp_buffer_append_str(buffer, printed, strlen(printed));
+}
+
+static void
+prettyprint_node(yp_buffer_t *buffer, yp_parser_t *parser, yp_node_t *node) {
+    switch (YP_NODE_TYPE(node)) {
+        // We do not need to print a ScopeNode as it's not part
+        // of the AST
+        case YP_SCOPE_NODE:
+            return;
+        <%- nodes.each do |node| -%>
+        case <%= node.type %>: {
+            yp_buffer_append_str(buffer, "<%= node.name %>(", <%= node.name.length + 1 %>);
+            <%- node.fields.each_with_index do |field, index| -%>
+            <%= "yp_buffer_append_str(buffer, \", \", 2);" if index != 0 -%>
+            <%- case field -%>
+            <%- when YARP::NodeField -%>
+            prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>);
+            <%- when YARP::OptionalNodeField -%>
+            if (((yp_<%= node.human %>_t *)node)-><%= field.name %> == NULL) {
+                yp_buffer_append_str(buffer, "nil", 3);
+            } else {
+                prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>);
+            }
+            <%- when YARP::StringField -%>
+            yp_buffer_append_str(buffer, "\"", 1);
+            yp_buffer_append_bytes(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= field.name %>), yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= field.name %>));
+            yp_buffer_append_str(buffer, "\"", 1);
+            <%- when YARP::NodeListField -%>
+            yp_buffer_append_str(buffer, "[", 1);
+            for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) {
+                if (index != 0) yp_buffer_append_str(buffer, ", ", 2);
+                prettyprint_node(buffer, parser, (yp_node_t *) ((yp_<%= node.human %>_t *) node)-><%= field.name %>.nodes[index]);
+            }
+            yp_buffer_append_str(buffer, "]", 1);
+            <%- when YARP::ConstantField -%>
+            char <%= field.name %>_buffer[12];
+            snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= field.name %>);
+            yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
+            <%- when YARP::OptionalConstantField -%>
+            if (((yp_<%= node.human %>_t *)node)-><%= field.name %> == 0) {
+                yp_buffer_append_str(buffer, "nil", 3);
+            } else {
+                char <%= field.name %>_buffer[12];
+                snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= field.name %>);
+                yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
+            }
+            <%- when YARP::ConstantListField -%>
+            yp_buffer_append_str(buffer, "[", 1);
+            for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) {
+                if (index != 0) yp_buffer_append_str(buffer, ", ", 2);
+                char <%= field.name %>_buffer[12];
+                snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]);
+                yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
+            }
+            yp_buffer_append_str(buffer, "]", 1);
+            <%- when YARP::LocationField -%>
+            prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>);
+            <%- when YARP::OptionalLocationField -%>
+            if (((yp_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) {
+                yp_buffer_append_str(buffer, "nil", 3);
+            } else {
+                prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>);
+            }
+            <%- when YARP::UInt32Field -%>
+            char <%= field.name %>_buffer[12];
+            snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", ((yp_<%= node.human %>_t *)node)-><%= field.name %>);
+            yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
+            <%- when YARP::FlagsField -%>
+            char <%= field.name %>_buffer[12];
+            snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", node->flags >> <%= YARP::COMMON_FLAGS %>);
+            yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer));
+            <%- else -%>
+            <%- raise -%>
+            <%- end -%>
+            <%- end -%>
+            yp_buffer_append_str(buffer, ")", 1);
+            break;
+        }
+        <%- end -%>
+    }
+}
+
+void
+yp_print_node(yp_parser_t *parser, yp_node_t *node) {
+    yp_buffer_t buffer;
+    if (!yp_buffer_init(&buffer)) return;
+
+    prettyprint_node(&buffer, parser, node);
+    printf("%.*s\n", (int) buffer.length, buffer.value);
+
+    yp_buffer_free(&buffer);
+}
+
+// Pretty-prints the AST represented by the given node to the given buffer.
+YP_EXPORTED_FUNCTION void
+yp_prettyprint(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
+    prettyprint_node(buffer, parser, node);
+}
diff --git a/prism/templates/src/serialize.c.erb b/prism/templates/src/serialize.c.erb
new file mode 100644
index 0000000000..9b3811106e
--- /dev/null
+++ b/prism/templates/src/serialize.c.erb
@@ -0,0 +1,299 @@
+#include "yarp.h"
+
+#include <stdio.h>
+
+static inline uint32_t
+yp_ptrdifft_to_u32(ptrdiff_t value) {
+    assert(value >= 0 && ((unsigned long) value) < UINT32_MAX);
+    return (uint32_t) value;
+}
+
+static inline uint32_t
+yp_sizet_to_u32(size_t value) {
+    assert(value < UINT32_MAX);
+    return (uint32_t) value;
+}
+
+static void
+yp_serialize_location(yp_parser_t *parser, yp_location_t *location, yp_buffer_t *buffer) {
+    assert(location->start);
+    assert(location->end);
+    assert(location->start <= location->end);
+
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(location->start - parser->start));
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(location->end - location->start));
+}
+
+static void
+yp_serialize_string(yp_parser_t *parser, yp_string_t *string, yp_buffer_t *buffer) {
+    switch (string->type) {
+        case YP_STRING_SHARED: {
+            yp_buffer_append_u8(buffer, 1);
+            yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(yp_string_source(string) - parser->start));
+            yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_string_length(string)));
+            break;
+        }
+        case YP_STRING_OWNED:
+        case YP_STRING_CONSTANT: {
+            uint32_t length = yp_sizet_to_u32(yp_string_length(string));
+            yp_buffer_append_u8(buffer, 2);
+            yp_buffer_append_u32(buffer, length);
+            yp_buffer_append_bytes(buffer, yp_string_source(string), length);
+            break;
+        }
+        case YP_STRING_MAPPED:
+            assert(false && "Cannot serialize mapped strings.");
+            break;
+    }
+}
+
+void
+yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
+    yp_buffer_append_u8(buffer, (uint8_t) YP_NODE_TYPE(node));
+
+    size_t offset = buffer->length;
+
+    yp_serialize_location(parser, &node->location, buffer);
+
+    switch (YP_NODE_TYPE(node)) {
+        // We do not need to serialize a ScopeNode ever as
+        // it is not part of the AST
+        case YP_SCOPE_NODE:
+            return;
+        <%- nodes.each do |node| -%>
+        case <%= node.type %>: {
+            <%- if node.needs_serialized_length? -%>
+            // serialize length
+            // encoding of location u32s make us need to save this offset.
+            size_t length_offset = buffer->length;
+            yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
+            <%- end -%>
+            <%- node.fields.each do |field| -%>
+            <%- case field -%>
+            <%- when YARP::NodeField -%>
+            yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
+            <%- when YARP::OptionalNodeField -%>
+            if (((yp_<%= node.human %>_t *)node)-><%= field.name %> == NULL) {
+                yp_buffer_append_u8(buffer, 0);
+            } else {
+                yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
+            }
+            <%- when YARP::StringField -%>
+            yp_serialize_string(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
+            <%- when YARP::NodeListField -%>
+            uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size);
+            yp_buffer_append_u32(buffer, <%= field.name %>_size);
+            for (uint32_t index = 0; index < <%= field.name %>_size; index++) {
+                yp_serialize_node(parser, (yp_node_t *) ((yp_<%= node.human %>_t *)node)-><%= field.name %>.nodes[index], buffer);
+            }
+            <%- when YARP::ConstantField, YARP::OptionalConstantField -%>
+            yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>));
+            <%- when YARP::ConstantListField -%>
+            uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size);
+            yp_buffer_append_u32(buffer, <%= field.name %>_size);
+            for (uint32_t index = 0; index < <%= field.name %>_size; index++) {
+                yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]));
+            }
+            <%- when YARP::LocationField -%>
+            <%- if field.should_be_serialized? -%>
+            yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
+            <%- end -%>
+            <%- when YARP::OptionalLocationField -%>
+            <%- if field.should_be_serialized? -%>
+            if (((yp_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) {
+                yp_buffer_append_u8(buffer, 0);
+            } else {
+                yp_buffer_append_u8(buffer, 1);
+                yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer);
+            }
+            <%- end -%>
+            <%- when YARP::UInt32Field -%>
+            yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= field.name %>);
+            <%- when YARP::FlagsField -%>
+            yp_buffer_append_u32(buffer, node->flags >> <%= YARP::COMMON_FLAGS %>);
+            <%- else -%>
+            <%- raise -%>
+            <%- end -%>
+            <%- end -%>
+            <%- if node.needs_serialized_length? -%>
+            // serialize length
+            uint32_t length = yp_sizet_to_u32(buffer->length - offset - sizeof(uint32_t));
+            memcpy(buffer->value + length_offset, &length, sizeof(uint32_t));
+            <%- end -%>
+            break;
+        }
+        <%- end -%>
+    }
+}
+
+static void
+yp_serialize_comment(yp_parser_t *parser, yp_comment_t *comment, yp_buffer_t *buffer) {
+    // serialize type
+    yp_buffer_append_u8(buffer, (uint8_t) comment->type);
+
+    // serialize location
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->start - parser->start));
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->end - comment->start));
+}
+
+static void
+yp_serialize_comment_list(yp_parser_t *parser, yp_list_t *list, yp_buffer_t *buffer) {
+    yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(list)));
+
+    yp_comment_t *comment;
+    for (comment = (yp_comment_t *) list->head; comment != NULL; comment = (yp_comment_t *) comment->node.next) {
+        yp_serialize_comment(parser, comment, buffer);
+    }
+}
+
+static void
+yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, yp_buffer_t *buffer) {
+    // serialize message
+    size_t message_length = strlen(diagnostic->message);
+    yp_buffer_append_u32(buffer, yp_sizet_to_u32(message_length));
+    yp_buffer_append_str(buffer, diagnostic->message, message_length);
+
+    // serialize location
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(diagnostic->start - parser->start));
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(diagnostic->end - diagnostic->start));
+}
+
+static void
+yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t *list, yp_buffer_t *buffer) {
+    yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(list)));
+
+    yp_diagnostic_t *diagnostic;
+    for (diagnostic = (yp_diagnostic_t *) list->head; diagnostic != NULL; diagnostic = (yp_diagnostic_t *) diagnostic->node.next) {
+        yp_serialize_diagnostic(parser, diagnostic, buffer);
+    }
+}
+
+static void
+yp_serialize_encoding(yp_encoding_t *encoding, yp_buffer_t *buffer) {
+    size_t encoding_length = strlen(encoding->name);
+    yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length));
+    yp_buffer_append_str(buffer, encoding->name, encoding_length);
+}
+
+#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
+void
+yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
+    yp_serialize_encoding(&parser->encoding, buffer);
+    yp_serialize_comment_list(parser, &parser->comment_list, buffer);
+    yp_serialize_diagnostic_list(parser, &parser->error_list, buffer);
+    yp_serialize_diagnostic_list(parser, &parser->warning_list, buffer);
+
+    // Here we're going to leave space for the offset of the constant pool in
+    // the buffer.
+    size_t offset = buffer->length;
+    yp_buffer_append_zeroes(buffer, 4);
+
+    // Next, encode the length of the constant pool.
+    yp_buffer_append_u32(buffer, parser->constant_pool.size);
+
+    // Now we're going to serialize the content of the node.
+    yp_serialize_node(parser, node, buffer);
+
+    // Now we're going to serialize the offset of the constant pool back where
+    // we left space for it.
+    uint32_t length = yp_sizet_to_u32(buffer->length);
+    memcpy(buffer->value + offset, &length, sizeof(uint32_t));
+
+    // Now we're going to serialize the constant pool.
+    offset = buffer->length;
+    yp_buffer_append_zeroes(buffer, parser->constant_pool.size * 8);
+
+    yp_constant_t *constant;
+    for (uint32_t index = 0; index < parser->constant_pool.capacity; index++) {
+        constant = &parser->constant_pool.constants[index];
+
+        // If we find a constant at this index, serialize it at the correct
+        // index in the buffer.
+        if (constant->id != 0) {
+            size_t buffer_offset = offset + ((((size_t) constant->id) - 1) * 8);
+
+            if (constant->owned) {
+                // Since this is an owned constant, we are going to write its
+                // contents into the buffer after the constant pool. So
+                // effectively in place of the source offset, we have a buffer
+                // offset. We will add a leading 1 to indicate that this is a
+                // buffer offset.
+                uint32_t content_offset = yp_sizet_to_u32(buffer->length);
+                uint32_t owned_mask = (uint32_t) (1 << 31);
+
+                assert(content_offset < owned_mask);
+                content_offset |= owned_mask;
+
+                memcpy(buffer->value + buffer_offset, &content_offset, 4);
+                yp_buffer_append_bytes(buffer, constant->start, constant->length);
+            } else {
+                // Since this is a shared constant, we are going to write its
+                // source offset directly into the buffer.
+                uint32_t source_offset = yp_ptrdifft_to_u32(constant->start - parser->start);
+                memcpy(buffer->value + buffer_offset, &source_offset, 4);
+            }
+
+            // Now we can write the length of the constant into the buffer.
+            uint32_t constant_length = yp_sizet_to_u32(constant->length);
+            memcpy(buffer->value + buffer_offset + 4, &constant_length, 4);
+        }
+    }
+}
+
+static void
+serialize_token(void *data, yp_parser_t *parser, yp_token_t *token) {
+    yp_buffer_t *buffer = (yp_buffer_t *) data;
+
+    yp_buffer_append_u32(buffer, token->type);
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->start - parser->start));
+    yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->end - token->start));
+    yp_buffer_append_u32(buffer, parser->lex_state);
+}
+
+YP_EXPORTED_FUNCTION void
+yp_lex_serialize(const uint8_t *source, size_t size, const char *filepath, yp_buffer_t *buffer) {
+    yp_parser_t parser;
+    yp_parser_init(&parser, source, size, filepath);
+
+    yp_lex_callback_t lex_callback = (yp_lex_callback_t) {
+        .data = (void *) buffer,
+        .callback = serialize_token,
+    };
+
+    parser.lex_callback = &lex_callback;
+    yp_node_t *node = yp_parse(&parser);
+
+    // Append 0 to mark end of tokens
+    yp_buffer_append_u8(buffer, 0);
+
+    yp_serialize_encoding(&parser.encoding, buffer);
+    yp_serialize_comment_list(&parser, &parser.comment_list, buffer);
+    yp_serialize_diagnostic_list(&parser, &parser.error_list, buffer);
+    yp_serialize_diagnostic_list(&parser, &parser.warning_list, buffer);
+
+    yp_node_destroy(&parser, node);
+    yp_parser_free(&parser);
+}
+
+// Parse and serialize both the AST and the tokens represented by the given
+// source to the given buffer.
+YP_EXPORTED_FUNCTION void
+yp_parse_lex_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata) {
+    yp_parser_t parser;
+    yp_parser_init(&parser, source, size, NULL);
+    if (metadata) yp_parser_metadata(&parser, metadata);
+
+    yp_lex_callback_t lex_callback = (yp_lex_callback_t) {
+        .data = (void *) buffer,
+        .callback = serialize_token,
+    };
+
+    parser.lex_callback = &lex_callback;
+    yp_node_t *node = yp_parse(&parser);
+
+    yp_buffer_append_u8(buffer, 0);
+    yp_serialize(&parser, node, buffer);
+
+    yp_node_destroy(&parser, node);
+    yp_parser_free(&parser);
+}
diff --git a/prism/templates/src/token_type.c.erb b/prism/templates/src/token_type.c.erb
new file mode 100644
index 0000000000..d861352eec
--- /dev/null
+++ b/prism/templates/src/token_type.c.erb
@@ -0,0 +1,18 @@
+#include <string.h>
+
+#include "yarp/ast.h"
+
+// Returns a string representation of the given token type.
+YP_EXPORTED_FUNCTION const char *
+yp_token_type_to_str(yp_token_type_t token_type)
+{
+    switch (token_type) {
+<%- tokens.each do |token| -%>
+        case YP_TOKEN_<%= token.name %>:
+            return "<%= token.name %>";
+<%- end -%>
+        case YP_TOKEN_MAXIMUM:
+            return "MAXIMUM";
+    }
+    return "\0";
+}