[ruby/prism] Wire up options through the FFI API

https://github.com/ruby/prism/commit/f0aa8ad93b

lib/prism.rb

@@ -35,13 +35,15 @@ module Prism
   private_constant :LexRipper
 
   # :call-seq:
-  #   Prism::lex_compat(source, filepath = "") -> Array
+  #   Prism::lex_compat(source, **options) -> Array
   #
   # Returns an array of tokens that closely resembles that of the Ripper lexer.
   # The only difference is that since we don't keep track of lexer state in the
   # same way, it's going to always return the NONE state.
-  def self.lex_compat(source, filepath = "")
-    LexCompat.new(source, filepath).result
+  #
+  # For supported options, see Prism::parse.
+  def self.lex_compat(source, **options)
+    LexCompat.new(source, **options).result
   end
 
   # :call-seq:

lib/prism/debug.rb

@@ -187,13 +187,5 @@ module Prism
     def self.newlines(source)
       Prism.parse(source).source.offsets
     end
-
-    # :call-seq:
-    #   Debug::parse_serialize_file(filepath) -> dumped
-    #
-    # For the given file, parse the AST and dump it to a string.
-    def self.parse_serialize_file(filepath)
-      parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
-    end
   end
 end

lib/prism/ffi.rb

@@ -167,15 +167,6 @@ module Prism
         end
       end
     end
-
-    # Dump the given source into a serialized format.
-    def self.dump_internal(source, source_size, filepath)
-      PrismBuffer.with do |buffer|
-        metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-        pm_parse_serialize(source, source_size, buffer.pointer, metadata)
-        buffer.read
-      end
-    end
   end
 
   # Mark the LibRubyParser module as private as it should only be called through
@@ -185,93 +176,152 @@ module Prism
   # The version constant is set by reading the result of calling pm_version.
   VERSION = LibRubyParser.pm_version.read_string
 
-  # Mirror the Prism.dump API by using the serialization API.
-  def self.dump(code, filepath = nil)
-    LibRubyParser.dump_internal(code, code.bytesize, filepath)
-  end
-
-  # Mirror the Prism.dump_file API by using the serialization API.
-  def self.dump_file(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      LibRubyParser.dump_internal(string.source, string.length, filepath)
+  class << self
+    # Mirror the Prism.dump API by using the serialization API.
+    def dump(code, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        LibRubyParser.pm_parse_serialize(code, code.bytesize, buffer.pointer, dump_options(options))
+        buffer.read
+      end
     end
-  end
 
-  # Mirror the Prism.lex API by using the serialization API.
-  def self.lex(code, filepath = nil)
-    LibRubyParser::PrismBuffer.with do |buffer|
-      LibRubyParser.pm_lex_serialize(code, code.bytesize, filepath, buffer.pointer)
-      Serialize.load_tokens(Source.new(code), buffer.read)
+    # Mirror the Prism.dump_file API by using the serialization API.
+    def dump_file(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        dump(string.read, **options, filepath: filepath)
+      end
     end
-  end
 
-  # Mirror the Prism.lex_file API by using the serialization API.
-  def self.lex_file(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      lex(string.read, filepath)
+    # Mirror the Prism.lex API by using the serialization API.
+    def lex(code, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        LibRubyParser.pm_lex_serialize(code, code.bytesize, dump_options(options), buffer.pointer)
+        Serialize.load_tokens(Source.new(code), buffer.read)
+      end
     end
-  end
 
-  # Mirror the Prism.parse API by using the serialization API.
-  def self.parse(code, filepath = nil)
-    Prism.load(code, dump(code, filepath))
-  end
-
-  # Mirror the Prism.parse_file API by using the serialization API. This uses
-  # native strings instead of Ruby strings because it allows us to use mmap when
-  # it is available.
-  def self.parse_file(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      parse(string.read, filepath)
+    # Mirror the Prism.lex_file API by using the serialization API.
+    def lex_file(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        lex(string.read, **options, filepath: filepath)
+      end
     end
-  end
 
-  # Mirror the Prism.parse_comments API by using the serialization API.
-  def self.parse_comments(code, filepath = nil)
-    LibRubyParser::PrismBuffer.with do |buffer|
-      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-      LibRubyParser.pm_parse_serialize_comments(code, code.bytesize, buffer.pointer, metadata)
-
-      source = Source.new(code)
-      loader = Serialize::Loader.new(source, buffer.read)
-
-      loader.load_header
-      loader.load_force_encoding
-      loader.load_comments
+    # Mirror the Prism.parse API by using the serialization API.
+    def parse(code, **options)
+      Prism.load(code, dump(code, **options))
     end
-  end
 
-  # Mirror the Prism.parse_file_comments API by using the serialization
-  # API. This uses native strings instead of Ruby strings because it allows us
-  # to use mmap when it is available.
-  def self.parse_file_comments(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      parse_comments(string.read, filepath)
+    # Mirror the Prism.parse_file API by using the serialization API. This uses
+    # native strings instead of Ruby strings because it allows us to use mmap when
+    # it is available.
+    def parse_file(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        parse(string.read, **options, filepath: filepath)
+      end
     end
-  end
 
-  # Mirror the Prism.parse_lex API by using the serialization API.
-  def self.parse_lex(code, filepath = nil)
-    LibRubyParser::PrismBuffer.with do |buffer|
-      metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath
-      LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata)
+    # Mirror the Prism.parse_comments API by using the serialization API.
+    def parse_comments(code, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        LibRubyParser.pm_parse_serialize_comments(code, code.bytesize, buffer.pointer, dump_options(options))
 
-      source = Source.new(code)
-      loader = Serialize::Loader.new(source, buffer.read)
+        source = Source.new(code)
+        loader = Serialize::Loader.new(source, buffer.read)
 
-      tokens = loader.load_tokens
-      node, comments, magic_comments, errors, warnings = loader.load_nodes
-
-      tokens.each { |token,| token.value.force_encoding(loader.encoding) }
-
-      ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
+        loader.load_header
+        loader.load_force_encoding
+        loader.load_comments
+      end
     end
-  end
 
-  # Mirror the Prism.parse_lex_file API by using the serialization API.
-  def self.parse_lex_file(filepath)
-    LibRubyParser::PrismString.with(filepath) do |string|
-      parse_lex(string.read, filepath)
+    # Mirror the Prism.parse_file_comments API by using the serialization
+    # API. This uses native strings instead of Ruby strings because it allows us
+    # to use mmap when it is available.
+    def parse_file_comments(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        parse_comments(string.read, **options, filepath: filepath)
+      end
+    end
+
+    # Mirror the Prism.parse_lex API by using the serialization API.
+    def parse_lex(code, **options)
+      LibRubyParser::PrismBuffer.with do |buffer|
+        LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, dump_options(options))
+
+        source = Source.new(code)
+        loader = Serialize::Loader.new(source, buffer.read)
+
+        tokens = loader.load_tokens
+        node, comments, magic_comments, errors, warnings = loader.load_nodes
+
+        tokens.each { |token,| token.value.force_encoding(loader.encoding) }
+        ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
+      end
+    end
+
+    # Mirror the Prism.parse_lex_file API by using the serialization API.
+    def parse_lex_file(filepath, **options)
+      LibRubyParser::PrismString.with(filepath) do |string|
+        parse_lex(string.read, **options, filepath: filepath)
+      end
+    end
+
+    private
+
+    # Convert the given options into a serialized options string.
+    def dump_options(options)
+      template = +""
+      values = []
+
+      template << "L"
+      if (filepath = options[:filepath])
+        values.push(filepath.bytesize, filepath.b)
+        template << "A*"
+      else
+        values << 0
+      end
+
+      template << "L"
+      values << options.fetch(:line, 0)
+
+      template << "L"
+      if (encoding = options[:encoding])
+        name = encoding.name
+        values.push(name.bytesize, name.b)
+        template << "A*"
+      else
+        values << 0
+      end
+
+      template << "C"
+      values << (options.fetch(:frozen_string_literal, false) ? 1 : 0)
+
+      template << "C"
+      values << (options.fetch(:suppress_warnings, false) ? 1 : 0)
+
+      template << "L"
+      if (scopes = options[:scopes])
+        values << scopes.length
+
+        scopes.each do |scope|
+          template << "L"
+          values << scope.length
+
+          scope.each do |local|
+            name = local.name
+            template << "L"
+            values << name.bytesize
+
+            template << "A*"
+            values << name.b
+          end
+        end
+      else
+        values << 0
+      end
+
+      values.pack(template)
     end
   end
 end

lib/prism/lex_compat.rb

@@ -594,11 +594,11 @@ module Prism
 
     private_constant :Heredoc
 
-    attr_reader :source, :filepath
+    attr_reader :source, :options
 
-    def initialize(source, filepath = "")
+    def initialize(source, **options)
       @source = source
-      @filepath = filepath || ""
+      @options = options
     end
 
     def result
@@ -607,7 +607,7 @@ module Prism
       state = :default
       heredoc_stack = [[]]
 
-      result = Prism.lex(source, filepath: @filepath)
+      result = Prism.lex(source, **options)
       result_value = result.value
       previous_state = nil
 

prism/extension.c

@@ -268,7 +268,7 @@ dump_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -561,7 +561,7 @@ lex_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -672,7 +672,7 @@ parse_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -746,7 +746,7 @@ parse_file_comments(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -815,7 +815,7 @@ parse_lex_file(int argc, VALUE *argv, VALUE self) {
     extract_options(&options, filepath, keywords);
 
     pm_string_t input;
-    if (!pm_string_mapped_init(&input, options.filepath)) {
+    if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) {
         pm_options_free(&options);
         return Qnil;
     }
@@ -913,30 +913,6 @@ profile_file(VALUE self, VALUE filepath) {
     return Qnil;
 }
 
-/**
- * call-seq:
- *   Debug::parse_serialize_file_metadata(filepath, metadata) -> dumped
- *
- * Parse the file and serialize the result. This is mostly used to test this
- * path since it is used by client libraries.
- */
-static VALUE
-parse_serialize_file_metadata(VALUE self, VALUE filepath, VALUE metadata) {
-    pm_string_t input;
-    pm_buffer_t buffer;
-    pm_buffer_init(&buffer);
-
-    const char *checked = check_string(filepath);
-    if (!pm_string_mapped_init(&input, checked)) return Qnil;
-
-    pm_parse_serialize(pm_string_source(&input), pm_string_length(&input), &buffer, check_string(metadata));
-    VALUE result = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer));
-
-    pm_string_free(&input);
-    pm_buffer_free(&buffer);
-    return result;
-}
-
 /**
  * call-seq:
  *   Debug::inspect_node(source) -> inspected
@@ -1039,7 +1015,6 @@ Init_prism(void) {
     rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1);
     rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1);
     rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1);
-    rb_define_singleton_method(rb_cPrismDebug, "parse_serialize_file_metadata", parse_serialize_file_metadata, 2);
     rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1);
 
     // Next, initialize the other APIs.

prism/options.c

@@ -5,7 +5,7 @@
  */
 PRISM_EXPORTED_FUNCTION void
 pm_options_filepath_set(pm_options_t *options, const char *filepath) {
-    options->filepath = filepath;
+    pm_string_constant_init(&options->filepath, filepath, strlen(filepath));
 }
 
 /**
@@ -13,7 +13,7 @@ pm_options_filepath_set(pm_options_t *options, const char *filepath) {
  */
 PRISM_EXPORTED_FUNCTION void
 pm_options_encoding_set(pm_options_t *options, const char *encoding) {
-    options->encoding = encoding;
+    pm_string_constant_init(&options->encoding, encoding, strlen(encoding));
 }
 
 /**
@@ -82,6 +82,9 @@ pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index) {
  */
 PRISM_EXPORTED_FUNCTION void
 pm_options_free(pm_options_t *options) {
+    pm_string_free(&options->filepath);
+    pm_string_free(&options->encoding);
+
     for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) {
         pm_options_scope_t *scope = &options->scopes[scope_index];
 
@@ -94,3 +97,74 @@ pm_options_free(pm_options_t *options) {
 
     free(options->scopes);
 }
+
+/**
+ * Read a 32-bit unsigned integer from a pointer. This function is used to read
+ * the options that are passed into the parser from the Ruby implementation. It
+ * handles aligned and unaligned reads.
+ */
+static uint32_t
+pm_options_read_u32(const char *data) {
+    if (((uintptr_t) data) % sizeof(uint32_t) == 0) {
+        return *((uint32_t *) data);
+    } else {
+        uint32_t value;
+        memcpy(&value, data, sizeof(uint32_t));
+        return value;
+    }
+}
+
+/**
+ * Deserialize an options struct from the given binary string. This is used to
+ * pass options to the parser from an FFI call so that consumers of the library
+ * from an FFI perspective don't have to worry about the structure of our
+ * options structs. Since the source of these calls will be from Ruby
+ * implementation internals we assume it is from a trusted source.
+ */
+void
+pm_options_read(pm_options_t *options, const char *data) {
+    uint32_t filepath_length = pm_options_read_u32(data);
+    data += 4;
+
+    if (filepath_length > 0) {
+        pm_string_constant_init(&options->filepath, data, filepath_length);
+        data += filepath_length;
+    }
+
+    options->line = pm_options_read_u32(data);
+    data += 4;
+
+    uint32_t encoding_length = pm_options_read_u32(data);
+    data += 4;
+
+    if (encoding_length > 0) {
+        pm_string_constant_init(&options->encoding, data, encoding_length);
+        data += encoding_length;
+    }
+
+    options->frozen_string_literal = *data++;
+    options->suppress_warnings = *data++;
+
+    uint32_t scopes_count = pm_options_read_u32(data);
+    data += 4;
+
+    if (scopes_count > 0) {
+        pm_options_scopes_init(options, scopes_count);
+
+        for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) {
+            uint32_t locals_count = pm_options_read_u32(data);
+            data += 4;
+
+            pm_options_scope_t *scope = &options->scopes[scope_index];
+            pm_options_scope_init(scope, locals_count);
+
+            for (size_t local_index = 0; local_index < locals_count; local_index++) {
+                uint32_t local_length = pm_options_read_u32(data);
+                data += 4;
+
+                pm_string_constant_init(&scope->locals[local_index], data, local_length);
+                data += local_length;
+            }
+        }
+    }
+}

prism/options.h

@@ -29,13 +29,7 @@ typedef struct pm_options_scope {
  */
 typedef struct {
     /** The name of the file that is currently being parsed. */
-    const char *filepath;
-
-    /**
-     * The name of the encoding that the source file is in. Note that this must
-     * correspond to a name that can be found with Encoding.find in Ruby.
-     */
-    const char *encoding;
+    pm_string_t filepath;
 
     /**
      * The line within the file that the parse starts on. This value is
@@ -43,6 +37,12 @@ typedef struct {
      */
     uint32_t line;
 
+    /**
+     * The name of the encoding that the source file is in. Note that this must
+     * correspond to a name that can be found with Encoding.find in Ruby.
+     */
+    pm_string_t encoding;
+
     /**
      * The number of scopes surrounding the code that is being parsed.
      */
@@ -72,17 +72,7 @@ typedef struct {
  * @param options The options struct to set the filepath on.
  * @param filepath The filepath to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_filepath_set(pm_options_t *options, const char *filepath);
-
-/**
- * Set the encoding option on the given options struct.
- *
- * @param options The options struct to set the encoding on.
- * @param encoding The encoding to set.
- */
-PRISM_EXPORTED_FUNCTION void
-pm_options_encoding_set(pm_options_t *options, const char *encoding);
+PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, const char *filepath);
 
 /**
  * Set the line option on the given options struct.
@@ -90,8 +80,15 @@ pm_options_encoding_set(pm_options_t *options, const char *encoding);
  * @param options The options struct to set the line on.
  * @param line The line to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_line_set(pm_options_t *options, uint32_t line);
+PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, uint32_t line);
+
+/**
+ * Set the encoding option on the given options struct.
+ *
+ * @param options The options struct to set the encoding on.
+ * @param encoding The encoding to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding);
 
 /**
  * Set the frozen string literal option on the given options struct.
@@ -99,8 +96,7 @@ pm_options_line_set(pm_options_t *options, uint32_t line);
  * @param options The options struct to set the frozen string literal value on.
  * @param frozen_string_literal The frozen string literal value to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
+PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
 
 /**
  * Set the suppress warnings option on the given options struct.
@@ -108,8 +104,7 @@ pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_l
  * @param options The options struct to set the suppress warnings value on.
  * @param suppress_warnings The suppress warnings value to set.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
+PRISM_EXPORTED_FUNCTION void pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
 
 /**
  * Allocate and zero out the scopes array on the given options struct.
@@ -117,8 +112,7 @@ pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
  * @param options The options struct to initialize the scopes array on.
  * @param scopes_count The number of scopes to allocate.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
+PRISM_EXPORTED_FUNCTION void pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
 
 /**
  * Return a pointer to the scope at the given index within the given options.
@@ -127,8 +121,7 @@ pm_options_scopes_init(pm_options_t *options, size_t scopes_count);
  * @param index The index of the scope to get.
  * @return A pointer to the scope at the given index.
  */
-PRISM_EXPORTED_FUNCTION const pm_options_scope_t *
-pm_options_scope_get(const pm_options_t *options, size_t index);
+PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm_options_t *options, size_t index);
 
 /**
  * Create a new options scope struct. This will hold a set of locals that are in
@@ -137,8 +130,7 @@ pm_options_scope_get(const pm_options_t *options, size_t index);
  * @param scope The scope struct to initialize.
  * @param locals_count The number of locals to allocate.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
+PRISM_EXPORTED_FUNCTION void pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
 
 /**
  * Return a pointer to the local at the given index within the given scope.
@@ -147,15 +139,66 @@ pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count);
  * @param index The index of the local to get.
  * @return A pointer to the local at the given index.
  */
-PRISM_EXPORTED_FUNCTION const pm_string_t *
-pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
+PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
 
 /**
  * Free the internal memory associated with the options.
  *
  * @param options The options struct whose internal memory should be freed.
  */
-PRISM_EXPORTED_FUNCTION void
-pm_options_free(pm_options_t *options);
+PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
+
+/**
+ * Deserialize an options struct from the given binary string. This is used to
+ * pass options to the parser from an FFI call so that consumers of the library
+ * from an FFI perspective don't have to worry about the structure of our
+ * options structs. Since the source of these calls will be from Ruby
+ * implementation internals we assume it is from a trusted source.
+ *
+ * `data` is assumed to be a valid pointer pointing to well-formed data. The
+ * layout of this data should be the same every time, and is described below:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the length of the filepath |
+ * | ...     | the filepath bytes         |
+ * | 4       | the line number            |
+ * | 4       | the length the encoding    |
+ * | ...     | the encoding bytes         |
+ * | 1       | frozen string literal      |
+ * | 1       | suppress warnings          |
+ * | 4       | the number of scopes       |
+ * | ...     | the scopes                 |
+ *
+ * Each scope is layed out as follows:
+ * 
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the number of locals       |
+ * | ...     | the locals                 |
+ *
+ * Each local is layed out as follows:
+ *
+ * | # bytes | field                      |
+ * | ------- | -------------------------- |
+ * | 4       | the length of the local    |
+ * | ...     | the local bytes            |
+ *
+ * Some additional things to note about this layout:
+ *
+ * * The filepath can have a length of 0, in which case we'll consider it an
+ *   empty string.
+ * * The line number should be 0-indexed.
+ * * The encoding can have a length of 0, in which case we'll use the default
+ *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
+ *   encoding that can be passed to `Encoding.find` in Ruby.
+ * * The frozen string literal and suppress warnings fields are booleans, so
+ *   their values should be either 0 or 1.
+ * * The number of scopes can be 0.
+ *
+ * @param options The options struct to deserialize into.
+ * @param data The binary string to deserialize from.
+ */
+void pm_options_read(pm_options_t *options, const char *data);
 
 #endif

prism/prism.c

@@ -16391,11 +16391,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm
     // If options were provided to this parse, establish them here.
     if (options != NULL) {
         // filepath option
-        if (options->filepath == NULL) {
-            pm_string_constant_init(&parser->filepath_string, "", 0);
-        } else {
-            pm_string_constant_init(&parser->filepath_string, options->filepath, strlen(options->filepath));
-        }
+        parser->filepath_string = options->filepath;
 
         // line option
         if (options->line > 0) {
@@ -16561,10 +16557,12 @@ pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) {
  * buffer.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) {
+pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) {
+    pm_options_t options = { 0 };
+    if (data != NULL) pm_options_read(&options, data);
+
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, NULL);
-    if (metadata) pm_parser_metadata(&parser, metadata);
+    pm_parser_init(&parser, source, size, &options);
 
     pm_node_t *node = pm_parse(&parser);
 
@@ -16574,16 +16572,19 @@ pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, cons
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
+    pm_options_free(&options);
 }
 
 /**
  * Parse and serialize the comments in the given source to the given buffer.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) {
+pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) {
+    pm_options_t options = { 0 };
+    if (data != NULL) pm_options_read(&options, data);
+
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, NULL);
-    if (metadata) pm_parser_metadata(&parser, metadata);
+    pm_parser_init(&parser, source, size, &options);
 
     pm_node_t *node = pm_parse(&parser);
     pm_serialize_header(buffer);
@@ -16592,6 +16593,7 @@ pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buf
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
+    pm_options_free(&options);
 }
 
 #undef PM_CASE_KEYWORD

prism/prism.h

@@ -123,30 +123,15 @@ void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buf
  */
 PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
 
-/**
- * Process any additional metadata being passed into a call to the parser via
- * the pm_parse_serialize function. Since the source of these calls will be from
- * Ruby implementation internals we assume it is from a trusted source.
- *
- * Currently, this is only passing in variable scoping surrounding an eval, but
- * eventually it will be extended to hold any additional metadata.  This data
- * is serialized to reduce the calling complexity for a foreign function call
- * vs a foreign runtime making a bindable in-memory version of a C structure.
- *
- * @param parser The parser to process the metadata for.
- * @param metadata The metadata to process.
- */
-void pm_parser_metadata(pm_parser_t *parser, const char *metadata);
-
 /**
  * Parse the given source to the AST and serialize the AST to the given buffer.
  *
  * @param source The source to parse.
  * @param size The size of the source.
  * @param buffer The buffer to serialize to.
- * @param metadata The optional metadata to pass to the parser.
+ * @param data The optional data to pass to the parser.
  */
-PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data);
 
 /**
  * Parse and serialize the comments in the given source to the given buffer.
@@ -154,9 +139,9 @@ PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t si
  * @param source The source to parse.
  * @param size The size of the source.
  * @param buffer The buffer to serialize to.
- * @param metadata The optional metadata to pass to the parser.
+ * @param data The optional data to pass to the parser.
  */
-PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data);
 
 /**
  * Lex the given source and serialize to the given buffer.

prism/templates/src/serialize.c.erb

@@ -287,9 +287,9 @@ serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) {
  * Lex the given source and serialize to the given buffer.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_buffer_t *buffer) {
+pm_lex_serialize(const uint8_t *source, size_t size, const char *data, pm_buffer_t *buffer) {
     pm_options_t options = { 0 };
-    pm_options_filepath_set(&options, filepath);
+    if (data != NULL) pm_options_read(&options, data);
 
     pm_parser_t parser;
     pm_parser_init(&parser, source, size, &options);
@@ -321,10 +321,12 @@ pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_bu
  * source to the given buffer.
  */
 PRISM_EXPORTED_FUNCTION void
-pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) {
+pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) {
+    pm_options_t options = { 0 };
+    if (data != NULL) pm_options_read(&options, data);
+
     pm_parser_t parser;
-    pm_parser_init(&parser, source, size, NULL);
-    if (metadata) pm_parser_metadata(&parser, metadata);
+    pm_parser_init(&parser, source, size, &options);
 
     pm_lex_callback_t lex_callback = (pm_lex_callback_t) {
         .data = (void *) buffer,
@@ -339,4 +341,5 @@ pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer,
 
     pm_node_destroy(&parser, node);
     pm_parser_free(&parser);
+    pm_options_free(&options);
 }

test/prism/parse_serialize_test.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "test_helper"
-
-return if Prism::BACKEND == :FFI
-
-module Prism
-  class ParseSerializeTest < TestCase
-    def test_parse_serialize
-      dumped = Debug.parse_serialize_file(__FILE__)
-      result = Prism.load(File.read(__FILE__), dumped)
-
-      assert_kind_of ParseResult, result, "Expected the return value to be a ParseResult"
-      assert_equal __FILE__, find_file_node(result)&.filepath, "Expected the filepath to be set correctly"
-    end
-
-    def test_parse_serialize_with_locals
-      filepath = __FILE__
-      metadata = [filepath.bytesize, filepath.b, 1, 1, 1, "foo".b].pack("LA*LLLA*")
-
-      dumped = Debug.parse_serialize_file_metadata(filepath, metadata)
-      result = Prism.load(File.read(__FILE__), dumped)
-
-      assert_kind_of ParseResult, result, "Expected the return value to be a ParseResult"
-    end
-
-    private
-
-    def find_file_node(result)
-      queue = [result.value]
-
-      while (node = queue.shift)
-        return node if node.is_a?(SourceFileNode)
-        queue.concat(node.compact_child_nodes)
-      end
-    end
-  end
-end

test/prism/ruby_api_test.rb

@@ -20,6 +20,19 @@ module Prism
       assert_equal_nodes ast2, ast3
     end
 
+    def test_options
+      assert_equal "", Prism.parse("__FILE__").value.statements.body[0].filepath
+      assert_equal "foo.rb", Prism.parse("__FILE__", filepath: "foo.rb").value.statements.body[0].filepath
+
+      refute Prism.parse("\"foo\"").value.statements.body[0].frozen?
+      assert Prism.parse("\"foo\"", frozen_string_literal: true).value.statements.body[0].frozen?
+      refute Prism.parse("\"foo\"", frozen_string_literal: false).value.statements.body[0].frozen?
+
+      assert_kind_of Prism::CallNode, Prism.parse("foo").value.statements.body[0]
+      assert_kind_of Prism::LocalVariableReadNode, Prism.parse("foo", scopes: [[:foo]]).value.statements.body[0]
+      assert_equal 2, Prism.parse("foo", scopes: [[:foo], []]).value.statements.body[0].depth
+    end
+
     def test_literal_value_method
       assert_equal 123, parse_expression("123").value
       assert_equal 3.14, parse_expression("3.14").value