diff --git a/lib/prism.rb b/lib/prism.rb index 86cdadcdad..350febcaa8 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -35,13 +35,15 @@ module Prism private_constant :LexRipper # :call-seq: - # Prism::lex_compat(source, filepath = "") -> Array + # Prism::lex_compat(source, **options) -> Array # # Returns an array of tokens that closely resembles that of the Ripper lexer. # The only difference is that since we don't keep track of lexer state in the # same way, it's going to always return the NONE state. - def self.lex_compat(source, filepath = "") - LexCompat.new(source, filepath).result + # + # For supported options, see Prism::parse. + def self.lex_compat(source, **options) + LexCompat.new(source, **options).result end # :call-seq: diff --git a/lib/prism/debug.rb b/lib/prism/debug.rb index f573d0958d..f5d19dc3df 100644 --- a/lib/prism/debug.rb +++ b/lib/prism/debug.rb @@ -187,13 +187,5 @@ module Prism def self.newlines(source) Prism.parse(source).source.offsets end - - # :call-seq: - # Debug::parse_serialize_file(filepath) -> dumped - # - # For the given file, parse the AST and dump it to a string. - def self.parse_serialize_file(filepath) - parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L")) - end end end diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index 170ca8b8d7..61ece6a641 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -167,15 +167,6 @@ module Prism end end end - - # Dump the given source into a serialized format. - def self.dump_internal(source, source_size, filepath) - PrismBuffer.with do |buffer| - metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath - pm_parse_serialize(source, source_size, buffer.pointer, metadata) - buffer.read - end - end end # Mark the LibRubyParser module as private as it should only be called through @@ -185,93 +176,152 @@ module Prism # The version constant is set by reading the result of calling pm_version. VERSION = LibRubyParser.pm_version.read_string - # Mirror the Prism.dump API by using the serialization API. - def self.dump(code, filepath = nil) - LibRubyParser.dump_internal(code, code.bytesize, filepath) - end - - # Mirror the Prism.dump_file API by using the serialization API. - def self.dump_file(filepath) - LibRubyParser::PrismString.with(filepath) do |string| - LibRubyParser.dump_internal(string.source, string.length, filepath) + class << self + # Mirror the Prism.dump API by using the serialization API. + def dump(code, **options) + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_parse_serialize(code, code.bytesize, buffer.pointer, dump_options(options)) + buffer.read + end end - end - # Mirror the Prism.lex API by using the serialization API. - def self.lex(code, filepath = nil) - LibRubyParser::PrismBuffer.with do |buffer| - LibRubyParser.pm_lex_serialize(code, code.bytesize, filepath, buffer.pointer) - Serialize.load_tokens(Source.new(code), buffer.read) + # Mirror the Prism.dump_file API by using the serialization API. + def dump_file(filepath, **options) + LibRubyParser::PrismString.with(filepath) do |string| + dump(string.read, **options, filepath: filepath) + end end - end - # Mirror the Prism.lex_file API by using the serialization API. - def self.lex_file(filepath) - LibRubyParser::PrismString.with(filepath) do |string| - lex(string.read, filepath) + # Mirror the Prism.lex API by using the serialization API. + def lex(code, **options) + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_lex_serialize(code, code.bytesize, dump_options(options), buffer.pointer) + Serialize.load_tokens(Source.new(code), buffer.read) + end end - end - # Mirror the Prism.parse API by using the serialization API. - def self.parse(code, filepath = nil) - Prism.load(code, dump(code, filepath)) - end - - # Mirror the Prism.parse_file API by using the serialization API. This uses - # native strings instead of Ruby strings because it allows us to use mmap when - # it is available. - def self.parse_file(filepath) - LibRubyParser::PrismString.with(filepath) do |string| - parse(string.read, filepath) + # Mirror the Prism.lex_file API by using the serialization API. + def lex_file(filepath, **options) + LibRubyParser::PrismString.with(filepath) do |string| + lex(string.read, **options, filepath: filepath) + end end - end - # Mirror the Prism.parse_comments API by using the serialization API. - def self.parse_comments(code, filepath = nil) - LibRubyParser::PrismBuffer.with do |buffer| - metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath - LibRubyParser.pm_parse_serialize_comments(code, code.bytesize, buffer.pointer, metadata) - - source = Source.new(code) - loader = Serialize::Loader.new(source, buffer.read) - - loader.load_header - loader.load_force_encoding - loader.load_comments + # Mirror the Prism.parse API by using the serialization API. + def parse(code, **options) + Prism.load(code, dump(code, **options)) end - end - # Mirror the Prism.parse_file_comments API by using the serialization - # API. This uses native strings instead of Ruby strings because it allows us - # to use mmap when it is available. - def self.parse_file_comments(filepath) - LibRubyParser::PrismString.with(filepath) do |string| - parse_comments(string.read, filepath) + # Mirror the Prism.parse_file API by using the serialization API. This uses + # native strings instead of Ruby strings because it allows us to use mmap when + # it is available. + def parse_file(filepath, **options) + LibRubyParser::PrismString.with(filepath) do |string| + parse(string.read, **options, filepath: filepath) + end end - end - # Mirror the Prism.parse_lex API by using the serialization API. - def self.parse_lex(code, filepath = nil) - LibRubyParser::PrismBuffer.with do |buffer| - metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath - LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata) + # Mirror the Prism.parse_comments API by using the serialization API. + def parse_comments(code, **options) + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_parse_serialize_comments(code, code.bytesize, buffer.pointer, dump_options(options)) - source = Source.new(code) - loader = Serialize::Loader.new(source, buffer.read) + source = Source.new(code) + loader = Serialize::Loader.new(source, buffer.read) - tokens = loader.load_tokens - node, comments, magic_comments, errors, warnings = loader.load_nodes - - tokens.each { |token,| token.value.force_encoding(loader.encoding) } - - ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source) + loader.load_header + loader.load_force_encoding + loader.load_comments + end end - end - # Mirror the Prism.parse_lex_file API by using the serialization API. - def self.parse_lex_file(filepath) - LibRubyParser::PrismString.with(filepath) do |string| - parse_lex(string.read, filepath) + # Mirror the Prism.parse_file_comments API by using the serialization + # API. This uses native strings instead of Ruby strings because it allows us + # to use mmap when it is available. + def parse_file_comments(filepath, **options) + LibRubyParser::PrismString.with(filepath) do |string| + parse_comments(string.read, **options, filepath: filepath) + end + end + + # Mirror the Prism.parse_lex API by using the serialization API. + def parse_lex(code, **options) + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_parse_lex_serialize(code, code.bytesize, buffer.pointer, dump_options(options)) + + source = Source.new(code) + loader = Serialize::Loader.new(source, buffer.read) + + tokens = loader.load_tokens + node, comments, magic_comments, errors, warnings = loader.load_nodes + + tokens.each { |token,| token.value.force_encoding(loader.encoding) } + ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source) + end + end + + # Mirror the Prism.parse_lex_file API by using the serialization API. + def parse_lex_file(filepath, **options) + LibRubyParser::PrismString.with(filepath) do |string| + parse_lex(string.read, **options, filepath: filepath) + end + end + + private + + # Convert the given options into a serialized options string. + def dump_options(options) + template = +"" + values = [] + + template << "L" + if (filepath = options[:filepath]) + values.push(filepath.bytesize, filepath.b) + template << "A*" + else + values << 0 + end + + template << "L" + values << options.fetch(:line, 0) + + template << "L" + if (encoding = options[:encoding]) + name = encoding.name + values.push(name.bytesize, name.b) + template << "A*" + else + values << 0 + end + + template << "C" + values << (options.fetch(:frozen_string_literal, false) ? 1 : 0) + + template << "C" + values << (options.fetch(:suppress_warnings, false) ? 1 : 0) + + template << "L" + if (scopes = options[:scopes]) + values << scopes.length + + scopes.each do |scope| + template << "L" + values << scope.length + + scope.each do |local| + name = local.name + template << "L" + values << name.bytesize + + template << "A*" + values << name.b + end + end + else + values << 0 + end + + values.pack(template) end end end diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index c1f5cfe944..b6d12053a0 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -594,11 +594,11 @@ module Prism private_constant :Heredoc - attr_reader :source, :filepath + attr_reader :source, :options - def initialize(source, filepath = "") + def initialize(source, **options) @source = source - @filepath = filepath || "" + @options = options end def result @@ -607,7 +607,7 @@ module Prism state = :default heredoc_stack = [[]] - result = Prism.lex(source, filepath: @filepath) + result = Prism.lex(source, **options) result_value = result.value previous_state = nil diff --git a/prism/extension.c b/prism/extension.c index 679d83c1be..86221a7bc5 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -268,7 +268,7 @@ dump_file(int argc, VALUE *argv, VALUE self) { extract_options(&options, filepath, keywords); pm_string_t input; - if (!pm_string_mapped_init(&input, options.filepath)) { + if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) { pm_options_free(&options); return Qnil; } @@ -561,7 +561,7 @@ lex_file(int argc, VALUE *argv, VALUE self) { extract_options(&options, filepath, keywords); pm_string_t input; - if (!pm_string_mapped_init(&input, options.filepath)) { + if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) { pm_options_free(&options); return Qnil; } @@ -672,7 +672,7 @@ parse_file(int argc, VALUE *argv, VALUE self) { extract_options(&options, filepath, keywords); pm_string_t input; - if (!pm_string_mapped_init(&input, options.filepath)) { + if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) { pm_options_free(&options); return Qnil; } @@ -746,7 +746,7 @@ parse_file_comments(int argc, VALUE *argv, VALUE self) { extract_options(&options, filepath, keywords); pm_string_t input; - if (!pm_string_mapped_init(&input, options.filepath)) { + if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) { pm_options_free(&options); return Qnil; } @@ -815,7 +815,7 @@ parse_lex_file(int argc, VALUE *argv, VALUE self) { extract_options(&options, filepath, keywords); pm_string_t input; - if (!pm_string_mapped_init(&input, options.filepath)) { + if (!pm_string_mapped_init(&input, (const char *) pm_string_source(&options.filepath))) { pm_options_free(&options); return Qnil; } @@ -913,30 +913,6 @@ profile_file(VALUE self, VALUE filepath) { return Qnil; } -/** - * call-seq: - * Debug::parse_serialize_file_metadata(filepath, metadata) -> dumped - * - * Parse the file and serialize the result. This is mostly used to test this - * path since it is used by client libraries. - */ -static VALUE -parse_serialize_file_metadata(VALUE self, VALUE filepath, VALUE metadata) { - pm_string_t input; - pm_buffer_t buffer; - pm_buffer_init(&buffer); - - const char *checked = check_string(filepath); - if (!pm_string_mapped_init(&input, checked)) return Qnil; - - pm_parse_serialize(pm_string_source(&input), pm_string_length(&input), &buffer, check_string(metadata)); - VALUE result = rb_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer)); - - pm_string_free(&input); - pm_buffer_free(&buffer); - return result; -} - /** * call-seq: * Debug::inspect_node(source) -> inspected @@ -1039,7 +1015,6 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrismDebug, "named_captures", named_captures, 1); rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1); rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1); - rb_define_singleton_method(rb_cPrismDebug, "parse_serialize_file_metadata", parse_serialize_file_metadata, 2); rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1); // Next, initialize the other APIs. diff --git a/prism/options.c b/prism/options.c index 7f45c2026c..84c1fcbb39 100644 --- a/prism/options.c +++ b/prism/options.c @@ -5,7 +5,7 @@ */ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, const char *filepath) { - options->filepath = filepath; + pm_string_constant_init(&options->filepath, filepath, strlen(filepath)); } /** @@ -13,7 +13,7 @@ pm_options_filepath_set(pm_options_t *options, const char *filepath) { */ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding) { - options->encoding = encoding; + pm_string_constant_init(&options->encoding, encoding, strlen(encoding)); } /** @@ -82,6 +82,9 @@ pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index) { */ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options) { + pm_string_free(&options->filepath); + pm_string_free(&options->encoding); + for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) { pm_options_scope_t *scope = &options->scopes[scope_index]; @@ -94,3 +97,74 @@ pm_options_free(pm_options_t *options) { free(options->scopes); } + +/** + * Read a 32-bit unsigned integer from a pointer. This function is used to read + * the options that are passed into the parser from the Ruby implementation. It + * handles aligned and unaligned reads. + */ +static uint32_t +pm_options_read_u32(const char *data) { + if (((uintptr_t) data) % sizeof(uint32_t) == 0) { + return *((uint32_t *) data); + } else { + uint32_t value; + memcpy(&value, data, sizeof(uint32_t)); + return value; + } +} + +/** + * Deserialize an options struct from the given binary string. This is used to + * pass options to the parser from an FFI call so that consumers of the library + * from an FFI perspective don't have to worry about the structure of our + * options structs. Since the source of these calls will be from Ruby + * implementation internals we assume it is from a trusted source. + */ +void +pm_options_read(pm_options_t *options, const char *data) { + uint32_t filepath_length = pm_options_read_u32(data); + data += 4; + + if (filepath_length > 0) { + pm_string_constant_init(&options->filepath, data, filepath_length); + data += filepath_length; + } + + options->line = pm_options_read_u32(data); + data += 4; + + uint32_t encoding_length = pm_options_read_u32(data); + data += 4; + + if (encoding_length > 0) { + pm_string_constant_init(&options->encoding, data, encoding_length); + data += encoding_length; + } + + options->frozen_string_literal = *data++; + options->suppress_warnings = *data++; + + uint32_t scopes_count = pm_options_read_u32(data); + data += 4; + + if (scopes_count > 0) { + pm_options_scopes_init(options, scopes_count); + + for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) { + uint32_t locals_count = pm_options_read_u32(data); + data += 4; + + pm_options_scope_t *scope = &options->scopes[scope_index]; + pm_options_scope_init(scope, locals_count); + + for (size_t local_index = 0; local_index < locals_count; local_index++) { + uint32_t local_length = pm_options_read_u32(data); + data += 4; + + pm_string_constant_init(&scope->locals[local_index], data, local_length); + data += local_length; + } + } + } +} diff --git a/prism/options.h b/prism/options.h index 5abf16a601..6faadc2c47 100644 --- a/prism/options.h +++ b/prism/options.h @@ -29,13 +29,7 @@ typedef struct pm_options_scope { */ typedef struct { /** The name of the file that is currently being parsed. */ - const char *filepath; - - /** - * The name of the encoding that the source file is in. Note that this must - * correspond to a name that can be found with Encoding.find in Ruby. - */ - const char *encoding; + pm_string_t filepath; /** * The line within the file that the parse starts on. This value is @@ -43,6 +37,12 @@ typedef struct { */ uint32_t line; + /** + * The name of the encoding that the source file is in. Note that this must + * correspond to a name that can be found with Encoding.find in Ruby. + */ + pm_string_t encoding; + /** * The number of scopes surrounding the code that is being parsed. */ @@ -72,17 +72,7 @@ typedef struct { * @param options The options struct to set the filepath on. * @param filepath The filepath to set. */ -PRISM_EXPORTED_FUNCTION void -pm_options_filepath_set(pm_options_t *options, const char *filepath); - -/** - * Set the encoding option on the given options struct. - * - * @param options The options struct to set the encoding on. - * @param encoding The encoding to set. - */ -PRISM_EXPORTED_FUNCTION void -pm_options_encoding_set(pm_options_t *options, const char *encoding); +PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, const char *filepath); /** * Set the line option on the given options struct. @@ -90,8 +80,15 @@ pm_options_encoding_set(pm_options_t *options, const char *encoding); * @param options The options struct to set the line on. * @param line The line to set. */ -PRISM_EXPORTED_FUNCTION void -pm_options_line_set(pm_options_t *options, uint32_t line); +PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, uint32_t line); + +/** + * Set the encoding option on the given options struct. + * + * @param options The options struct to set the encoding on. + * @param encoding The encoding to set. + */ +PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, const char *encoding); /** * Set the frozen string literal option on the given options struct. @@ -99,8 +96,7 @@ pm_options_line_set(pm_options_t *options, uint32_t line); * @param options The options struct to set the frozen string literal value on. * @param frozen_string_literal The frozen string literal value to set. */ -PRISM_EXPORTED_FUNCTION void -pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal); +PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal); /** * Set the suppress warnings option on the given options struct. @@ -108,8 +104,7 @@ pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_l * @param options The options struct to set the suppress warnings value on. * @param suppress_warnings The suppress warnings value to set. */ -PRISM_EXPORTED_FUNCTION void -pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings); +PRISM_EXPORTED_FUNCTION void pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings); /** * Allocate and zero out the scopes array on the given options struct. @@ -117,8 +112,7 @@ pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings); * @param options The options struct to initialize the scopes array on. * @param scopes_count The number of scopes to allocate. */ -PRISM_EXPORTED_FUNCTION void -pm_options_scopes_init(pm_options_t *options, size_t scopes_count); +PRISM_EXPORTED_FUNCTION void pm_options_scopes_init(pm_options_t *options, size_t scopes_count); /** * Return a pointer to the scope at the given index within the given options. @@ -127,8 +121,7 @@ pm_options_scopes_init(pm_options_t *options, size_t scopes_count); * @param index The index of the scope to get. * @return A pointer to the scope at the given index. */ -PRISM_EXPORTED_FUNCTION const pm_options_scope_t * -pm_options_scope_get(const pm_options_t *options, size_t index); +PRISM_EXPORTED_FUNCTION const pm_options_scope_t * pm_options_scope_get(const pm_options_t *options, size_t index); /** * Create a new options scope struct. This will hold a set of locals that are in @@ -137,8 +130,7 @@ pm_options_scope_get(const pm_options_t *options, size_t index); * @param scope The scope struct to initialize. * @param locals_count The number of locals to allocate. */ -PRISM_EXPORTED_FUNCTION void -pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count); +PRISM_EXPORTED_FUNCTION void pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count); /** * Return a pointer to the local at the given index within the given scope. @@ -147,15 +139,66 @@ pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count); * @param index The index of the local to get. * @return A pointer to the local at the given index. */ -PRISM_EXPORTED_FUNCTION const pm_string_t * -pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index); +PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index); /** * Free the internal memory associated with the options. * * @param options The options struct whose internal memory should be freed. */ -PRISM_EXPORTED_FUNCTION void -pm_options_free(pm_options_t *options); +PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options); + +/** + * Deserialize an options struct from the given binary string. This is used to + * pass options to the parser from an FFI call so that consumers of the library + * from an FFI perspective don't have to worry about the structure of our + * options structs. Since the source of these calls will be from Ruby + * implementation internals we assume it is from a trusted source. + * + * `data` is assumed to be a valid pointer pointing to well-formed data. The + * layout of this data should be the same every time, and is described below: + * + * | # bytes | field | + * | ------- | -------------------------- | + * | 4 | the length of the filepath | + * | ... | the filepath bytes | + * | 4 | the line number | + * | 4 | the length the encoding | + * | ... | the encoding bytes | + * | 1 | frozen string literal | + * | 1 | suppress warnings | + * | 4 | the number of scopes | + * | ... | the scopes | + * + * Each scope is layed out as follows: + * + * | # bytes | field | + * | ------- | -------------------------- | + * | 4 | the number of locals | + * | ... | the locals | + * + * Each local is layed out as follows: + * + * | # bytes | field | + * | ------- | -------------------------- | + * | 4 | the length of the local | + * | ... | the local bytes | + * + * Some additional things to note about this layout: + * + * * The filepath can have a length of 0, in which case we'll consider it an + * empty string. + * * The line number should be 0-indexed. + * * The encoding can have a length of 0, in which case we'll use the default + * encoding (UTF-8). If it's not 0, it should correspond to a name of an + * encoding that can be passed to `Encoding.find` in Ruby. + * * The frozen string literal and suppress warnings fields are booleans, so + * their values should be either 0 or 1. + * * The number of scopes can be 0. + * + * @param options The options struct to deserialize into. + * @param data The binary string to deserialize from. + */ +void pm_options_read(pm_options_t *options, const char *data); #endif diff --git a/prism/prism.c b/prism/prism.c index fc836a956d..2c04860de1 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -16391,11 +16391,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm // If options were provided to this parse, establish them here. if (options != NULL) { // filepath option - if (options->filepath == NULL) { - pm_string_constant_init(&parser->filepath_string, "", 0); - } else { - pm_string_constant_init(&parser->filepath_string, options->filepath, strlen(options->filepath)); - } + parser->filepath_string = options->filepath; // line option if (options->line > 0) { @@ -16561,10 +16557,12 @@ pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { * buffer. */ PRISM_EXPORTED_FUNCTION void -pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) { +pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) { + pm_options_t options = { 0 }; + if (data != NULL) pm_options_read(&options, data); + pm_parser_t parser; - pm_parser_init(&parser, source, size, NULL); - if (metadata) pm_parser_metadata(&parser, metadata); + pm_parser_init(&parser, source, size, &options); pm_node_t *node = pm_parse(&parser); @@ -16574,16 +16572,19 @@ pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, cons pm_node_destroy(&parser, node); pm_parser_free(&parser); + pm_options_free(&options); } /** * Parse and serialize the comments in the given source to the given buffer. */ PRISM_EXPORTED_FUNCTION void -pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) { +pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) { + pm_options_t options = { 0 }; + if (data != NULL) pm_options_read(&options, data); + pm_parser_t parser; - pm_parser_init(&parser, source, size, NULL); - if (metadata) pm_parser_metadata(&parser, metadata); + pm_parser_init(&parser, source, size, &options); pm_node_t *node = pm_parse(&parser); pm_serialize_header(buffer); @@ -16592,6 +16593,7 @@ pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buf pm_node_destroy(&parser, node); pm_parser_free(&parser); + pm_options_free(&options); } #undef PM_CASE_KEYWORD diff --git a/prism/prism.h b/prism/prism.h index 84bd3f84a7..e2c5d06418 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -123,30 +123,15 @@ void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buf */ PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer); -/** - * Process any additional metadata being passed into a call to the parser via - * the pm_parse_serialize function. Since the source of these calls will be from - * Ruby implementation internals we assume it is from a trusted source. - * - * Currently, this is only passing in variable scoping surrounding an eval, but - * eventually it will be extended to hold any additional metadata. This data - * is serialized to reduce the calling complexity for a foreign function call - * vs a foreign runtime making a bindable in-memory version of a C structure. - * - * @param parser The parser to process the metadata for. - * @param metadata The metadata to process. - */ -void pm_parser_metadata(pm_parser_t *parser, const char *metadata); - /** * Parse the given source to the AST and serialize the AST to the given buffer. * * @param source The source to parse. * @param size The size of the source. * @param buffer The buffer to serialize to. - * @param metadata The optional metadata to pass to the parser. + * @param data The optional data to pass to the parser. */ -PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata); +PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data); /** * Parse and serialize the comments in the given source to the given buffer. @@ -154,9 +139,9 @@ PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t si * @param source The source to parse. * @param size The size of the source. * @param buffer The buffer to serialize to. - * @param metadata The optional metadata to pass to the parser. + * @param data The optional data to pass to the parser. */ -PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata); +PRISM_EXPORTED_FUNCTION void pm_parse_serialize_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data); /** * Lex the given source and serialize to the given buffer. diff --git a/prism/templates/src/serialize.c.erb b/prism/templates/src/serialize.c.erb index db10fab6de..b8e93df6fb 100644 --- a/prism/templates/src/serialize.c.erb +++ b/prism/templates/src/serialize.c.erb @@ -287,9 +287,9 @@ serialize_token(void *data, pm_parser_t *parser, pm_token_t *token) { * Lex the given source and serialize to the given buffer. */ PRISM_EXPORTED_FUNCTION void -pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_buffer_t *buffer) { +pm_lex_serialize(const uint8_t *source, size_t size, const char *data, pm_buffer_t *buffer) { pm_options_t options = { 0 }; - pm_options_filepath_set(&options, filepath); + if (data != NULL) pm_options_read(&options, data); pm_parser_t parser; pm_parser_init(&parser, source, size, &options); @@ -321,10 +321,12 @@ pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_bu * source to the given buffer. */ PRISM_EXPORTED_FUNCTION void -pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) { +pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *data) { + pm_options_t options = { 0 }; + if (data != NULL) pm_options_read(&options, data); + pm_parser_t parser; - pm_parser_init(&parser, source, size, NULL); - if (metadata) pm_parser_metadata(&parser, metadata); + pm_parser_init(&parser, source, size, &options); pm_lex_callback_t lex_callback = (pm_lex_callback_t) { .data = (void *) buffer, @@ -339,4 +341,5 @@ pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, pm_node_destroy(&parser, node); pm_parser_free(&parser); + pm_options_free(&options); } diff --git a/test/prism/parse_serialize_test.rb b/test/prism/parse_serialize_test.rb deleted file mode 100644 index 001518c14d..0000000000 --- a/test/prism/parse_serialize_test.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require_relative "test_helper" - -return if Prism::BACKEND == :FFI - -module Prism - class ParseSerializeTest < TestCase - def test_parse_serialize - dumped = Debug.parse_serialize_file(__FILE__) - result = Prism.load(File.read(__FILE__), dumped) - - assert_kind_of ParseResult, result, "Expected the return value to be a ParseResult" - assert_equal __FILE__, find_file_node(result)&.filepath, "Expected the filepath to be set correctly" - end - - def test_parse_serialize_with_locals - filepath = __FILE__ - metadata = [filepath.bytesize, filepath.b, 1, 1, 1, "foo".b].pack("LA*LLLA*") - - dumped = Debug.parse_serialize_file_metadata(filepath, metadata) - result = Prism.load(File.read(__FILE__), dumped) - - assert_kind_of ParseResult, result, "Expected the return value to be a ParseResult" - end - - private - - def find_file_node(result) - queue = [result.value] - - while (node = queue.shift) - return node if node.is_a?(SourceFileNode) - queue.concat(node.compact_child_nodes) - end - end - end -end diff --git a/test/prism/ruby_api_test.rb b/test/prism/ruby_api_test.rb index efe8bc1c1b..844a7796a3 100644 --- a/test/prism/ruby_api_test.rb +++ b/test/prism/ruby_api_test.rb @@ -20,6 +20,19 @@ module Prism assert_equal_nodes ast2, ast3 end + def test_options + assert_equal "", Prism.parse("__FILE__").value.statements.body[0].filepath + assert_equal "foo.rb", Prism.parse("__FILE__", filepath: "foo.rb").value.statements.body[0].filepath + + refute Prism.parse("\"foo\"").value.statements.body[0].frozen? + assert Prism.parse("\"foo\"", frozen_string_literal: true).value.statements.body[0].frozen? + refute Prism.parse("\"foo\"", frozen_string_literal: false).value.statements.body[0].frozen? + + assert_kind_of Prism::CallNode, Prism.parse("foo").value.statements.body[0] + assert_kind_of Prism::LocalVariableReadNode, Prism.parse("foo", scopes: [[:foo]]).value.statements.body[0] + assert_equal 2, Prism.parse("foo", scopes: [[:foo], []]).value.statements.body[0].depth + end + def test_literal_value_method assert_equal 123, parse_expression("123").value assert_equal 3.14, parse_expression("3.14").value