github
/
ruby
зеркало из https://github.com/github/ruby.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Render YARP templates in the build process (#8228)

This commit is contained in:
Takashi Kokubun 2023-08-17 14:39:04 -07:00 коммит произвёл GitHub
Родитель 5d48825d55
Коммит 67b5f63e97
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
24 изменённых файлов: 4269 добавлений и 19023 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

10
.gitignore поставляемый
Просмотреть файл

@ -254,3 +254,13 @@ lcov*.info
# /wasm/
/wasm/tests/*.wasm
# YARP
/lib/yarp/node.rb
/lib/yarp/serialize.rb
/yarp/api_node.c
/yarp/ast.h
/yarp/node.c
/yarp/prettyprint.c
/yarp/serialize.c
/yarp/token_type.c

26
common.mk
Просмотреть файл

@ -198,10 +198,34 @@ COMMONOBJS = array.$(OBJEXT) \
$(YARP_FILES): $(YARP_BUILD_DIR)/.time $(YARP_BUILD_DIR)/enc/.time $(YARP_BUILD_DIR)/util/.time
$(YARP_BUILD_DIR)/.time $(YARP_BUILD_DIR)/enc/.time $(YARP_BUILD_DIR)/util/.time:
$(YARP_BUILD_DIR)/.time $(YARP_BUILD_DIR)/enc/.time $(YARP_BUILD_DIR)/util/.time: $(top_srcdir)/lib/yarp/node.rb $(top_srcdir)/lib/yarp/serialize.rb
$(Q) $(MAKEDIRS) $(@D)
@$(NULLCMD) > $@
$(top_srcdir)/yarp/api_node.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/ext/yarp/api_node.c.erb
$(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb ext/yarp/api_node.c $(top_srcdir)/yarp/api_node.c
$(top_srcdir)/yarp/ast.h: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/include/yarp/ast.h.erb
$(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb include/yarp/ast.h $(top_srcdir)/yarp/ast.h
$(top_srcdir)/lib/yarp/node.rb: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/lib/yarp/node.rb.erb
$(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb lib/yarp/node.rb $(top_srcdir)/lib/yarp/node.rb
$(top_srcdir)/lib/yarp/serialize.rb: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/lib/yarp/serialize.rb.erb
$(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb lib/yarp/serialize.rb $(top_srcdir)/lib/yarp/serialize.rb
$(top_srcdir)/yarp/node.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/src/node.c.erb
$(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb src/node.c $(top_srcdir)/yarp/node.c
$(top_srcdir)/yarp/prettyprint.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/src/prettyprint.c.erb
$(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb src/prettyprint.c $(top_srcdir)/yarp/prettyprint.c
$(top_srcdir)/yarp/serialize.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/src/serialize.c.erb
$(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb src/serialize.c $(top_srcdir)/yarp/serialize.c
$(top_srcdir)/yarp/token_type.c: $(top_srcdir)/yarp/templates/template.rb $(top_srcdir)/yarp/templates/src/token_type.c.erb
$(Q) $(BASERUBY) $(top_srcdir)/yarp/templates/template.rb src/token_type.c $(top_srcdir)/yarp/token_type.c
EXPORTOBJS = $(DLNOBJ) \
localeinit.$(OBJEXT) \
loadpath.$(OBJEXT) \

7480
lib/yarp/node.rb
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

582
lib/yarp/serialize.rb
Просмотреть файл

@ -1,582 +0,0 @@
# frozen_string_literal: true
=begin
This file is generated by the bin/template script and should not be
modified manually. See templates/lib/yarp/serialize.rb.erb
if you are looking to modify the template
=end
require "stringio"
# Polyfill for String#unpack1 with the offset parameter.
if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset }
String.prepend(
Module.new {
def unpack1(format, offset: 0)
offset == 0 ? super(format) : self[offset..].unpack1(format)
end
}
)
end
module YARP
module Serialize
MAJOR_VERSION = 0
MINOR_VERSION = 7
PATCH_VERSION = 0
def self.load(input, serialized)
Loader.new(Source.new(input), serialized).load
end
def self.load_tokens(source, serialized)
Loader.new(source, serialized).load_tokens
end
class Loader
attr_reader :encoding, :input, :serialized, :io
attr_reader :constant_pool_offset, :constant_pool, :source
def initialize(source, serialized)
@encoding = Encoding::UTF_8
@input = source.source.dup
@serialized = serialized
@io = StringIO.new(serialized)
@io.set_encoding(Encoding::BINARY)
@constant_pool_offset = nil
@constant_pool = nil
@source = source
end
def load_tokens
tokens = []
while type = TOKEN_TYPES.fetch(load_varint)
start = load_varint
length = load_varint
lex_state = load_varint
location = Location.new(@source, start, length)
tokens << [YARP::Token.new(type, location.slice, location), lex_state]
end
comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) }
errors = load_varint.times.map { ParseError.new(load_string, load_location) }
warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) }
raise "Expected to consume all bytes while deserializing" unless @io.eof?
YARP::ParseResult.new(tokens, comments, errors, warnings, @source)
end
def load
raise "Invalid serialization" if io.read(4) != "YARP"
raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION]
@encoding = Encoding.find(io.read(load_varint))
@input = input.force_encoding(@encoding).freeze
comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(io.getbyte), load_location) }
errors = load_varint.times.map { ParseError.new(load_string, load_location) }
warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) }
@constant_pool_offset = io.read(4).unpack1("L")
@constant_pool = Array.new(load_varint, nil)
ast = load_node
YARP::ParseResult.new(ast, comments, errors, warnings, @source)
end
private
# variable-length integer using https://en.wikipedia.org/wiki/LEB128
# This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints
def load_varint
n = io.getbyte
if n < 128
n
else
n -= 128
shift = 0
while (b = io.getbyte) >= 128
n += (b - 128) << (shift += 7)
end
n + (b << (shift + 7))
end
end
def load_serialized_length
io.read(4).unpack1("L")
end
def load_optional_node
if io.getbyte != 0
io.pos -= 1
load_node
end
end
def load_string
io.read(load_varint).force_encoding(encoding)
end
def load_location
Location.new(source, load_varint, load_varint)
end
def load_optional_location
load_location if io.getbyte != 0
end
def load_constant
index = load_varint - 1
constant = constant_pool[index]
unless constant
offset = constant_pool_offset + index * 8
start = serialized.unpack1("L", offset: offset)
length = serialized.unpack1("L", offset: offset + 4)
constant = input.byteslice(start, length).to_sym
constant_pool[index] = constant
end
constant
end
def load_node
type = io.getbyte
location = load_location
case type
when 1 then
AliasNode.new(load_node, load_node, load_location, location)
when 2 then
AlternationPatternNode.new(load_node, load_node, load_location, location)
when 3 then
AndNode.new(load_node, load_node, load_location, location)
when 4 then
ArgumentsNode.new(Array.new(load_varint) { load_node }, location)
when 5 then
ArrayNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
when 6 then
ArrayPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_location, load_optional_location, location)
when 7 then
AssocNode.new(load_node, load_optional_node, load_optional_location, location)
when 8 then
AssocSplatNode.new(load_optional_node, load_location, location)
when 9 then
BackReferenceReadNode.new(location)
when 10 then
BeginNode.new(load_optional_location, load_optional_node, load_optional_node, load_optional_node, load_optional_node, load_optional_location, location)
when 11 then
BlockArgumentNode.new(load_optional_node, load_location, location)
when 12 then
BlockNode.new(Array.new(load_varint) { load_constant }, load_optional_node, load_optional_node, load_location, load_location, location)
when 13 then
BlockParameterNode.new(load_optional_location, load_location, location)
when 14 then
BlockParametersNode.new(load_optional_node, Array.new(load_varint) { load_location }, load_optional_location, load_optional_location, location)
when 15 then
BreakNode.new(load_optional_node, load_location, location)
when 16 then
CallNode.new(load_optional_node, load_optional_location, load_optional_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, load_varint, load_string, location)
when 17 then
CallOperatorAndWriteNode.new(load_node, load_location, load_node, location)
when 18 then
CallOperatorOrWriteNode.new(load_node, load_node, load_location, location)
when 19 then
CallOperatorWriteNode.new(load_node, load_location, load_node, load_constant, location)
when 20 then
CapturePatternNode.new(load_node, load_node, load_location, location)
when 21 then
CaseNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_location, load_location, location)
when 22 then
ClassNode.new(Array.new(load_varint) { load_constant }, load_location, load_node, load_optional_location, load_optional_node, load_optional_node, load_location, location)
when 23 then
ClassVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
when 24 then
ClassVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
when 25 then
ClassVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
when 26 then
ClassVariableReadNode.new(location)
when 27 then
ClassVariableWriteNode.new(load_location, load_optional_node, load_optional_location, location)
when 28 then
ConstantOperatorAndWriteNode.new(load_location, load_location, load_node, location)
when 29 then
ConstantOperatorOrWriteNode.new(load_location, load_location, load_node, location)
when 30 then
ConstantOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
when 31 then
ConstantPathNode.new(load_optional_node, load_node, load_location, location)
when 32 then
ConstantPathOperatorAndWriteNode.new(load_node, load_location, load_node, location)
when 33 then
ConstantPathOperatorOrWriteNode.new(load_node, load_location, load_node, location)
when 34 then
ConstantPathOperatorWriteNode.new(load_node, load_location, load_node, load_constant, location)
when 35 then
ConstantPathWriteNode.new(load_node, load_optional_location, load_optional_node, location)
when 36 then
ConstantReadNode.new(location)
when 37 then
ConstantWriteNode.new(load_location, load_optional_node, load_optional_location, location)
when 38 then
load_serialized_length
DefNode.new(load_location, load_optional_node, load_optional_node, load_optional_node, Array.new(load_varint) { load_constant }, load_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, load_optional_location, location)
when 39 then
DefinedNode.new(load_optional_location, load_node, load_optional_location, load_location, location)
when 40 then
ElseNode.new(load_location, load_optional_node, load_optional_location, location)
when 41 then
EmbeddedStatementsNode.new(load_location, load_optional_node, load_location, location)
when 42 then
EmbeddedVariableNode.new(load_location, load_node, location)
when 43 then
EnsureNode.new(load_location, load_optional_node, load_location, location)
when 44 then
FalseNode.new(location)
when 45 then
FindPatternNode.new(load_optional_node, load_node, Array.new(load_varint) { load_node }, load_node, load_optional_location, load_optional_location, location)
when 46 then
FlipFlopNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
when 47 then
FloatNode.new(location)
when 48 then
ForNode.new(load_node, load_node, load_optional_node, load_location, load_location, load_optional_location, load_location, location)
when 49 then
ForwardingArgumentsNode.new(location)
when 50 then
ForwardingParameterNode.new(location)
when 51 then
ForwardingSuperNode.new(load_optional_node, location)
when 52 then
GlobalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
when 53 then
GlobalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
when 54 then
GlobalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
when 55 then
GlobalVariableReadNode.new(location)
when 56 then
GlobalVariableWriteNode.new(load_location, load_optional_location, load_optional_node, location)
when 57 then
HashNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
when 58 then
HashPatternNode.new(load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_location, load_optional_location, location)
when 59 then
IfNode.new(load_optional_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
when 60 then
ImaginaryNode.new(load_node, location)
when 61 then
InNode.new(load_node, load_optional_node, load_location, load_optional_location, location)
when 62 then
InstanceVariableOperatorAndWriteNode.new(load_location, load_location, load_node, location)
when 63 then
InstanceVariableOperatorOrWriteNode.new(load_location, load_location, load_node, location)
when 64 then
InstanceVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, location)
when 65 then
InstanceVariableReadNode.new(location)
when 66 then
InstanceVariableWriteNode.new(load_location, load_optional_node, load_optional_location, location)
when 67 then
IntegerNode.new(location)
when 68 then
InterpolatedRegularExpressionNode.new(load_location, Array.new(load_varint) { load_node }, load_location, load_varint, location)
when 69 then
InterpolatedStringNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
when 70 then
InterpolatedSymbolNode.new(load_optional_location, Array.new(load_varint) { load_node }, load_optional_location, location)
when 71 then
InterpolatedXStringNode.new(load_location, Array.new(load_varint) { load_node }, load_location, location)
when 72 then
KeywordHashNode.new(Array.new(load_varint) { load_node }, location)
when 73 then
KeywordParameterNode.new(load_location, load_optional_node, location)
when 74 then
KeywordRestParameterNode.new(load_location, load_optional_location, location)
when 75 then
LambdaNode.new(Array.new(load_varint) { load_constant }, load_location, load_optional_node, load_optional_node, location)
when 76 then
LocalVariableOperatorAndWriteNode.new(load_location, load_location, load_node, load_constant, location)
when 77 then
LocalVariableOperatorOrWriteNode.new(load_location, load_location, load_node, load_constant, location)
when 78 then
LocalVariableOperatorWriteNode.new(load_location, load_location, load_node, load_constant, load_constant, location)
when 79 then
LocalVariableReadNode.new(load_constant, load_varint, location)
when 80 then
LocalVariableWriteNode.new(load_constant, load_varint, load_optional_node, load_location, load_optional_location, location)
when 81 then
MatchPredicateNode.new(load_node, load_node, load_location, location)
when 82 then
MatchRequiredNode.new(load_node, load_node, load_location, location)
when 83 then
MissingNode.new(location)
when 84 then
ModuleNode.new(Array.new(load_varint) { load_constant }, load_location, load_node, load_optional_node, load_location, location)
when 85 then
MultiWriteNode.new(Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_location, load_optional_location, location)
when 86 then
NextNode.new(load_optional_node, load_location, location)
when 87 then
NilNode.new(location)
when 88 then
NoKeywordsParameterNode.new(load_location, load_location, location)
when 89 then
NumberedReferenceReadNode.new(location)
when 90 then
OptionalParameterNode.new(load_constant, load_location, load_location, load_node, location)
when 91 then
OrNode.new(load_node, load_node, load_location, location)
when 92 then
ParametersNode.new(Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, Array.new(load_varint) { load_node }, load_optional_node, Array.new(load_varint) { load_node }, load_optional_node, load_optional_node, location)
when 93 then
ParenthesesNode.new(load_optional_node, load_location, load_location, location)
when 94 then
PinnedExpressionNode.new(load_node, load_location, load_location, load_location, location)
when 95 then
PinnedVariableNode.new(load_node, load_location, location)
when 96 then
PostExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
when 97 then
PreExecutionNode.new(load_optional_node, load_location, load_location, load_location, location)
when 98 then
ProgramNode.new(Array.new(load_varint) { load_constant }, load_node, location)
when 99 then
RangeNode.new(load_optional_node, load_optional_node, load_location, load_varint, location)
when 100 then
RationalNode.new(load_node, location)
when 101 then
RedoNode.new(location)
when 102 then
RegularExpressionNode.new(load_location, load_location, load_location, load_string, load_varint, location)
when 103 then
RequiredDestructuredParameterNode.new(Array.new(load_varint) { load_node }, load_location, load_location, location)
when 104 then
RequiredParameterNode.new(load_constant, location)
when 105 then
RescueModifierNode.new(load_node, load_location, load_node, location)
when 106 then
RescueNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_location, load_optional_node, load_optional_node, load_optional_node, location)
when 107 then
RestParameterNode.new(load_location, load_optional_location, location)
when 108 then
RetryNode.new(location)
when 109 then
ReturnNode.new(load_location, load_optional_node, location)
when 110 then
SelfNode.new(location)
when 111 then
SingletonClassNode.new(Array.new(load_varint) { load_constant }, load_location, load_location, load_node, load_optional_node, load_location, location)
when 112 then
SourceEncodingNode.new(location)
when 113 then
SourceFileNode.new(load_string, location)
when 114 then
SourceLineNode.new(location)
when 115 then
SplatNode.new(load_location, load_optional_node, location)
when 116 then
StatementsNode.new(Array.new(load_varint) { load_node }, location)
when 117 then
StringConcatNode.new(load_node, load_node, location)
when 118 then
StringNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
when 119 then
SuperNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, load_optional_node, location)
when 120 then
SymbolNode.new(load_optional_location, load_location, load_optional_location, load_string, location)
when 121 then
TrueNode.new(location)
when 122 then
UndefNode.new(Array.new(load_varint) { load_node }, load_location, location)
when 123 then
UnlessNode.new(load_location, load_node, load_optional_node, load_optional_node, load_optional_location, location)
when 124 then
UntilNode.new(load_location, load_node, load_optional_node, load_varint, location)
when 125 then
WhenNode.new(load_location, Array.new(load_varint) { load_node }, load_optional_node, location)
when 126 then
WhileNode.new(load_location, load_node, load_optional_node, load_varint, location)
when 127 then
XStringNode.new(load_location, load_location, load_location, load_string, location)
when 128 then
YieldNode.new(load_location, load_optional_location, load_optional_node, load_optional_location, location)
end
end
end
TOKEN_TYPES = [
nil,
:EOF,
:MISSING,
:NOT_PROVIDED,
:AMPERSAND,
:AMPERSAND_AMPERSAND,
:AMPERSAND_AMPERSAND_EQUAL,
:AMPERSAND_DOT,
:AMPERSAND_EQUAL,
:BACKTICK,
:BACK_REFERENCE,
:BANG,
:BANG_EQUAL,
:BANG_TILDE,
:BRACE_LEFT,
:BRACE_RIGHT,
:BRACKET_LEFT,
:BRACKET_LEFT_ARRAY,
:BRACKET_LEFT_RIGHT,
:BRACKET_LEFT_RIGHT_EQUAL,
:BRACKET_RIGHT,
:CARET,
:CARET_EQUAL,
:CHARACTER_LITERAL,
:CLASS_VARIABLE,
:COLON,
:COLON_COLON,
:COMMA,
:COMMENT,
:CONSTANT,
:DOT,
:DOT_DOT,
:DOT_DOT_DOT,
:EMBDOC_BEGIN,
:EMBDOC_END,
:EMBDOC_LINE,
:EMBEXPR_BEGIN,
:EMBEXPR_END,
:EMBVAR,
:EQUAL,
:EQUAL_EQUAL,
:EQUAL_EQUAL_EQUAL,
:EQUAL_GREATER,
:EQUAL_TILDE,
:FLOAT,
:FLOAT_IMAGINARY,
:FLOAT_RATIONAL,
:FLOAT_RATIONAL_IMAGINARY,
:GLOBAL_VARIABLE,
:GREATER,
:GREATER_EQUAL,
:GREATER_GREATER,
:GREATER_GREATER_EQUAL,
:HEREDOC_END,
:HEREDOC_START,
:IDENTIFIER,
:IGNORED_NEWLINE,
:INSTANCE_VARIABLE,
:INTEGER,
:INTEGER_IMAGINARY,
:INTEGER_RATIONAL,
:INTEGER_RATIONAL_IMAGINARY,
:KEYWORD_ALIAS,
:KEYWORD_AND,
:KEYWORD_BEGIN,
:KEYWORD_BEGIN_UPCASE,
:KEYWORD_BREAK,
:KEYWORD_CASE,
:KEYWORD_CLASS,
:KEYWORD_DEF,
:KEYWORD_DEFINED,
:KEYWORD_DO,
:KEYWORD_DO_LOOP,
:KEYWORD_ELSE,
:KEYWORD_ELSIF,
:KEYWORD_END,
:KEYWORD_END_UPCASE,
:KEYWORD_ENSURE,
:KEYWORD_FALSE,
:KEYWORD_FOR,
:KEYWORD_IF,
:KEYWORD_IF_MODIFIER,
:KEYWORD_IN,
:KEYWORD_MODULE,
:KEYWORD_NEXT,
:KEYWORD_NIL,
:KEYWORD_NOT,
:KEYWORD_OR,
:KEYWORD_REDO,
:KEYWORD_RESCUE,
:KEYWORD_RESCUE_MODIFIER,
:KEYWORD_RETRY,
:KEYWORD_RETURN,
:KEYWORD_SELF,
:KEYWORD_SUPER,
:KEYWORD_THEN,
:KEYWORD_TRUE,
:KEYWORD_UNDEF,
:KEYWORD_UNLESS,
:KEYWORD_UNLESS_MODIFIER,
:KEYWORD_UNTIL,
:KEYWORD_UNTIL_MODIFIER,
:KEYWORD_WHEN,
:KEYWORD_WHILE,
:KEYWORD_WHILE_MODIFIER,
:KEYWORD_YIELD,
:KEYWORD___ENCODING__,
:KEYWORD___FILE__,
:KEYWORD___LINE__,
:LABEL,
:LABEL_END,
:LAMBDA_BEGIN,
:LESS,
:LESS_EQUAL,
:LESS_EQUAL_GREATER,
:LESS_LESS,
:LESS_LESS_EQUAL,
:MINUS,
:MINUS_EQUAL,
:MINUS_GREATER,
:NEWLINE,
:NUMBERED_REFERENCE,
:PARENTHESIS_LEFT,
:PARENTHESIS_LEFT_PARENTHESES,
:PARENTHESIS_RIGHT,
:PERCENT,
:PERCENT_EQUAL,
:PERCENT_LOWER_I,
:PERCENT_LOWER_W,
:PERCENT_LOWER_X,
:PERCENT_UPPER_I,
:PERCENT_UPPER_W,
:PIPE,
:PIPE_EQUAL,
:PIPE_PIPE,
:PIPE_PIPE_EQUAL,
:PLUS,
:PLUS_EQUAL,
:QUESTION_MARK,
:REGEXP_BEGIN,
:REGEXP_END,
:SEMICOLON,
:SLASH,
:SLASH_EQUAL,
:STAR,
:STAR_EQUAL,
:STAR_STAR,
:STAR_STAR_EQUAL,
:STRING_BEGIN,
:STRING_CONTENT,
:STRING_END,
:SYMBOL_BEGIN,
:TILDE,
:UAMPERSAND,
:UCOLON_COLON,
:UDOT_DOT,
:UDOT_DOT_DOT,
:UMINUS,
:UMINUS_NUM,
:UPLUS,
:USTAR,
:USTAR_STAR,
:WORDS_SEP,
:__END__,
]
end
end

5
tool/sync_default_gems.rb
Просмотреть файл

@ -403,7 +403,6 @@ module SyncDefaultGems
rm_rf(%w[test/yarp yarp])
# Run the YARP templating scripts
system("ruby #{upstream}/templates/template.rb")
cp_r("#{upstream}/ext/yarp", "yarp")
cp_r("#{upstream}/lib/.", "lib")
cp_r("#{upstream}/test", "test/yarp")
@ -420,6 +419,9 @@ module SyncDefaultGems
cp_r("#{upstream}/include/yarp/.", "yarp")
cp_r("#{upstream}/include/yarp.h", "yarp")
cp_r("#{upstream}/config.yml", "yarp/")
cp_r("#{upstream}/templates", "yarp/")
rm_f("yarp/config.h")
File.write("yarp/config.h", "#include \"ruby/config.h\"\n")
rm("yarp/extconf.rb")
@ -456,7 +458,6 @@ module SyncDefaultGems
|configure\.ac
|rakelib\/.*
|rust\/.*
|templates\/.*
|test\/lib\/.*
|tasks\/.*
|ext\/yarp\/extconf\.rb

3712
yarp/api_node.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

1416
yarp/ast.h
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

2171
yarp/config.yml Normal file
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

2033
yarp/node.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

1801
yarp/prettyprint.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

1649
yarp/serialize.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

204
yarp/templates/ext/yarp/api_node.c.erb Normal file
Просмотреть файл

@ -0,0 +1,204 @@
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
#include "yarp/extension.h"
extern VALUE rb_cYARP;
extern VALUE rb_cYARPNode;
extern VALUE rb_cYARPSource;
extern VALUE rb_cYARPToken;
extern VALUE rb_cYARPLocation;
<%- nodes.each do |node| -%>
static VALUE rb_cYARP<%= node.name %>;
<%- end -%>
static VALUE
yp_location_new(yp_parser_t *parser, const char *start, const char *end, VALUE source) {
VALUE argv[] = { source, LONG2FIX(start - parser->start), LONG2FIX(end - start) };
return rb_class_new_instance(3, argv, rb_cYARPLocation);
}
VALUE
yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALUE source) {
ID type = rb_intern(yp_token_type_to_str(token->type));
VALUE location = yp_location_new(parser, token->start, token->end, source);
VALUE argv[] = {
ID2SYM(type),
rb_enc_str_new(token->start, token->end - token->start, encoding),
location
};
return rb_class_new_instance(3, argv, rb_cYARPToken);
}
static VALUE
yp_string_new(yp_string_t *string, rb_encoding *encoding) {
return rb_enc_str_new(yp_string_source(string), yp_string_length(string), encoding);
}
// Create a YARP::Source object from the given parser.
VALUE
yp_source_new(yp_parser_t *parser) {
VALUE source = rb_str_new(parser->start, parser->end - parser->start);
VALUE offsets = rb_ary_new_capa(parser->newline_list.size);
for (size_t index = 0; index < parser->newline_list.size; index++) {
rb_ary_push(offsets, INT2FIX(parser->newline_list.offsets[index]));
}
VALUE source_argv[] = { source, offsets };
return rb_class_new_instance(2, source_argv, rb_cYARPSource);
}
typedef struct yp_node_stack_node {
struct yp_node_stack_node *prev;
yp_node_t *visit;
bool visited;
} yp_node_stack_node_t;
static void
yp_node_stack_push(yp_node_stack_node_t **stack, yp_node_t *visit) {
yp_node_stack_node_t *node = malloc(sizeof(yp_node_stack_node_t));
node->prev = *stack;
node->visit = visit;
node->visited = false;
*stack = node;
}
static yp_node_t *
yp_node_stack_pop(yp_node_stack_node_t **stack) {
yp_node_stack_node_t *current = *stack;
yp_node_t *visit = current->visit;
*stack = current->prev;
free(current);
return visit;
}
VALUE
yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) {
VALUE source = yp_source_new(parser);
ID *constants = calloc(parser->constant_pool.size, sizeof(ID));
for (size_t index = 0; index < parser->constant_pool.capacity; index++) {
yp_constant_t constant = parser->constant_pool.constants[index];
if (constant.id != 0) {
constants[constant.id - 1] = rb_intern3(constant.start, constant.length, encoding);
}
}
yp_node_stack_node_t *node_stack = NULL;
yp_node_stack_push(&node_stack, node);
VALUE value_stack = rb_ary_new();
while (node_stack != NULL) {
if (!node_stack->visited) {
if (node_stack->visit == NULL) {
yp_node_stack_pop(&node_stack);
rb_ary_push(value_stack, Qnil);
continue;
}
yp_node_t *node = node_stack->visit;
node_stack->visited = true;
switch (YP_NODE_TYPE(node)) {
<%- nodes.each do |node| -%>
<%- if node.params.any? { |param| [NodeParam, OptionalNodeParam, NodeListParam].include?(param.class) } -%>
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
case <%= node.type %>: {
yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node;
<%- node.params.each do |param| -%>
<%- case param -%>
<%- when NodeParam, OptionalNodeParam -%>
yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= param.name %>);
<%- when NodeListParam -%>
for (size_t index = 0; index < cast-><%= param.name %>.size; index++) {
yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= param.name %>.nodes[index]);
}
<%- end -%>
<%- end -%>
break;
}
<%- end -%>
<%- end -%>
default:
break;
}
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
} else {
yp_node_t *node = yp_node_stack_pop(&node_stack);
switch (YP_NODE_TYPE(node)) {
<%- nodes.each do |node| -%>
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
case <%= node.type %>: {
<%- if node.params.any? { |param| ![NodeParam, OptionalNodeParam].include?(param.class) } -%>
yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node;
<%- end -%>
VALUE argv[<%= node.params.length + 1 %>];
<%- node.params.each_with_index do |param, index| -%>
// <%= param.name %>
<%- case param -%>
<%- when NodeParam, OptionalNodeParam -%>
argv[<%= index %>] = rb_ary_pop(value_stack);
<%- when NodeListParam -%>
argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size);
for (size_t index = 0; index < cast-><%= param.name %>.size; index++) {
rb_ary_push(argv[<%= index %>], rb_ary_pop(value_stack));
}
<%- when StringParam -%>
argv[<%= index %>] = yp_string_new(&cast-><%= param.name %>, encoding);
<%- when LocationListParam -%>
argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size);
for (size_t index = 0; index < cast-><%= param.name %>.size; index++) {
yp_location_t location = cast-><%= param.name %>.locations[index];
rb_ary_push(argv[<%= index %>], yp_location_new(parser, location.start, location.end, source));
}
<%- when ConstantParam -%>
argv[<%= index %>] = rb_id2sym(constants[cast-><%= param.name %> - 1]);
<%- when ConstantListParam -%>
argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size);
for (size_t index = 0; index < cast-><%= param.name %>.size; index++) {
rb_ary_push(argv[<%= index %>], rb_id2sym(constants[cast-><%= param.name %>.ids[index] - 1]));
}
<%- when LocationParam -%>
argv[<%= index %>] = yp_location_new(parser, cast-><%= param.name %>.start, cast-><%= param.name %>.end, source);
<%- when OptionalLocationParam -%>
argv[<%= index %>] = cast-><%= param.name %>.start == NULL ? Qnil : yp_location_new(parser, cast-><%= param.name %>.start, cast-><%= param.name %>.end, source);
<%- when UInt32Param -%>
argv[<%= index %>] = ULONG2NUM(cast-><%= param.name %>);
<%- when FlagsParam -%>
argv[<%= index %>] = ULONG2NUM(node->flags >> <%= COMMON_FLAGS %>);
<%- else -%>
<%- raise -%>
<%- end -%>
<%- end -%>
// location
argv[<%= node.params.length %>] = yp_location_new(parser, node->location.start, node->location.end, source);
rb_ary_push(value_stack, rb_class_new_instance(<%= node.params.length + 1 %>, argv, rb_cYARP<%= node.name %>));
break;
}
<%- end -%>
default:
rb_raise(rb_eRuntimeError, "unknown node type: %d", YP_NODE_TYPE(node));
}
}
}
VALUE result = rb_ary_pop(value_stack);
free(constants);
return result;
}
void
Init_yarp_api_node(void) {
<%- nodes.each do |node| -%>
rb_cYARP<%= node.name %> = rb_define_class_under(rb_cYARP, "<%= node.name %>", rb_cYARPNode);
<%- end -%>
}

111
yarp/templates/include/yarp/ast.h.erb Normal file
Просмотреть файл

@ -0,0 +1,111 @@
#ifndef YARP_AST_H
#define YARP_AST_H
#include "yarp/defines.h"
#include "yarp/util/yp_constant_pool.h"
#include "yarp/util/yp_string.h"
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
// This enum represents every type of token in the Ruby source.
typedef enum yp_token_type {
<%- tokens.each do |token| -%>
<%= token.declaration %>
<%- end -%>
YP_TOKEN_MAXIMUM, // the maximum token value
} yp_token_type_t;
// This struct represents a token in the Ruby source. We use it to track both
// type and location information.
typedef struct {
yp_token_type_t type;
const char *start;
const char *end;
} yp_token_t;
// This represents a range of bytes in the source string to which a node or
// token corresponds.
typedef struct {
const char *start;
const char *end;
} yp_location_t;
typedef struct {
yp_location_t *locations;
size_t size;
size_t capacity;
} yp_location_list_t;
struct yp_node;
typedef struct yp_node_list {
struct yp_node **nodes;
size_t size;
size_t capacity;
} yp_node_list_t;
enum yp_node_type {
<%- nodes.each_with_index do |node, index| -%>
<%= node.type %> = <%= index + 1 %>,
<%- end -%>
};
typedef uint16_t yp_node_type_t;
typedef uint16_t yp_node_flags_t;
// We store the flags enum in every node in the tree. Some flags are common to
// all nodes (the ones listed below). Others are specific to certain node types.
static const yp_node_flags_t YP_NODE_FLAG_NEWLINE = 0x1;
// For easy access, we define some macros to check node type
#define YP_NODE_TYPE(node) ((enum yp_node_type)node->type)
#define YP_NODE_TYPE_P(node, type) (YP_NODE_TYPE(node) == (type))
// This is the overall tagged union representing a node in the syntax tree.
typedef struct yp_node {
// This represents the type of the node. It somewhat maps to the nodes that
// existed in the original grammar and ripper, but it's not a 1:1 mapping.
yp_node_type_t type;
// This represents any flags on the node. Currently, this is only a newline
// flag
yp_node_flags_t flags;
// This is the location of the node in the source. It's a range of bytes
// containing a start and an end.
yp_location_t location;
} yp_node_t;
<%- nodes.each do |node| -%>
// <%= node.name %>
typedef struct yp_<%= node.human %> {
yp_node_t base;
<%- node.params.grep_v(FlagsParam).each do |param| -%>
<%= case param
when NodeParam, OptionalNodeParam then "struct #{param.c_type} *#{param.name}"
when NodeListParam then "struct yp_node_list #{param.name}"
when LocationListParam then "yp_location_list_t #{param.name}"
when ConstantParam then "yp_constant_id_t #{param.name}"
when ConstantListParam then "yp_constant_id_list_t #{param.name}"
when StringParam then "yp_string_t #{param.name}"
when LocationParam, OptionalLocationParam then "yp_location_t #{param.name}"
when UInt32Param then "uint32_t #{param.name}"
else raise param.class.name
end
%>;
<%- end -%>
} yp_<%= node.human %>_t;
<%- end -%>
<%- flags.each do |flag| -%>
// <%= flag.name %>
typedef enum {
<%- flag.values.each.with_index(COMMON_FLAGS) do |value, index| -%>
YP_<%= flag.human.upcase %>_<%= value.name %> = 1 << <%= index %>,
<%- end -%>
} yp_<%= flag.human %>_t;
<%- end -%>
#endif // YARP_AST_H

14
yarp/templates/java/org/yarp/AbstractNodeVisitor.java.erb Normal file
Просмотреть файл

@ -0,0 +1,14 @@
package org.yarp;
// GENERATED BY <%= File.basename(__FILE__) %>
public abstract class AbstractNodeVisitor<T> {
protected abstract T defaultVisit(Nodes.Node node);
<%- nodes.each do |node| -%>
public T visit<%= node.name -%>(Nodes.<%= node.name -%> node) {
return defaultVisit(node);
}
<%- end -%>
}

282
yarp/templates/java/org/yarp/Loader.java.erb Normal file
Просмотреть файл

@ -0,0 +1,282 @@
package org.yarp;
import org.yarp.ParseResult;
import java.lang.Short;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
// GENERATED BY <%= File.basename(__FILE__) %>
// @formatter:off
public class Loader {
public static ParseResult load(byte[] serialized, Nodes.Source source) {
return new Loader(serialized, source).load();
}
private static final class ConstantPool {
private final byte[] source;
private final int bufferOffset;
private final byte[][] cache;
ConstantPool(byte[] source, int bufferOffset, int length) {
this.source = source;
this.bufferOffset = bufferOffset;
cache = new byte[length][];
}
byte[] get(ByteBuffer buffer, int oneBasedIndex) {
int index = oneBasedIndex - 1;
byte[] constant = cache[index];
if (constant == null) {
int offset = bufferOffset + index * 8;
int start = buffer.getInt(offset);
int length = buffer.getInt(offset + 4);
constant = new byte[length];
System.arraycopy(source, start, constant, 0, length);
cache[index] = constant;
}
return constant;
}
}
private final ByteBuffer buffer;
private ConstantPool constantPool;
private final Nodes.Source source;
private byte MAJOR_VERSION = (byte) 0;
private byte MINOR_VERSION = (byte) 7;
private byte PATCH_VERSION = (byte) 0;
private Loader(byte[] serialized, Nodes.Source source) {
this.buffer = ByteBuffer.wrap(serialized).order(ByteOrder.nativeOrder());
this.source = source;
}
private ParseResult load() {
expect((byte) 'Y');
expect((byte) 'A');
expect((byte) 'R');
expect((byte) 'P');
expect(MAJOR_VERSION);
expect(MINOR_VERSION);
expect(PATCH_VERSION);
// This loads the name of the encoding. We don't actually do anything
// with it just yet.
int encodingLength = loadVarInt();
byte[] encodingName = new byte[encodingLength];
buffer.get(encodingName);
ParseResult.Comment[] comments = loadComments();
ParseResult.Error[] errors = loadSyntaxErrors();
ParseResult.Warning[] warnings = loadWarnings();
int constantPoolBufferOffset = buffer.getInt();
int constantPoolLength = loadVarInt();
this.constantPool = new ConstantPool(source.bytes, constantPoolBufferOffset, constantPoolLength);
Nodes.Node node = loadNode();
int left = constantPoolBufferOffset - buffer.position();
if (left != 0) {
throw new Error("Expected to consume all bytes while deserializing but there were " + left + " bytes left");
}
boolean[] newlineMarked = new boolean[1 + source.getLineCount()];
MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source, newlineMarked);
node.accept(visitor);
return new ParseResult(node, comments, errors, warnings);
}
private byte[] loadString() {
int length = loadVarInt();
byte[] string = new byte[length];
buffer.get(string);
return string;
}
private ParseResult.Comment[] loadComments() {
int count = loadVarInt();
ParseResult.Comment[] comments = new ParseResult.Comment[count];
for (int i = 0; i < count; i++) {
ParseResult.CommentType type = ParseResult.CommentType.VALUES[buffer.get()];
Nodes.Location location = loadLocation();
ParseResult.Comment comment = new ParseResult.Comment(type, location);
comments[i] = comment;
}
return comments;
}
private ParseResult.Error[] loadSyntaxErrors() {
int count = loadVarInt();
ParseResult.Error[] errors = new ParseResult.Error[count];
// error messages only contain ASCII characters
for (int i = 0; i < count; i++) {
byte[] bytes = loadString();
String message = new String(bytes, StandardCharsets.US_ASCII);
Nodes.Location location = loadLocation();
ParseResult.Error error = new ParseResult.Error(message, location);
errors[i] = error;
}
return errors;
}
private ParseResult.Warning[] loadWarnings() {
int count = loadVarInt();
ParseResult.Warning[] warnings = new ParseResult.Warning[count];
// warning messages only contain ASCII characters
for (int i = 0; i < count; i++) {
byte[] bytes = loadString();
String message = new String(bytes, StandardCharsets.US_ASCII);
Nodes.Location location = loadLocation();
ParseResult.Warning warning = new ParseResult.Warning(message, location);
warnings[i] = warning;
}
return warnings;
}
private Nodes.Node loadOptionalNode() {
if (buffer.get(buffer.position()) != 0) {
return loadNode();
} else {
buffer.position(buffer.position() + 1); // continue after the 0 byte
return null;
}
}
private Nodes.Location[] loadLocations() {
int length = loadVarInt();
if (length == 0) {
return Nodes.Location.EMPTY_ARRAY;
}
Nodes.Location[] locations = new Nodes.Location[length];
for (int i = 0; i < length; i++) {
locations[i] = loadLocation();
}
return locations;
}
private byte[] loadConstant() {
return constantPool.get(buffer, loadVarInt());
}
private byte[][] loadConstants() {
int length = loadVarInt();
if (length == 0) {
return Nodes.EMPTY_BYTE_ARRAY_ARRAY;
}
byte[][] constants = new byte[length][];
for (int i = 0; i < length; i++) {
constants[i] = constantPool.get(buffer, loadVarInt());
}
return constants;
}
private Nodes.Node[] loadNodes() {
int length = loadVarInt();
if (length == 0) {
return Nodes.Node.EMPTY_ARRAY;
}
Nodes.Node[] nodes = new Nodes.Node[length];
for (int i = 0; i < length; i++) {
nodes[i] = loadNode();
}
return nodes;
}
private Nodes.Location loadLocation() {
return new Nodes.Location(loadVarInt(), loadVarInt());
}
private Nodes.Location loadOptionalLocation() {
if (buffer.get() != 0) {
return loadLocation();
} else {
return null;
}
}
// From https://github.com/protocolbuffers/protobuf/blob/v23.1/java/core/src/main/java/com/google/protobuf/BinaryReader.java#L1507
private int loadVarInt() {
int x;
if ((x = buffer.get()) >= 0) {
return x;
} else if ((x ^= (buffer.get() << 7)) < 0) {
x ^= (~0 << 7);
} else if ((x ^= (buffer.get() << 14)) >= 0) {
x ^= (~0 << 7) ^ (~0 << 14);
} else if ((x ^= (buffer.get() << 21)) < 0) {
x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21);
} else {
x ^= buffer.get() << 28;
x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28);
}
return x;
}
private short loadFlags() {
int flags = loadVarInt();
assert flags >= 0 && flags <= Short.MAX_VALUE;
return (short) flags;
}
private Nodes.Node loadNode() {
int type = buffer.get() & 0xFF;
int startOffset = loadVarInt();
int length = loadVarInt();
switch (type) {
<%- nodes.each_with_index do |node, index| -%>
case <%= index + 1 %>:
<%-
params = node.needs_serialized_length? ? ["buffer.getInt()"] : []
params.concat node.params.map { |param|
case param
when NodeParam then "#{param.java_cast}loadNode()"
when OptionalNodeParam then "#{param.java_cast}loadOptionalNode()"
when StringParam then "loadString()"
when NodeListParam then "loadNodes()"
when LocationListParam then "loadLocations()"
when ConstantParam then "loadConstant()"
when ConstantListParam then "loadConstants()"
when LocationParam then "loadLocation()"
when OptionalLocationParam then "loadOptionalLocation()"
when UInt32Param then "loadVarInt()"
when FlagsParam then "loadFlags()"
else raise
end
}
params.concat ["startOffset", "length"]
-%>
return new Nodes.<%= node.name %>(<%= params.join(", ") -%>);
<%- end -%>
default:
throw new Error("Unknown node type: " + type);
}
}
private void expect(byte value) {
byte b = buffer.get();
if (b != value) {
throw new Error("Expected " + value + " but was " + b + " at position " + buffer.position());
}
}
}
// @formatter:on

291
yarp/templates/java/org/yarp/Nodes.java.erb Normal file
Просмотреть файл

@ -0,0 +1,291 @@
package org.yarp;
import java.lang.Override;
import java.lang.String;
import java.lang.StringBuilder;
import java.util.ArrayList;
import java.util.Arrays;
// GENERATED BY <%= File.basename(__FILE__) %>
// @formatter:off
public abstract class Nodes {
public static final byte[][] EMPTY_BYTE_ARRAY_ARRAY = {};
public static final class Location {
public static final Location[] EMPTY_ARRAY = {};
public final int startOffset;
public final int length;
public Location(int startOffset, int length) {
this.startOffset = startOffset;
this.length = length;
}
public int endOffset() {
return startOffset + length;
}
}
public static final class Source {
public final byte[] bytes;
private final int[] lineOffsets;
public Source(byte[] bytes) {
this(bytes, computeLineOffsets(bytes));
}
public Source(byte[] bytes, int[] lineOffsets) {
assert lineOffsets[0] == 0;
this.bytes = bytes;
this.lineOffsets = lineOffsets;
}
public static int[] computeLineOffsets(byte[] bytes) {
int[] lineOffsets = new int[8];
int lineOffsetsSize = 0;
lineOffsets[lineOffsetsSize++] = 0;
for (int i = 0; i < bytes.length; i++) {
if (bytes[i] == '\n') {
if (lineOffsetsSize == lineOffsets.length) {
lineOffsets = Arrays.copyOf(lineOffsets, lineOffsets.length * 2);
}
lineOffsets[lineOffsetsSize++] = i + 1;
}
}
return Arrays.copyOf(lineOffsets, lineOffsetsSize);
}
public int line(int byteOffset) {
assert byteOffset >= 0 && byteOffset < bytes.length : byteOffset;
int index = Arrays.binarySearch(lineOffsets, byteOffset);
int line;
if (index < 0) {
line = -index - 1;
} else {
line = index + 1;
}
assert line >= 1 && line <= getLineCount() : line;
return line;
}
public int getLineCount() {
return lineOffsets.length;
}
}
public static abstract class Node {
public static final Node[] EMPTY_ARRAY = {};
public final int startOffset;
public final int length;
private boolean newLineFlag = false;
public Node(int startOffset, int length) {
this.startOffset = startOffset;
this.length = length;
}
public final int endOffset() {
return startOffset + length;
}
public final boolean hasNewLineFlag() {
return newLineFlag;
}
public void setNewLineFlag(Source source, boolean[] newlineMarked) {
int line = source.line(this.startOffset);
if (!newlineMarked[line]) {
newlineMarked[line] = true;
this.newLineFlag = true;
}
}
public abstract <T> T accept(AbstractNodeVisitor<T> visitor);
public abstract <T> void visitChildNodes(AbstractNodeVisitor<T> visitor);
public abstract Node[] childNodes();
@Override
public String toString() {
return toString("");
}
private String toString(String indent) {
StringBuilder builder = new StringBuilder();
builder.append(indent).append(this.getClass().getSimpleName());
if (hasNewLineFlag()) {
builder.append("[Li]");
}
builder.append('\n');
for (Node child : childNodes()) {
if (child != null) {
builder.append(child.toString(indent + " "));
}
}
return builder.toString();
}
}
<%# FLAGS -%>
<%- flags.each do |group| -%>
public static final class <%= group.name %> implements Comparable<<%= group.name %>> {
<%- group.values.each_with_index do |value, index| -%>
// <%= value.comment %>
public static final short <%= value.name %> = 1 << <%= index %>;
<%- end -%>
<%- group.values.each do |value| -%>
public static boolean is<%= value.camelcase %>(short flags) {
return (flags & <%= value.name %>) != 0;
}
<%- end -%>
private final short flags;
public <%= group.name %>(short flags) {
this.flags = flags;
}
@Override
public int hashCode() {
return flags;
}
@Override
public boolean equals(Object other) {
if (!(other instanceof <%= group.name %>)) {
return false;
}
return flags == ((<%= group.name %>) other).flags;
}
@Override
public int compareTo(<%= group.name %> other) {
return flags - other.flags;
}
<%- group.values.each do |value| -%>
public boolean is<%= value.camelcase %>() {
return (flags & <%= value.name %>) != 0;
}
<%- end -%>
}
<%- end -%>
<%# NODES -%>
<%- nodes.each do |node| -%>
<%= "#{node.comment.split("\n").map { |line| "// #{line}" }.join("\n ")}\n" if node.comment -%>
public static final class <%= node.name -%> extends Node {
<%- if node.needs_serialized_length? -%>
public final int serializedLength;
<%- end -%>
<%- node.params.each do |param| -%>
public final <%= param.java_type %> <%= param.name %>;<%= ' // optional' if param.class.name.start_with?('Optional') %>
<%- end -%>
<%-
params = node.needs_serialized_length? ? ["int serializedLength"] : []
params.concat node.params.map { "#{_1.java_type} #{_1.name}" }
params.concat ["int startOffset", "int length"]
-%>
public <%=node.name -%>(<%= params.join(", ") %>) {
super(startOffset, length);
<%- if node.needs_serialized_length? -%>
this.serializedLength = serializedLength;
<%- end -%>
<%- node.params.each do |param| -%>
this.<%= param.name %> = <%= param.name %>;
<%- end -%>
}
<%# methods for flags -%>
<%- node.params.each do |param| -%>
<%- if param.is_a?(FlagsParam) -%>
<%- flags.find { |flag| flag.name == param.kind }.tap { raise "Expected to find #{param.kind}" unless _1 }.values.each do |value| -%>
public boolean is<%= value.camelcase %>() {
return <%= param.kind %>.is<%= value.camelcase %>(this.<%= param.name %>);
}
<%- end -%>
<%- end -%>
<%- end -%>
<%# potential override of setNewLineFlag() -%>
<%- if node.newline == false -%>
@Override
public void setNewLineFlag(Source source, boolean[] newlineMarked) {
// Never mark <%= node.name %> with a newline flag, mark children instead
}
<%- elsif node.newline.is_a?(String) -%>
@Override
public void setNewLineFlag(Source source, boolean[] newlineMarked) {
<%- param = node.params.find { |p| p.name == node.newline } or raise node.newline -%>
<%- case param -%>
<%- when SingleNodeParam -%>
this.<%= param.name %>.setNewLineFlag(source, newlineMarked);
<%- when NodeListParam -%>
Node first = this.<%= param.name %>.length > 0 ? this.<%= param.name %>[0] : null;
if (first != null) {
first.setNewLineFlag(source, newlineMarked);
}
<%- else raise param.class.name -%>
<%- end -%>
}
<%- end -%>
public <T> void visitChildNodes(AbstractNodeVisitor<T> visitor) {
<%- node.params.each do |param| -%>
<%- case param -%>
<%- when NodeListParam -%>
for (Nodes.Node child : this.<%= param.name %>) {
child.accept(visitor);
}
<%- when NodeParam -%>
this.<%= param.name %>.accept(visitor);
<%- when OptionalNodeParam -%>
if (this.<%= param.name %> != null) {
this.<%= param.name %>.accept(visitor);
}
<%- end -%>
<%- end -%>
}
public Node[] childNodes() {
<%- if node.params.none?(NodeListParam) and node.params.none?(SingleNodeParam) -%>
return EMPTY_ARRAY;
<%- elsif node.params.one?(NodeListParam) and node.params.none?(SingleNodeParam) -%>
return this.<%= node.params.grep(NodeListParam).first.name %>;
<%- elsif node.params.none?(NodeListParam) -%>
return new Node[] { <%= node.params.grep(SingleNodeParam).map { "this.#{_1.name}" }.join(', ') %> };
<%- else -%>
ArrayList<Node> childNodes = new ArrayList<>();
<%- node.params.each do |param| -%>
<%- case param -%>
<%- when SingleNodeParam -%>
childNodes.add(this.<%= param.name %>);
<%- when NodeListParam -%>
childNodes.addAll(Arrays.asList(this.<%= param.name %>));
<%- end -%>
<%- end -%>
return childNodes.toArray(EMPTY_ARRAY);
<%- end -%>
}
public <T> T accept(AbstractNodeVisitor<T> visitor) {
return visitor.visit<%= node.name -%>(this);
}
}
<%- end -%>
}
// @formatter:on

122
yarp/templates/lib/yarp/node.rb.erb Normal file
Просмотреть файл

@ -0,0 +1,122 @@
module YARP
<%- nodes.each do |node| -%>
<%= "#{node.comment.split("\n").map { |line| line.empty? ? "#" : "# #{line}" }.join("\n ")}\n " if node.comment %>class <%= node.name -%> < Node
<%- node.params.each do |param| -%>
# attr_reader <%= param.name %>: <%= param.rbs_class %>
attr_reader :<%= param.name %>
<%- end -%>
# def initialize: (<%= (node.params.map { |param| "#{param.name}: #{param.rbs_class}" } + ["location: Location"]).join(", ") %>) -> void
def initialize(<%= (node.params.map(&:name) + ["location"]).join(", ") %>)
<%- node.params.each do |param| -%>
@<%= param.name %> = <%= param.name %>
<%- end -%>
@location = location
end
# def accept: (visitor: Visitor) -> void
def accept(visitor)
visitor.visit_<%= node.human %>(self)
end
<%- if node.newline == false -%>
def set_newline_flag(newline_marked)
# Never mark <%= node.name %> with a newline flag, mark children instead
end
<%- elsif node.newline.is_a?(String) -%>
def set_newline_flag(newline_marked)
<%- param = node.params.find { |p| p.name == node.newline } or raise node.newline -%>
<%- case param -%>
<%- when SingleNodeParam -%>
<%= param.name %>.set_newline_flag(newline_marked)
<%- when NodeListParam -%>
first = <%= param.name %>.first
first.set_newline_flag(newline_marked) if first
<%- else raise param.class.name -%>
<%- end -%>
end
<%- end -%>
# def child_nodes: () -> Array[nil | Node]
def child_nodes
[<%= node.params.map { |param|
case param
when SingleNodeParam then param.name
when NodeListParam then "*#{param.name}"
end
}.compact.join(", ") %>]
end
# def deconstruct: () -> Array[nil | Node]
alias deconstruct child_nodes
# def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location]
def deconstruct_keys(keys)
{ <%= (node.params.map { |param| "#{param.name}: #{param.name}" } + ["location: location"]).join(", ") %> }
end
<%- node.params.each do |param| -%>
<%- case param -%>
<%- when LocationParam -%>
<%- raise unless param.name.end_with?("_loc") -%>
<%- next if node.params.any? { |other| other.name == param.name.delete_suffix("_loc") } -%>
# def <%= param.name.delete_suffix("_loc") %>: () -> String
def <%= param.name.delete_suffix("_loc") %>
<%= param.name %>.slice
end
<%- when OptionalLocationParam -%>
<%- raise unless param.name.end_with?("_loc") -%>
<%- next if node.params.any? { |other| other.name == param.name.delete_suffix("_loc") } -%>
# def <%= param.name.delete_suffix("_loc") %>: () -> String?
def <%= param.name.delete_suffix("_loc") %>
<%= param.name %>&.slice
end
<%- when FlagsParam -%>
<%- flags.find { |flag| flag.name == param.kind }.tap { |flag| raise "Expected to find #{param.kind}" unless flag }.values.each do |value| -%>
# def <%= value.name.downcase %>?: () -> bool
def <%= value.name.downcase %>?
<%= param.name %>.anybits?(<%= param.kind %>::<%= value.name %>)
end
<%- end -%>
<%- end -%>
<%- end -%>
end
<%- end -%>
<%- flags.each do |flag| -%>
module <%= flag.name %>
<%- flag.values.each_with_index do |value, index| -%>
# <%= value.comment %>
<%= value.name %> = 1 << <%= index %>
<%= "\n" if value != flag.values.last -%>
<%- end -%>
end
<%- end -%>
class Visitor < BasicVisitor
<%- nodes.each do |node| -%>
# Visit a <%= node.name %> node
alias visit_<%= node.human %> visit_child_nodes
<%= "\n" if node != nodes.last -%>
<%- end -%>
end
module DSL
private
# Create a new Location object
def Location(source = nil, start_offset = 0, length = 0)
Location.new(source, start_offset, length)
end
<%- nodes.each do |node| -%>
# Create a new <%= node.name %> node
def <%= node.name %>(<%= (node.params.map(&:name) + ["location = Location()"]).join(", ") %>)
<%= node.name %>.new(<%= (node.params.map(&:name) + ["location"]).join(", ") %>)
end
<%- end -%>
end
end

179
yarp/templates/lib/yarp/serialize.rb.erb Normal file
Просмотреть файл

@ -0,0 +1,179 @@
require "stringio"
# Polyfill for String#unpack1 with the offset parameter.
if String.instance_method(:unpack1).parameters.none? { |_, name| name == :offset }
String.prepend(
Module.new {
def unpack1(format, offset: 0)
offset == 0 ? super(format) : self[offset..].unpack1(format)
end
}
)
end
module YARP
module Serialize
MAJOR_VERSION = 0
MINOR_VERSION = 7
PATCH_VERSION = 0
def self.load(input, serialized)
Loader.new(Source.new(input), serialized).load
end
def self.load_tokens(source, serialized)
Loader.new(source, serialized).load_tokens
end
class Loader
attr_reader :encoding, :input, :serialized, :io
attr_reader :constant_pool_offset, :constant_pool, :source
def initialize(source, serialized)
@encoding = Encoding::UTF_8
@input = source.source.dup
@serialized = serialized
@io = StringIO.new(serialized)
@io.set_encoding(Encoding::BINARY)
@constant_pool_offset = nil
@constant_pool = nil
@source = source
end
def load_tokens
tokens = []
while type = TOKEN_TYPES.fetch(load_varint)
start = load_varint
length = load_varint
lex_state = load_varint
location = Location.new(@source, start, length)
tokens << [YARP::Token.new(type, location.slice, location), lex_state]
end
comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) }
errors = load_varint.times.map { ParseError.new(load_string, load_location) }
warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) }
raise "Expected to consume all bytes while deserializing" unless @io.eof?
YARP::ParseResult.new(tokens, comments, errors, warnings, @source)
end
def load
raise "Invalid serialization" if io.read(4) != "YARP"
raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION]
@encoding = Encoding.find(io.read(load_varint))
@input = input.force_encoding(@encoding).freeze
comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(io.getbyte), load_location) }
errors = load_varint.times.map { ParseError.new(load_string, load_location) }
warnings = load_varint.times.map { ParseWarning.new(load_string, load_location) }
@constant_pool_offset = io.read(4).unpack1("L")
@constant_pool = Array.new(load_varint, nil)
ast = load_node
YARP::ParseResult.new(ast, comments, errors, warnings, @source)
end
private
# variable-length integer using https://en.wikipedia.org/wiki/LEB128
# This is also what protobuf uses: https://protobuf.dev/programming-guides/encoding/#varints
def load_varint
n = io.getbyte
if n < 128
n
else
n -= 128
shift = 0
while (b = io.getbyte) >= 128
n += (b - 128) << (shift += 7)
end
n + (b << (shift + 7))
end
end
def load_serialized_length
io.read(4).unpack1("L")
end
def load_optional_node
if io.getbyte != 0
io.pos -= 1
load_node
end
end
def load_string
io.read(load_varint).force_encoding(encoding)
end
def load_location
Location.new(source, load_varint, load_varint)
end
def load_optional_location
load_location if io.getbyte != 0
end
def load_constant
index = load_varint - 1
constant = constant_pool[index]
unless constant
offset = constant_pool_offset + index * 8
start = serialized.unpack1("L", offset: offset)
length = serialized.unpack1("L", offset: offset + 4)
constant = input.byteslice(start, length).to_sym
constant_pool[index] = constant
end
constant
end
def load_node
type = io.getbyte
location = load_location
case type
<%- nodes.each_with_index do |node, index| -%>
when <%= index + 1 %> then
<%- if node.needs_serialized_length? -%>
load_serialized_length
<%- end -%>
<%= node.name %>.new(<%= (node.params.map { |param|
case param
when NodeParam then "load_node"
when OptionalNodeParam then "load_optional_node"
when StringParam then "load_string"
when NodeListParam then "Array.new(load_varint) { load_node }"
when LocationListParam then "Array.new(load_varint) { load_location }"
when ConstantParam then "load_constant"
when ConstantListParam then "Array.new(load_varint) { load_constant }"
when LocationParam then "load_location"
when OptionalLocationParam then "load_optional_location"
when UInt32Param, FlagsParam then "load_varint"
else raise
end
} + ["location"]).join(", ") -%>)
<%- end -%>
end
end
end
TOKEN_TYPES = [
nil,
<%- tokens.each do |token| -%>
<%= token.name.to_sym.inspect %>,
<%- end -%>
]
end
end

168
yarp/templates/src/node.c.erb Normal file
Просмотреть файл

@ -0,0 +1,168 @@
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
#include "yarp/node.h"
// Clear the node but preserves the location.
void yp_node_clear(yp_node_t *node) {
yp_location_t location = node->location;
memset(node, 0, sizeof(yp_node_t));
node->location = location;
}
// Calculate the size of the token list in bytes.
static size_t
yp_location_list_memsize(yp_location_list_t *list) {
return sizeof(yp_location_list_t) + (list->capacity * sizeof(yp_location_t));
}
// Append a token to the given list.
void
yp_location_list_append(yp_location_list_t *list, const yp_token_t *token) {
if (list->size == list->capacity) {
list->capacity = list->capacity == 0 ? 2 : list->capacity * 2;
list->locations = (yp_location_t *) realloc(list->locations, sizeof(yp_location_t) * list->capacity);
}
list->locations[list->size++] = (yp_location_t) { .start = token->start, .end = token->end };
}
// Free the memory associated with the token list.
static void
yp_location_list_free(yp_location_list_t *list) {
if (list->locations != NULL) {
free(list->locations);
}
}
static void
yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize);
// Calculate the size of the node list in bytes.
static size_t
yp_node_list_memsize(yp_node_list_t *node_list, yp_memsize_t *memsize) {
size_t size = sizeof(yp_node_list_t) + (node_list->capacity * sizeof(yp_node_t *));
for (size_t index = 0; index < node_list->size; index++) {
yp_node_memsize_node(node_list->nodes[index], memsize);
}
return size;
}
// Append a new node onto the end of the node list.
void
yp_node_list_append(yp_node_list_t *list, yp_node_t *node) {
if (list->size == list->capacity) {
list->capacity = list->capacity == 0 ? 4 : list->capacity * 2;
list->nodes = (yp_node_t **) realloc(list->nodes, sizeof(yp_node_t *) * list->capacity);
}
list->nodes[list->size++] = node;
}
YP_EXPORTED_FUNCTION void
yp_node_destroy(yp_parser_t *parser, yp_node_t *node);
// Deallocate the inner memory of a list of nodes. The parser argument is not
// used, but is here for the future possibility of pre-allocating memory pools.
static void
yp_node_list_free(yp_parser_t *parser, yp_node_list_t *list) {
if (list->capacity > 0) {
for (size_t index = 0; index < list->size; index++) {
yp_node_destroy(parser, list->nodes[index]);
}
free(list->nodes);
}
}
// Deallocate the space for a yp_node_t. Similarly to yp_node_alloc, we're not
// using the parser argument, but it's there to allow for the future possibility
// of pre-allocating larger memory pools.
YP_EXPORTED_FUNCTION void
yp_node_destroy(yp_parser_t *parser, yp_node_t *node) {
switch (YP_NODE_TYPE(node)) {
<%- nodes.each do |node| -%>
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
case <%= node.type %>:
<%- node.params.each do |param| -%>
<%- case param -%>
<%- when LocationParam, OptionalLocationParam, UInt32Param, FlagsParam, ConstantParam -%>
<%- when NodeParam -%>
yp_node_destroy(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when OptionalNodeParam -%>
if (((yp_<%= node.human %>_t *)node)-><%= param.name %> != NULL) {
yp_node_destroy(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>);
}
<%- when StringParam -%>
yp_string_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when NodeListParam -%>
yp_node_list_free(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when LocationListParam -%>
yp_location_list_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when ConstantListParam -%>
yp_constant_id_list_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- else -%>
<%- raise -%>
<%- end -%>
<%- end -%>
break;
<%- end -%>
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
default:
assert(false && "unreachable");
break;
}
free(node);
}
static void
yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize) {
memsize->node_count++;
switch (YP_NODE_TYPE(node)) {
<%- nodes.each do |node| -%>
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
case <%= node.type %>: {
memsize->memsize += sizeof(yp_<%= node.human %>_t);
<%- node.params.each do |param| -%>
<%- case param -%>
<%- when ConstantParam, UInt32Param, FlagsParam, LocationParam, OptionalLocationParam -%>
<%- when NodeParam -%>
yp_node_memsize_node((yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize);
<%- when OptionalNodeParam -%>
if (((yp_<%= node.human %>_t *)node)-><%= param.name %> != NULL) {
yp_node_memsize_node((yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize);
}
<%- when StringParam -%>
memsize->memsize += yp_string_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when NodeListParam -%>
yp_node_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize);
<%- when LocationListParam -%>
memsize->memsize += yp_location_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when ConstantListParam -%>
memsize->memsize += yp_constant_id_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- else -%>
<%- raise -%>
<%- end -%>
<%- end -%>
break;
}
<%- end -%>
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
}
}
// Calculates the memory footprint of a given node.
YP_EXPORTED_FUNCTION void
yp_node_memsize(yp_node_t *node, yp_memsize_t *memsize) {
*memsize = (yp_memsize_t) { .memsize = 0, .node_count = 0 };
yp_node_memsize_node(node, memsize);
}
// Returns a string representation of the given node type.
YP_EXPORTED_FUNCTION const char *
yp_node_type_to_str(yp_node_type_t node_type)
{
switch (node_type) {
<%- nodes.each do |node| -%>
case <%= node.type %>:
return "<%= node.type %>";
<%- end -%>
}
return "\0";
}

106
yarp/templates/src/prettyprint.c.erb Normal file
Просмотреть файл

@ -0,0 +1,106 @@
#include "yarp/defines.h"
#include <stdio.h>
#include "yarp/ast.h"
#include "yarp/parser.h"
#include "yarp/util/yp_buffer.h"
static void
prettyprint_location(yp_buffer_t *buffer, yp_parser_t *parser, yp_location_t *location) {
char printed[] = "[0000-0000]";
snprintf(printed, sizeof(printed), "[%04ld-%04ld]", (long int)(location->start - parser->start), (long int)(location->end - parser->start));
yp_buffer_append_str(buffer, printed, strlen(printed));
}
static void
prettyprint_node(yp_buffer_t *buffer, yp_parser_t *parser, yp_node_t *node) {
switch (YP_NODE_TYPE(node)) {
<%- nodes.each do |node| -%>
case <%= node.type %>: {
yp_buffer_append_str(buffer, "<%= node.name %>(", <%= node.name.length + 1 %>);
<%- node.params.each_with_index do |param, index| -%>
<%= "yp_buffer_append_str(buffer, \", \", 2);" if index != 0 -%>
<%- case param -%>
<%- when NodeParam -%>
prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when OptionalNodeParam -%>
if (((yp_<%= node.human %>_t *)node)-><%= param.name %> == NULL) {
yp_buffer_append_str(buffer, "nil", 3);
} else {
prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>);
}
<%- when StringParam -%>
yp_buffer_append_str(buffer, "\"", 1);
yp_buffer_append_str(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= param.name %>), yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= param.name %>));
yp_buffer_append_str(buffer, "\"", 1);
<%- when NodeListParam -%>
yp_buffer_append_str(buffer, "[", 1);
for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) {
if (index != 0) yp_buffer_append_str(buffer, ", ", 2);
prettyprint_node(buffer, parser, (yp_node_t *) ((yp_<%= node.human %>_t *) node)-><%= param.name %>.nodes[index]);
}
yp_buffer_append_str(buffer, "]", 1);
<%- when LocationListParam -%>
yp_buffer_append_str(buffer, "[", 1);
for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) {
if (index != 0) yp_buffer_append_str(buffer, ", ", 2);
prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>.locations[index]);
}
yp_buffer_append_str(buffer, "]", 1);
<%- when ConstantParam -%>
char <%= param.name %>_buffer[12];
snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= param.name %>);
yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer));
<%- when ConstantListParam -%>
yp_buffer_append_str(buffer, "[", 1);
for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) {
if (index != 0) yp_buffer_append_str(buffer, ", ", 2);
char <%= param.name %>_buffer[12];
snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= param.name %>.ids[index]);
yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer));
}
yp_buffer_append_str(buffer, "]", 1);
<%- when LocationParam -%>
prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when OptionalLocationParam -%>
if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.start == NULL) {
yp_buffer_append_str(buffer, "nil", 3);
} else {
prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>);
}
<%- when UInt32Param -%>
char <%= param.name %>_buffer[12];
snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "+%d", ((yp_<%= node.human %>_t *)node)-><%= param.name %>);
yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer));
<%- when FlagsParam -%>
char <%= param.name %>_buffer[12];
snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "+%d", node->flags >> <%= COMMON_FLAGS %>);
yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer));
<%- else -%>
<%- raise -%>
<%- end -%>
<%- end -%>
yp_buffer_append_str(buffer, ")", 1);
break;
}
<%- end -%>
}
}
void
yp_print_node(yp_parser_t *parser, yp_node_t *node) {
yp_buffer_t buffer;
if (!yp_buffer_init(&buffer)) return;
prettyprint_node(&buffer, parser, node);
printf("%.*s\n", (int) buffer.length, buffer.value);
yp_buffer_free(&buffer);
}
// Pretty-prints the AST represented by the given node to the given buffer.
YP_EXPORTED_FUNCTION void
yp_prettyprint(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
prettyprint_node(buffer, parser, node);
}

236
yarp/templates/src/serialize.c.erb Normal file
Просмотреть файл

@ -0,0 +1,236 @@
#include "yarp.h"
#include <stdio.h>
static inline uint32_t
yp_ptrdifft_to_u32(ptrdiff_t value) {
assert(value >= 0 && ((unsigned long) value) < UINT32_MAX);
return (uint32_t) value;
}
static inline uint32_t
yp_sizet_to_u32(size_t value) {
assert(value < UINT32_MAX);
return (uint32_t) value;
}
static void
serialize_location(yp_parser_t *parser, yp_location_t *location, yp_buffer_t *buffer) {
assert(location->start);
assert(location->end);
assert(location->start <= location->end);
yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(location->start - parser->start));
yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(location->end - location->start));
}
void
yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
yp_buffer_append_u8(buffer, (uint8_t) YP_NODE_TYPE(node));
size_t offset = buffer->length;
serialize_location(parser, &node->location, buffer);
switch (YP_NODE_TYPE(node)) {
<%- nodes.each do |node| -%>
case <%= node.type %>: {
<%- if node.needs_serialized_length? -%>
// serialize length
// encoding of location u32s make us need to save this offset.
size_t length_offset = buffer->length;
yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */
<%- end -%>
<%- node.params.each do |param| -%>
<%- case param -%>
<%- when NodeParam -%>
yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer);
<%- when OptionalNodeParam -%>
if (((yp_<%= node.human %>_t *)node)-><%= param.name %> == NULL) {
yp_buffer_append_u8(buffer, 0);
} else {
yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer);
}
<%- when StringParam -%>
uint32_t <%= param.name %>_length = yp_sizet_to_u32(yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= param.name %>));
yp_buffer_append_u32(buffer, <%= param.name %>_length);
yp_buffer_append_str(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= param.name %>), <%= param.name %>_length);
<%- when NodeListParam -%>
uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size);
yp_buffer_append_u32(buffer, <%= param.name %>_size);
for (uint32_t index = 0; index < <%= param.name %>_size; index++) {
yp_serialize_node(parser, (yp_node_t *) ((yp_<%= node.human %>_t *)node)-><%= param.name %>.nodes[index], buffer);
}
<%- when LocationListParam -%>
uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size);
yp_buffer_append_u32(buffer, <%= param.name %>_size);
for (uint32_t index = 0; index < <%= param.name %>_size; index++) {
serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>.locations[index], buffer);
}
<%- when ConstantParam -%>
yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>));
<%- when ConstantListParam -%>
uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size);
yp_buffer_append_u32(buffer, <%= param.name %>_size);
for (uint32_t index = 0; index < <%= param.name %>_size; index++) {
yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.ids[index]));
}
<%- when LocationParam -%>
serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer);
<%- when OptionalLocationParam -%>
if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.start == NULL) {
yp_buffer_append_u8(buffer, 0);
} else {
yp_buffer_append_u8(buffer, 1);
serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer);
}
<%- when UInt32Param -%>
yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= param.name %>);
<%- when FlagsParam -%>
yp_buffer_append_u32(buffer, node->flags >> <%= COMMON_FLAGS %>);
<%- else -%>
<%- raise -%>
<%- end -%>
<%- end -%>
<%- if node.needs_serialized_length? -%>
// serialize length
uint32_t length = yp_sizet_to_u32(buffer->length - offset - sizeof(uint32_t));
memcpy(buffer->value + length_offset, &length, sizeof(uint32_t));
<%- end -%>
break;
}
<%- end -%>
}
}
void yp_serialize_comment(yp_parser_t *parser, yp_comment_t *comment, yp_buffer_t *buffer) {
// serialize type
yp_buffer_append_u8(buffer, (uint8_t) comment->type);
// serialize location
yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->start - parser->start));
yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(comment->end - comment->start));
}
void yp_serialize_comment_list(yp_parser_t *parser, yp_list_t list, yp_buffer_t *buffer) {
yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(&list)));
yp_comment_t *comment;
for (comment = (yp_comment_t *) list.head; comment != NULL; comment = (yp_comment_t *) comment->node.next) {
yp_serialize_comment(parser, comment, buffer);
}
}
void yp_serialize_diagnostic(yp_parser_t *parser, yp_diagnostic_t *diagnostic, yp_buffer_t *buffer) {
// serialize message
size_t message_length = strlen(diagnostic->message);
yp_buffer_append_u32(buffer, yp_sizet_to_u32(message_length));
yp_buffer_append_str(buffer, diagnostic->message, message_length);
// serialize location
yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(diagnostic->start - parser->start));
yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(diagnostic->end - diagnostic->start));
}
void yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t list, yp_buffer_t *buffer) {
yp_buffer_append_u32(buffer, yp_sizet_to_u32(yp_list_size(&list)));
yp_diagnostic_t *diagnostic;
for (diagnostic = (yp_diagnostic_t *) list.head; diagnostic != NULL; diagnostic = (yp_diagnostic_t *) diagnostic->node.next) {
yp_serialize_diagnostic(parser, diagnostic, buffer);
}
}
#line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>"
void
yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
// First, serialize the encoding of the parser.
size_t encoding_length = strlen(parser->encoding.name);
yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length));
yp_buffer_append_str(buffer, parser->encoding.name, encoding_length);
// Serialize the comments
yp_serialize_comment_list(parser, parser->comment_list, buffer);
// Serialize the errors
yp_serialize_diagnostic_list(parser, parser->error_list, buffer);
// Serialize the warnings
yp_serialize_diagnostic_list(parser, parser->warning_list, buffer);
// Here we're going to leave space for the offset of the constant pool in
// the buffer.
size_t offset = buffer->length;
yp_buffer_append_zeroes(buffer, 4);
// Next, encode the length of the constant pool.
yp_buffer_append_u32(buffer, yp_sizet_to_u32(parser->constant_pool.size));
// Now we're going to serialize the content of the node.
yp_serialize_node(parser, node, buffer);
// Now we're going to serialize the offset of the constant pool back where
// we left space for it.
uint32_t length = yp_sizet_to_u32(buffer->length);
memcpy(buffer->value + offset, &length, sizeof(uint32_t));
// Now we're going to serialize the constant pool.
offset = buffer->length;
yp_buffer_append_zeroes(buffer, parser->constant_pool.size * 8);
yp_constant_t *constant;
for (size_t index = 0; index < parser->constant_pool.capacity; index++) {
constant = &parser->constant_pool.constants[index];
// If we find a constant at this index, serialize it at the correct
// index in the buffer.
if (constant->id != 0) {
size_t buffer_offset = offset + ((constant->id - 1) * 8);
uint32_t source_offset = yp_ptrdifft_to_u32(constant->start - parser->start);
uint32_t constant_length = yp_sizet_to_u32(constant->length);
memcpy(buffer->value + buffer_offset, &source_offset, 4);
memcpy(buffer->value + buffer_offset + 4, &constant_length, 4);
}
}
}
static void
serialize_token(void *data, yp_parser_t *parser, yp_token_t *token) {
yp_buffer_t *buffer = (yp_buffer_t *) data;
yp_buffer_append_u32(buffer, token->type);
yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->start - parser->start));
yp_buffer_append_u32(buffer, yp_ptrdifft_to_u32(token->end - token->start));
yp_buffer_append_u32(buffer, parser->lex_state);
}
YP_EXPORTED_FUNCTION void
yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer) {
yp_parser_t parser;
yp_parser_init(&parser, source, size, filepath);
yp_lex_callback_t lex_callback = (yp_lex_callback_t) {
.data = (void *) buffer,
.callback = serialize_token,
};
parser.lex_callback = &lex_callback;
yp_node_t *node = yp_parse(&parser);
// Append 0 to mark end of tokens
yp_buffer_append_u32(buffer, 0);
// Serialize the comments
yp_serialize_comment_list(&parser, parser.comment_list, buffer);
// Serialize the errors
yp_serialize_diagnostic_list(&parser, parser.error_list, buffer);
// Serialize the warnings
yp_serialize_diagnostic_list(&parser, parser.warning_list, buffer);
yp_node_destroy(&parser, node);
yp_parser_free(&parser);
}

18
yarp/templates/src/token_type.c.erb Normal file
Просмотреть файл

@ -0,0 +1,18 @@
#include <string.h>
#include "yarp/ast.h"
// Returns a string representation of the given token type.
YP_EXPORTED_FUNCTION const char *
yp_token_type_to_str(yp_token_type_t token_type)
{
switch (token_type) {
<%- tokens.each do |token| -%>
case YP_TOKEN_<%= token.name %>:
return "<%= token.name %>";
<%- end -%>
case YP_TOKEN_MAXIMUM:
return "MAXIMUM";
}
return "\0";
}

329
yarp/templates/template.rb Executable file
Просмотреть файл

@ -0,0 +1,329 @@
#!/usr/bin/env ruby
require "erb"
require "fileutils"
require "yaml"
COMMON_FLAGS = 1
class Param
attr_reader :name, :options
def initialize(name:, type:, **options)
@name, @type, @options = name, type, options
end
end
module KindTypes
def c_type
if options[:kind]
"yp_#{options[:kind].gsub(/(?<=.)[A-Z]/, "_\\0").downcase}"
else
"yp_node"
end
end
def java_type
options[:kind] || "Node"
end
def java_cast
if options[:kind]
"(Nodes.#{options[:kind]}) "
else
""
end
end
end
# This represents a parameter to a node that is itself a node. We pass them as
# references and store them as references.
class NodeParam < Param
include KindTypes
def rbs_class
"Node"
end
end
# This represents a parameter to a node that is itself a node and can be
# optionally null. We pass them as references and store them as references.
class OptionalNodeParam < Param
include KindTypes
def rbs_class
"Node?"
end
end
SingleNodeParam = -> (node) { NodeParam === node or OptionalNodeParam === node }
# This represents a parameter to a node that is a list of nodes. We pass them as
# references and store them as references.
class NodeListParam < Param
def rbs_class
"Array[Node]"
end
def java_type
"Node[]"
end
end
# This represents a parameter to a node that is a list of locations.
class LocationListParam < Param
def rbs_class
"Array[Location]"
end
def java_type
"Location[]"
end
end
# This represents a parameter to a node that is the ID of a string interned
# through the parser's constant pool.
class ConstantParam < Param
def rbs_class
"Symbol"
end
def java_type
"byte[]"
end
end
# This represents a parameter to a node that is a list of IDs that are
# associated with strings interned through the parser's constant pool.
class ConstantListParam < Param
def rbs_class
"Array[Symbol]"
end
def java_type
"byte[][]"
end
end
# This represents a parameter to a node that is a string.
class StringParam < Param
def rbs_class
"String"
end
def java_type
"byte[]"
end
end
# This represents a parameter to a node that is a location.
class LocationParam < Param
def rbs_class
"Location"
end
def java_type
"Location"
end
end
# This represents a parameter to a node that is a location that is optional.
class OptionalLocationParam < Param
def rbs_class
"Location?"
end
def java_type
"Location"
end
end
# This represents an integer parameter.
class UInt32Param < Param
def rbs_class
"Integer"
end
def java_type
"int"
end
end
# This represents a set of flags. It is very similar to the UInt32Param, but can
# be directly embedded into the flags field on the struct and provides
# convenient methods for checking if a flag is set.
class FlagsParam < Param
def rbs_class
"Integer"
end
def java_type
"short"
end
def kind
options.fetch(:kind)
end
end
PARAM_TYPES = {
"node" => NodeParam,
"node?" => OptionalNodeParam,
"node[]" => NodeListParam,
"string" => StringParam,
"location[]" => LocationListParam,
"constant" => ConstantParam,
"constant[]" => ConstantListParam,
"location" => LocationParam,
"location?" => OptionalLocationParam,
"uint32" => UInt32Param,
"flags" => FlagsParam
}
# This class represents a node in the tree, configured by the config.yml file in
# YAML format. It contains information about the name of the node and the
# various child nodes it contains.
class NodeType
attr_reader :name, :type, :human, :params, :newline, :comment
def initialize(config)
@name = config.fetch("name")
type = @name.gsub(/(?<=.)[A-Z]/, "_\\0")
@type = "YP_NODE_#{type.upcase}"
@human = type.downcase
@params = config.fetch("child_nodes", []).map do |param|
param_type = PARAM_TYPES[param.fetch("type")] ||
raise("Unknown param type: #{param["type"].inspect}")
param_type.new(**param.transform_keys(&:to_sym))
end
@newline = config.fetch("newline", true)
@comment = config.fetch("comment")
end
# Should emit serialized length of node so implementations can skip
# the node to enable lazy parsing.
def needs_serialized_length?
@name == "DefNode"
end
end
# This represents a token in the lexer. They are configured through the
# config.yml file for now, but this will probably change as we transition to
# storing semantic strings instead of the lexer tokens.
class Token
attr_reader :name, :value, :comment
def initialize(config)
@name = config.fetch("name")
@value = config["value"]
@comment = config.fetch("comment")
end
def declaration
output = []
output << "YP_TOKEN_#{name}"
output << " = #{value}" if value
output << ", // #{comment}"
output.join
end
end
# Represents a set of flags that should be internally represented with an enum.
class Flags
attr_reader :name, :human, :values
def initialize(config)
@name = config.fetch("name")
@human = @name.gsub(/(?<=.)[A-Z]/, "_\\0").downcase
@values = config.fetch("values").map { |flag| Flag.new(flag) }
end
end
class Flag
attr_reader :name, :camelcase, :comment
def initialize(config)
@name = config.fetch("name")
@camelcase = @name.split("_").map(&:capitalize).join
@comment = config.fetch("comment")
end
end
# This templates out a file using ERB with the given locals. The locals are
# derived from the config.yml file.
def template(name, locals, write_to: nil)
filepath = "templates/#{name}.erb"
template = File.expand_path("../#{filepath}", __dir__)
write_to ||= File.expand_path("../#{name}", __dir__)
if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
erb = ERB.new(File.read(template), trim_mode: "-")
else
erb = ERB.new(File.read(template), nil, "-")
end
erb.filename = template
non_ruby_heading = <<~HEADING
/******************************************************************************/
/* This file is generated by the bin/template script and should not be */
/* modified manually. See */
/* #{filepath + " " * (74 - filepath.size) } */
/* if you are looking to modify the */
/* template */
/******************************************************************************/
HEADING
ruby_heading = <<~HEADING
# frozen_string_literal: true
=begin
This file is generated by the bin/template script and should not be
modified manually. See #{filepath}
if you are looking to modify the template
=end
HEADING
heading = if File.extname(filepath.gsub(".erb", "")) == ".rb"
ruby_heading
else
non_ruby_heading
end
contents = heading + erb.result_with_hash(locals)
FileUtils.mkdir_p(File.dirname(write_to))
File.write(write_to, contents)
end
def locals
config = YAML.load_file(File.expand_path("../config.yml", __dir__))
{
nodes: config.fetch("nodes").map { |node| NodeType.new(node) }.sort_by(&:name),
tokens: config.fetch("tokens").map { |token| Token.new(token) },
flags: config.fetch("flags").map { |flags| Flags.new(flags) }
}
end
TEMPLATES = [
"ext/yarp/api_node.c",
"include/yarp/ast.h",
"java/org/yarp/Loader.java",
"java/org/yarp/Nodes.java",
"java/org/yarp/AbstractNodeVisitor.java",
"lib/yarp/node.rb",
"lib/yarp/serialize.rb",
"src/node.c",
"src/prettyprint.c",
"src/serialize.c",
"src/token_type.c"
]
if __FILE__ == $0
if ARGV.empty?
TEMPLATES.each { |f| template(f, locals) }
else
name, write_to = ARGV
template(name, locals, write_to: write_to)
end
end

347
yarp/token_type.c
Просмотреть файл

@ -1,347 +0,0 @@
/******************************************************************************/
/* This file is generated by the bin/template script and should not be */
/* modified manually. See */
/* templates/src/token_type.c.erb */
/* if you are looking to modify the */
/* template */
/******************************************************************************/
#include <string.h>
#include "yarp/ast.h"
// Returns a string representation of the given token type.
YP_EXPORTED_FUNCTION const char *
yp_token_type_to_str(yp_token_type_t token_type)
{
switch (token_type) {
case YP_TOKEN_EOF:
return "EOF";
case YP_TOKEN_MISSING:
return "MISSING";
case YP_TOKEN_NOT_PROVIDED:
return "NOT_PROVIDED";
case YP_TOKEN_AMPERSAND:
return "AMPERSAND";
case YP_TOKEN_AMPERSAND_AMPERSAND:
return "AMPERSAND_AMPERSAND";
case YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL:
return "AMPERSAND_AMPERSAND_EQUAL";
case YP_TOKEN_AMPERSAND_DOT:
return "AMPERSAND_DOT";
case YP_TOKEN_AMPERSAND_EQUAL:
return "AMPERSAND_EQUAL";
case YP_TOKEN_BACKTICK:
return "BACKTICK";
case YP_TOKEN_BACK_REFERENCE:
return "BACK_REFERENCE";
case YP_TOKEN_BANG:
return "BANG";
case YP_TOKEN_BANG_EQUAL:
return "BANG_EQUAL";
case YP_TOKEN_BANG_TILDE:
return "BANG_TILDE";
case YP_TOKEN_BRACE_LEFT:
return "BRACE_LEFT";
case YP_TOKEN_BRACE_RIGHT:
return "BRACE_RIGHT";
case YP_TOKEN_BRACKET_LEFT:
return "BRACKET_LEFT";
case YP_TOKEN_BRACKET_LEFT_ARRAY:
return "BRACKET_LEFT_ARRAY";
case YP_TOKEN_BRACKET_LEFT_RIGHT:
return "BRACKET_LEFT_RIGHT";
case YP_TOKEN_BRACKET_LEFT_RIGHT_EQUAL:
return "BRACKET_LEFT_RIGHT_EQUAL";
case YP_TOKEN_BRACKET_RIGHT:
return "BRACKET_RIGHT";
case YP_TOKEN_CARET:
return "CARET";
case YP_TOKEN_CARET_EQUAL:
return "CARET_EQUAL";
case YP_TOKEN_CHARACTER_LITERAL:
return "CHARACTER_LITERAL";
case YP_TOKEN_CLASS_VARIABLE:
return "CLASS_VARIABLE";
case YP_TOKEN_COLON:
return "COLON";
case YP_TOKEN_COLON_COLON:
return "COLON_COLON";
case YP_TOKEN_COMMA:
return "COMMA";
case YP_TOKEN_COMMENT:
return "COMMENT";
case YP_TOKEN_CONSTANT:
return "CONSTANT";
case YP_TOKEN_DOT:
return "DOT";
case YP_TOKEN_DOT_DOT:
return "DOT_DOT";
case YP_TOKEN_DOT_DOT_DOT:
return "DOT_DOT_DOT";
case YP_TOKEN_EMBDOC_BEGIN:
return "EMBDOC_BEGIN";
case YP_TOKEN_EMBDOC_END:
return "EMBDOC_END";
case YP_TOKEN_EMBDOC_LINE:
return "EMBDOC_LINE";
case YP_TOKEN_EMBEXPR_BEGIN:
return "EMBEXPR_BEGIN";
case YP_TOKEN_EMBEXPR_END:
return "EMBEXPR_END";
case YP_TOKEN_EMBVAR:
return "EMBVAR";
case YP_TOKEN_EQUAL:
return "EQUAL";
case YP_TOKEN_EQUAL_EQUAL:
return "EQUAL_EQUAL";
case YP_TOKEN_EQUAL_EQUAL_EQUAL:
return "EQUAL_EQUAL_EQUAL";
case YP_TOKEN_EQUAL_GREATER:
return "EQUAL_GREATER";
case YP_TOKEN_EQUAL_TILDE:
return "EQUAL_TILDE";
case YP_TOKEN_FLOAT:
return "FLOAT";
case YP_TOKEN_FLOAT_IMAGINARY:
return "FLOAT_IMAGINARY";
case YP_TOKEN_FLOAT_RATIONAL:
return "FLOAT_RATIONAL";
case YP_TOKEN_FLOAT_RATIONAL_IMAGINARY:
return "FLOAT_RATIONAL_IMAGINARY";
case YP_TOKEN_GLOBAL_VARIABLE:
return "GLOBAL_VARIABLE";
case YP_TOKEN_GREATER:
return "GREATER";
case YP_TOKEN_GREATER_EQUAL:
return "GREATER_EQUAL";
case YP_TOKEN_GREATER_GREATER:
return "GREATER_GREATER";
case YP_TOKEN_GREATER_GREATER_EQUAL:
return "GREATER_GREATER_EQUAL";
case YP_TOKEN_HEREDOC_END:
return "HEREDOC_END";
case YP_TOKEN_HEREDOC_START:
return "HEREDOC_START";
case YP_TOKEN_IDENTIFIER:
return "IDENTIFIER";
case YP_TOKEN_IGNORED_NEWLINE:
return "IGNORED_NEWLINE";
case YP_TOKEN_INSTANCE_VARIABLE:
return "INSTANCE_VARIABLE";
case YP_TOKEN_INTEGER:
return "INTEGER";
case YP_TOKEN_INTEGER_IMAGINARY:
return "INTEGER_IMAGINARY";
case YP_TOKEN_INTEGER_RATIONAL:
return "INTEGER_RATIONAL";
case YP_TOKEN_INTEGER_RATIONAL_IMAGINARY:
return "INTEGER_RATIONAL_IMAGINARY";
case YP_TOKEN_KEYWORD_ALIAS:
return "KEYWORD_ALIAS";
case YP_TOKEN_KEYWORD_AND:
return "KEYWORD_AND";
case YP_TOKEN_KEYWORD_BEGIN:
return "KEYWORD_BEGIN";
case YP_TOKEN_KEYWORD_BEGIN_UPCASE:
return "KEYWORD_BEGIN_UPCASE";
case YP_TOKEN_KEYWORD_BREAK:
return "KEYWORD_BREAK";
case YP_TOKEN_KEYWORD_CASE:
return "KEYWORD_CASE";
case YP_TOKEN_KEYWORD_CLASS:
return "KEYWORD_CLASS";
case YP_TOKEN_KEYWORD_DEF:
return "KEYWORD_DEF";
case YP_TOKEN_KEYWORD_DEFINED:
return "KEYWORD_DEFINED";
case YP_TOKEN_KEYWORD_DO:
return "KEYWORD_DO";
case YP_TOKEN_KEYWORD_DO_LOOP:
return "KEYWORD_DO_LOOP";
case YP_TOKEN_KEYWORD_ELSE:
return "KEYWORD_ELSE";
case YP_TOKEN_KEYWORD_ELSIF:
return "KEYWORD_ELSIF";
case YP_TOKEN_KEYWORD_END:
return "KEYWORD_END";
case YP_TOKEN_KEYWORD_END_UPCASE:
return "KEYWORD_END_UPCASE";
case YP_TOKEN_KEYWORD_ENSURE:
return "KEYWORD_ENSURE";
case YP_TOKEN_KEYWORD_FALSE:
return "KEYWORD_FALSE";
case YP_TOKEN_KEYWORD_FOR:
return "KEYWORD_FOR";
case YP_TOKEN_KEYWORD_IF:
return "KEYWORD_IF";
case YP_TOKEN_KEYWORD_IF_MODIFIER:
return "KEYWORD_IF_MODIFIER";
case YP_TOKEN_KEYWORD_IN:
return "KEYWORD_IN";
case YP_TOKEN_KEYWORD_MODULE:
return "KEYWORD_MODULE";
case YP_TOKEN_KEYWORD_NEXT:
return "KEYWORD_NEXT";
case YP_TOKEN_KEYWORD_NIL:
return "KEYWORD_NIL";
case YP_TOKEN_KEYWORD_NOT:
return "KEYWORD_NOT";
case YP_TOKEN_KEYWORD_OR:
return "KEYWORD_OR";
case YP_TOKEN_KEYWORD_REDO:
return "KEYWORD_REDO";
case YP_TOKEN_KEYWORD_RESCUE:
return "KEYWORD_RESCUE";
case YP_TOKEN_KEYWORD_RESCUE_MODIFIER:
return "KEYWORD_RESCUE_MODIFIER";
case YP_TOKEN_KEYWORD_RETRY:
return "KEYWORD_RETRY";
case YP_TOKEN_KEYWORD_RETURN:
return "KEYWORD_RETURN";
case YP_TOKEN_KEYWORD_SELF:
return "KEYWORD_SELF";
case YP_TOKEN_KEYWORD_SUPER:
return "KEYWORD_SUPER";
case YP_TOKEN_KEYWORD_THEN:
return "KEYWORD_THEN";
case YP_TOKEN_KEYWORD_TRUE:
return "KEYWORD_TRUE";
case YP_TOKEN_KEYWORD_UNDEF:
return "KEYWORD_UNDEF";
case YP_TOKEN_KEYWORD_UNLESS:
return "KEYWORD_UNLESS";
case YP_TOKEN_KEYWORD_UNLESS_MODIFIER:
return "KEYWORD_UNLESS_MODIFIER";
case YP_TOKEN_KEYWORD_UNTIL:
return "KEYWORD_UNTIL";
case YP_TOKEN_KEYWORD_UNTIL_MODIFIER:
return "KEYWORD_UNTIL_MODIFIER";
case YP_TOKEN_KEYWORD_WHEN:
return "KEYWORD_WHEN";
case YP_TOKEN_KEYWORD_WHILE:
return "KEYWORD_WHILE";
case YP_TOKEN_KEYWORD_WHILE_MODIFIER:
return "KEYWORD_WHILE_MODIFIER";
case YP_TOKEN_KEYWORD_YIELD:
return "KEYWORD_YIELD";
case YP_TOKEN_KEYWORD___ENCODING__:
return "KEYWORD___ENCODING__";
case YP_TOKEN_KEYWORD___FILE__:
return "KEYWORD___FILE__";
case YP_TOKEN_KEYWORD___LINE__:
return "KEYWORD___LINE__";
case YP_TOKEN_LABEL:
return "LABEL";
case YP_TOKEN_LABEL_END:
return "LABEL_END";
case YP_TOKEN_LAMBDA_BEGIN:
return "LAMBDA_BEGIN";
case YP_TOKEN_LESS:
return "LESS";
case YP_TOKEN_LESS_EQUAL:
return "LESS_EQUAL";
case YP_TOKEN_LESS_EQUAL_GREATER:
return "LESS_EQUAL_GREATER";
case YP_TOKEN_LESS_LESS:
return "LESS_LESS";
case YP_TOKEN_LESS_LESS_EQUAL:
return "LESS_LESS_EQUAL";
case YP_TOKEN_MINUS:
return "MINUS";
case YP_TOKEN_MINUS_EQUAL:
return "MINUS_EQUAL";
case YP_TOKEN_MINUS_GREATER:
return "MINUS_GREATER";
case YP_TOKEN_NEWLINE:
return "NEWLINE";
case YP_TOKEN_NUMBERED_REFERENCE:
return "NUMBERED_REFERENCE";
case YP_TOKEN_PARENTHESIS_LEFT:
return "PARENTHESIS_LEFT";
case YP_TOKEN_PARENTHESIS_LEFT_PARENTHESES:
return "PARENTHESIS_LEFT_PARENTHESES";
case YP_TOKEN_PARENTHESIS_RIGHT:
return "PARENTHESIS_RIGHT";
case YP_TOKEN_PERCENT:
return "PERCENT";
case YP_TOKEN_PERCENT_EQUAL:
return "PERCENT_EQUAL";
case YP_TOKEN_PERCENT_LOWER_I:
return "PERCENT_LOWER_I";
case YP_TOKEN_PERCENT_LOWER_W:
return "PERCENT_LOWER_W";
case YP_TOKEN_PERCENT_LOWER_X:
return "PERCENT_LOWER_X";
case YP_TOKEN_PERCENT_UPPER_I:
return "PERCENT_UPPER_I";
case YP_TOKEN_PERCENT_UPPER_W:
return "PERCENT_UPPER_W";
case YP_TOKEN_PIPE:
return "PIPE";
case YP_TOKEN_PIPE_EQUAL:
return "PIPE_EQUAL";
case YP_TOKEN_PIPE_PIPE:
return "PIPE_PIPE";
case YP_TOKEN_PIPE_PIPE_EQUAL:
return "PIPE_PIPE_EQUAL";
case YP_TOKEN_PLUS:
return "PLUS";
case YP_TOKEN_PLUS_EQUAL:
return "PLUS_EQUAL";
case YP_TOKEN_QUESTION_MARK:
return "QUESTION_MARK";
case YP_TOKEN_REGEXP_BEGIN:
return "REGEXP_BEGIN";
case YP_TOKEN_REGEXP_END:
return "REGEXP_END";
case YP_TOKEN_SEMICOLON:
return "SEMICOLON";
case YP_TOKEN_SLASH:
return "SLASH";
case YP_TOKEN_SLASH_EQUAL:
return "SLASH_EQUAL";
case YP_TOKEN_STAR:
return "STAR";
case YP_TOKEN_STAR_EQUAL:
return "STAR_EQUAL";
case YP_TOKEN_STAR_STAR:
return "STAR_STAR";
case YP_TOKEN_STAR_STAR_EQUAL:
return "STAR_STAR_EQUAL";
case YP_TOKEN_STRING_BEGIN:
return "STRING_BEGIN";
case YP_TOKEN_STRING_CONTENT:
return "STRING_CONTENT";
case YP_TOKEN_STRING_END:
return "STRING_END";
case YP_TOKEN_SYMBOL_BEGIN:
return "SYMBOL_BEGIN";
case YP_TOKEN_TILDE:
return "TILDE";
case YP_TOKEN_UAMPERSAND:
return "UAMPERSAND";
case YP_TOKEN_UCOLON_COLON:
return "UCOLON_COLON";
case YP_TOKEN_UDOT_DOT:
return "UDOT_DOT";
case YP_TOKEN_UDOT_DOT_DOT:
return "UDOT_DOT_DOT";
case YP_TOKEN_UMINUS:
return "UMINUS";
case YP_TOKEN_UMINUS_NUM:
return "UMINUS_NUM";
case YP_TOKEN_UPLUS:
return "UPLUS";
case YP_TOKEN_USTAR:
return "USTAR";
case YP_TOKEN_USTAR_STAR:
return "USTAR_STAR";
case YP_TOKEN_WORDS_SEP:
return "WORDS_SEP";
case YP_TOKEN___END__:
return "__END__";
case YP_TOKEN_MAXIMUM:
return "MAXIMUM";
}
return "\0";
}