lib/yard/parser/ruby/legacy/ruby_lex.rb in yard-0.6.8 vs lib/yard/parser/ruby/legacy/ruby_lex.rb in yard-0.7.0
- old
+ new
@@ -10,24 +10,24 @@
EXPR_END = :EXPR_END
EXPR_ARG = :EXPR_ARG
EXPR_FNAME = :EXPR_FNAME
EXPR_DOT = :EXPR_DOT
EXPR_CLASS = :EXPR_CLASS
-
+
# Represents a token in the Ruby lexer
class Token
# @return [Integer] the line number in the file/stream the token is
# located.
attr_reader :line_no
-
+
# @return [Integer] the character number in the file/stream the token
# is located.
attr_reader :char_no
-
+
# @return [String] the token text value
attr_reader :text
-
+
# @return [Symbol] the lexical state at the token
attr_accessor :lex_state
# @private
NO_TEXT = "??".freeze
@@ -40,33 +40,33 @@
@char_no = char_no
@text = NO_TEXT
end
# Chainable way to sets the text attribute
- #
+ #
# @param [String] text the new text
# @return [Token] this token object
def set_text(text)
@text = text
self
end
end
-
+
# Represents a block
class TkBlockContents < Token
def text; '...' end
end
-
+
# Represents an end statement
class TkStatementEnd < Token
def text; '' end
end
class TkNode < Token
attr :node
end
-
+
# Represents whitespace
class TkWhitespace < Token
end
# Represents a Ruby identifier
@@ -129,12 +129,12 @@
when String, Symbol
source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token
if (tk = source[token]).nil?
IRB.fail TkReading2TokenNoKey, token
end
- tk = Token(tk[0], value)
- else
+ tk = Token(tk[0], value)
+ else
if token
tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
token.new(@prev_line_no, @prev_char_no)
else
token.new(@prev_line_no, @prev_char_no, value)
@@ -234,18 +234,18 @@
[:TkCOLON3, TkOp],
[:OPASGN, TkOp], # +=, -= etc. #
[:TkASSOC, TkOp, "=>"],
[:TkQUESTION, TkOp, "?"], #?
[:TkCOLON, TkOp, ":"], #:
-
+
# [:TkfLPAREN], # func( #
# [:TkfLBRACK], # func[ #
# [:TkfLBRACE], # func{ #
[:TkSTAR], # *arg
[:TkAMPER], # &arg #
# [:TkSYMBOL, TkId], # :SYMBOL
- [:TkSYMBEG, TkId],
+ [:TkSYMBEG, TkId],
[:TkGT, TkOp, ">"],
[:TkLT, TkOp, "<"],
[:TkPLUS, TkOp, "+"],
[:TkMINUS, TkOp, "-"],
[:TkMULT, TkOp, "*"],
@@ -293,11 +293,11 @@
end
token_c = Class.new super_token
RubyToken.const_set token_n, token_c
# token_c.inspect
-
+
if reading
if TkReading2Token[reading]
IRB.fail TkReading2TokenDuplicateError, token_n, reading
end
if opts.empty?
@@ -356,13 +356,13 @@
# here document. Once complete, it needs to read the rest of the
# original line, but then skip the here document body.
#
# @private
class BufferedReader
-
+
attr_reader :line_num
-
+
def initialize(content)
if /\t/ =~ content
tab_width = 2
content = content.split(/\n/).map do |line|
1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)} && $~ #`
@@ -377,65 +377,65 @@
@line_num = 1
@read_back_offset = 0
@last_newline = 0
@newline_pending = false
end
-
+
def column
@offset - @last_newline
end
-
+
def getc
return nil if @offset >= @size
ch = @content[@offset, 1]
-
+
@offset += 1
@hwm = @offset if @hwm < @offset
-
+
if @newline_pending
@line_num += 1
@last_newline = @offset - 1
@newline_pending = false
end
-
+
if ch == "\n"
@newline_pending = true
end
ch
end
-
+
def getc_already_read
getc
end
-
+
def ungetc(ch)
raise "unget past beginning of file" if @offset <= 0
@offset -= 1
if @content[@offset] == ?\n
@newline_pending = false
end
end
-
+
def get_read
res = @content[@read_back_offset...@offset]
@read_back_offset = @offset
res
end
-
+
def peek(at)
pos = @offset + at
if pos >= @size
nil
else
@content[pos, 1]
end
end
-
+
def peek_equal(str)
@content[@offset, str.length] == str
end
-
+
def divert_read_from(reserve)
@content[@offset, 0] = reserve
@size = @content.size
end
end
@@ -444,14 +444,14 @@
extend Exception2MessageMapper
def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
- def_exception(:TkReading2TokenDuplicateError,
+ def_exception(:TkReading2TokenDuplicateError,
"key duplicate(token_n='%s', key='%s')")
def_exception(:SyntaxError, "%s")
-
+
include RubyToken
include IRB
attr_reader :continue
attr_reader :lex_state
@@ -471,11 +471,11 @@
@ltype = nil
@quoted = nil
@lex_state = EXPR_BEG
@space_seen = false
-
+
@continue = false
@line = ""
@skip_space = false
@read_auto_clean_up = false
@@ -567,11 +567,11 @@
p tk if $DEBUG
tk.lex_state = lex_state if tk
tk
end
end
-
+
ENINDENT_CLAUSE = [
"case", "class", "def", "do", "for", "if",
"module", "unless", "until", "while", "begin" #, "when"
]
ACCEPTS_COLON = ["if", "for", "unless", "until", "while"]
@@ -584,11 +584,11 @@
"x" => "\`",
"r" => "/",
"w" => "]",
"W" => "]"
}
-
+
PERCENT_PAREN = {
"{" => "}",
"[" => "]",
"<" => ">",
"(" => ")"
@@ -668,23 +668,23 @@
@lex_state = EXPR_BEG
end
Token(TkNL).set_text("\n")
end
- @OP.def_rules("*", "**",
+ @OP.def_rules("*", "**",
"!", "!=", "!~",
- "=", "==", "===",
- "=~", "<=>",
+ "=", "==", "===",
+ "=~", "<=>",
"<", "<=",
">", ">=", ">>") do |op, io|
@lex_state = EXPR_BEG
Token(op).set_text(op)
end
@OP.def_rules("<<") do |op, io|
tk = nil
- if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
+ if @lex_state != EXPR_END && @lex_state != EXPR_CLASS &&
(@lex_state != EXPR_ARG || @space_seen)
c = peek(0)
tk = identify_here_document if /[-\w_\"\'\`]/ =~ c
end
if !tk
@@ -730,12 +730,12 @@
@OP.def_rules("&", "&&", "|", "||") do |op, io|
@lex_state = EXPR_BEG
Token(op).set_text(op)
end
-
- @OP.def_rules("+=", "-=", "*=", "**=",
+
+ @OP.def_rules("+=", "-=", "*=", "**=",
"&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io|
@lex_state = EXPR_BEG
op =~ /^(.*)=$/
Token(TkOPASGN, $1).set_text(op)
end
@@ -782,20 +782,20 @@
Token(op).set_text(op)
end
lex_int2
end
-
+
def lex_int2
@OP.def_rules("]", "}", ")") do
|op, io|
@lex_state = EXPR_END
@indent -= 1
Token(op).set_text(op)
end
- @OP.def_rule(":") do
+ @OP.def_rule(":") do
if (@colonblock_seen && @lex_state != EXPR_BEG) || peek(0) =~ /\s/
@lex_state = EXPR_BEG
tk = Token(TkCOLON)
else
@lex_state = EXPR_FNAME
@@ -823,11 +823,11 @@
getc
@lex_state = EXPR_BEG
Token(TkOPASGN, :/).set_text("/=") #")
elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/
identify_string(op)
- else
+ else
@lex_state = EXPR_BEG
Token("/").set_text(op)
end
end
@@ -838,11 +838,11 @@
# @OP.def_rules("^=") do
# @lex_state = EXPR_BEG
# Token(TkOPASGN, :^)
# end
-
+
@OP.def_rules(",", ";") do |op, io|
@colonblock_seen = false
@lex_state = EXPR_BEG
Token(op).set_text(op)
end
@@ -854,11 +854,11 @@
@OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do
@lex_state = EXPR_BEG
Token("~").set_text("~@")
end
-
+
@OP.def_rule("(") do
@indent += 1
# if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
# @lex_state = EXPR_BEG
# tk = Token(TkfLPAREN)
@@ -905,19 +905,19 @@
@lex_state = EXPR_BEG
t.set_text("{")
end
@OP.def_rule('\\') do #'
- if getc == "\n"
+ if getc == "\n"
@space_seen = true
@continue = true
Token(TkSPACE).set_text("\\\n")
- else
+ else
ungetc
Token("\\").set_text("\\") #"
- end
- end
+ end
+ end
@OP.def_rule('%') do
|op, io|
if @lex_state == EXPR_BEG || @lex_state == EXPR_MID
identify_quotation('%')
@@ -943,11 +943,11 @@
else
Token("@").set_text("@")
end
end
- # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
+ # @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
# |op, io|
# @indent += 1
# @lex_state = EXPR_FNAME
# # @lex_state = EXPR_END
# # until @rests[0] == "\n" or @rests[0] == ";"
@@ -967,31 +967,31 @@
t = identify_identifier
end
printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug?
t
end
-
+
p @OP if RubyLex.debug?
end
-
+
def identify_gvar
@lex_state = EXPR_END
str = "$"
tk = case ch = getc
when /[~_*$?!@\/\\;,=:<>".]/ #"
str << ch
Token(TkGVAR, str)
-
+
when "-"
str << "-" << getc
Token(TkGVAR, str)
-
+
when "&", "`", "'", "+"
str << ch
Token(TkBACK_REF, str)
-
+
when /[1-9]/
str << ch
while (ch = getc) =~ /[0-9]/
str << ch
end
@@ -999,28 +999,28 @@
Token(TkNTH_REF)
when /\w/
ungetc
ungetc
return identify_identifier
- else
+ else
ungetc
- Token("$")
+ Token("$")
end
tk.set_text(str)
end
-
+
def identify_identifier
token = ""
token.concat getc if peek(0) =~ /[$@]/
token.concat getc if peek(0) == "@"
while (ch = getc) =~ /\w|_/
print ":", ch, ":" if RubyLex.debug?
token.concat ch
end
ungetc
-
+
if ch == "!" or ch == "?"
token.concat getc
end
# fix token
@@ -1031,11 +1031,11 @@
return Token(TkGVAR, token).set_text(token)
when /^\@/
@lex_state = EXPR_END
return Token(TkIVAR, token).set_text(token)
end
-
+
if @lex_state != EXPR_DOT
print token, "\n" if RubyLex.debug?
token_c, *trans = TkReading2Token[token]
if token_c
@@ -1049,12 +1049,12 @@
@lex_state = trans[0]
else
if @lex_state != EXPR_FNAME
if ENINDENT_CLAUSE.include?(token)
@indent += 1
-
- if ACCEPTS_COLON.include?(token)
+
+ if ACCEPTS_COLON.include?(token)
@colonblock_seen = true
else
@colonblock_seen = false
end
elsif DEINDENT_CLAUSE.include?(token)
@@ -1136,11 +1136,11 @@
@ltype = ltback
@lex_state = EXPR_END
Token(Ltype2Token[lt], str).set_text(str.dump)
end
-
+
def identify_quotation(initial_char)
ch = getc
if lt = PERCENT_LTYPE[ch]
initial_char += ch
ch = getc
@@ -1217,11 +1217,11 @@
break
end
end
Token(type).set_text(str)
end
-
+
def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil)
@ltype = ltype
@quoted = quoted
subtype = nil
@@ -1229,13 +1229,13 @@
str << initial_char if initial_char
str << (opener||quoted)
nest = 0
begin
- while ch = getc
+ while ch = getc
str << ch
- if @quoted == ch
+ if @quoted == ch
if nest == 0
break
else
nest -= 1
end
@@ -1292,11 +1292,11 @@
if ch == "\\"
ch = getc
if ch == "\n"
ch = " "
else
- comment << "\\"
+ comment << "\\"
end
else
if ch == "\n"
@ltype = nil
ungetc
@@ -1305,11 +1305,11 @@
end
comment << ch
end
return Token(TkCOMMENT).set_text(comment)
end
-
+
def read_escape
res = ""
case ch = getc
when /[0-7]/
ungetc ch
@@ -1322,10 +1322,10 @@
ungetc
break
end
res << ch
end
-
+
when "x"
res << ch
2.times do
case ch = getc
when /[0-9a-fA-F]/