lib/yard/parser/ruby/legacy/ruby_lex.rb in yard-0.9.5 vs lib/yard/parser/ruby/legacy/ruby_lex.rb in yard-0.9.6

- old
+ new

@@ -97,18 +97,18 @@ end class TkOPASGN < TkOp def initialize(line_no, char_no, op) super(line_no, char_no) - op = TkReading2Token[op] unless op.kind_of?(Symbol) + op = TkReading2Token[op] unless op.is_a?(Symbol) @op = op end attr :op end class TkUnknownChar < Token - def initialize(line_no, char_no, id) + def initialize(line_no, char_no, _id) super(line_no, char_no) @name = char_no > 255 ? '?' : char_no.chr end attr :name end @@ -121,15 +121,15 @@ @prev_line_no = line @prev_char_no = char end # @private - def Token(token, value = nil) + def Token(token, value = nil) # rubocop:disable Style/MethodName tk = nil case token when String, Symbol - source = token.kind_of?(String) ? TkReading2Token : TkSymbol2Token + source = token.is_a?(String) ? TkReading2Token : TkSymbol2Token if (tk = source[token]).nil? IRB.fail TkReading2TokenNoKey, token end tk = Token(tk[0], value) else @@ -188,32 +188,28 @@ [:TkDEFINED, TkKW, "defined?", EXPR_END], [:TklBEGIN, TkKW, "BEGIN", EXPR_END], [:TklEND, TkKW, "END", EXPR_END], [:Tk__LINE__, TkKW, "__LINE__", EXPR_END], [:Tk__FILE__, TkKW, "__FILE__", EXPR_END], - [:TkIDENTIFIER, TkId], [:TkFID, TkId], [:TkGVAR, TkId], [:TkIVAR, TkId], [:TkCONSTANT, TkId], - [:TkINTEGER, TkVal], [:TkFLOAT, TkVal], [:TkSYMBOL, TkVal], [:TkLABEL, TkVal], [:TkSTRING, TkVal], [:TkXSTRING, TkVal], [:TkREGEXP, TkVal], [:TkCOMMENT, TkVal], - [:TkDSTRING, TkNode], [:TkDXSTRING, TkNode], [:TkDREGEXP, TkNode], [:TkNTH_REF, TkId], [:TkBACK_REF, TkId], - [:TkUPLUS, TkOp, "+@"], [:TkUMINUS, TkOp, "-@"], [:TkPOW, TkOp, "**"], [:TkCMP, TkOp, "<=>"], [:TkEQ, TkOp, "=="], @@ -231,21 +227,16 @@ [:TkASET, TkOp, "[]="], [:TkLSHFT, TkOp, "<<"], [:TkRSHFT, TkOp, ">>"], [:TkCOLON2, TkOp], [:TkCOLON3, TkOp], - [:OPASGN, TkOp], # +=, -= etc. # + [:OPASGN, TkOp], # +=, -= etc. # [:TkASSOC, TkOp, "=>"], - [:TkQUESTION, TkOp, "?"], #? - [:TkCOLON, TkOp, ":"], #: - -# [:TkfLPAREN], # func( # -# [:TkfLBRACK], # func[ # -# [:TkfLBRACE], # func{ # + [:TkQUESTION, TkOp, "?"], #? + [:TkCOLON, TkOp, ":"], #: [:TkSTAR], # *arg [:TkAMPER], # &arg # -# [:TkSYMBOL, TkId], # :SYMBOL [:TkSYMBEG, TkId], [:TkGT, TkOp, ">"], [:TkLT, TkOp, "<"], [:TkPLUS, TkOp, "+"], [:TkMINUS, TkOp, "-"], @@ -255,47 +246,42 @@ [:TkBITOR, TkOp, "|"], [:TkBITXOR, TkOp, "^"], [:TkBITAND, TkOp, "&"], [:TkBITNOT, TkOp, "~"], [:TkNOTOP, TkOp, "!"], - [:TkBACKQUOTE, TkOp, "`"], - [:TkASSIGN, Token, "="], [:TkDOT, Token, "."], - [:TkLPAREN, Token, "("], #(exp) - [:TkLBRACK, Token, "["], #[arry] - [:TkLBRACE, Token, "{"], #{hash} + [:TkLPAREN, Token, "("], # (exp) + [:TkLBRACK, Token, "["], # [arry] + [:TkLBRACE, Token, "{"], # {hash} [:TkRPAREN, Token, ")"], [:TkRBRACK, Token, "]"], [:TkRBRACE, Token, "}"], [:TkCOMMA, Token, ","], [:TkSEMICOLON, Token, ";"], - -# [:TkRD_COMMENT, TkVal], [:TkSPACE, TkWhitespace], [:TkNL, TkWhitespace], [:TkEND_OF_SCRIPT, TkWhitespace], - [:TkBACKSLASH, TkUnknownChar, "\\"], [:TkAT, TkUnknownChar, "@"], - [:TkDOLLAR, TkUnknownChar, "\$"], #" + [:TkDOLLAR, TkUnknownChar, "\$"] ] # { reading => token_class } # { reading => [token_class, *opt] } TkReading2Token = {} TkSymbol2Token = {} # @private - def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts) - token_n = token_n.id2name unless token_n.kind_of?(String) + def self.def_token(token_n, super_token = Token, reading = nil, *opts) + token_n = token_n.id2name unless token_n.is_a?(String) if RubyToken.const_defined?(token_n) # IRB.fail AlreadyDefinedToken, token_n end - token_c = Class.new super_token + token_c = Class.new super_token RubyToken.const_set token_n, token_c # token_c.inspect if reading if TkReading2Token[reading] @@ -318,17 +304,14 @@ for defs in TokenDefinitions def_token(*defs) end - NEWLINE_TOKEN = TkNL.new(0,0) + NEWLINE_TOKEN = TkNL.new(0, 0) NEWLINE_TOKEN.set_text("\n") - end - - # Lexical analyzer for Ruby source # @private class RubyLex # Read an input stream character by character. We allow for unlimited # ungetting of characters just read. @@ -357,23 +340,22 @@ # here document. Once complete, it needs to read the rest of the # original line, but then skip the here document body. # # @private class BufferedReader - attr_reader :line_num def initialize(content) if /\t/ =~ content tab_width = 2 content = content.split(/\n/).map do |line| - 1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)} && $~ #` + 1 while line.gsub!(/\t+/) { ' ' * (tab_width * $&.length - $`.length % tab_width) } && $~ #` line end .join("\n") end - @content = content - @content << "\n" unless @content[-1,1] == "\n" + @content = String.new(content) + @content << "\n" unless @content[-1, 1] == "\n" @size = @content.size @offset = 0 @hwm = 0 @line_num = 1 @read_back_offset = 0 @@ -406,11 +388,11 @@ def getc_already_read getc end - def ungetc(ch) + def ungetc(_ch) raise "unget past beginning of file" if @offset <= 0 @offset -= 1 if @content[@offset] == ?\n @newline_pending = false end @@ -435,11 +417,11 @@ @content[@offset, str.length] == str end def divert_read_from(reserve) @content[@offset, 0] = reserve - @size = @content.size + @size = @content.size end end # end of nested class BufferedReader @@ -455,11 +437,11 @@ include IRB attr_reader :continue attr_reader :lex_state - def RubyLex.debug? + def self.debug? false end def initialize(content) lex_init @@ -479,10 +461,12 @@ @line = "" @skip_space = false @read_auto_clean_up = false @exception_on_syntax_error = true + + @colonblock_seen = false end attr_accessor :skip_space attr_accessor :read_auto_clean_up attr_accessor :exception_on_syntax_error @@ -509,20 +493,19 @@ def getc_of_rests @reader.getc_already_read end def gets - c = getc or return + (c = getc) || return l = "" begin l.concat c unless c == "\r" break if c == "\n" - end while c = getc + end while c = getc # rubocop:disable Lint/Loop l end - def ungetc(c = nil) @reader.ungetc(c) end def peek_equal?(str) @@ -533,17 +516,17 @@ @reader.peek(i) end def lex catch(:eof) do - until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) && - !@continue or - tk.nil?) + until ((tk = token).is_a?(TkNL) || tk.is_a?(TkEND_OF_SCRIPT)) && + !@continue || + tk.nil? end line = get_read - if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil? + if line == "" && tk.is_a?(TkEND_OF_SCRIPT) || tk.nil? nil else line end end @@ -553,16 +536,16 @@ set_token_position(line_no, char_no) catch(:eof) do begin begin tk = @OP.match(self) - @space_seen = tk.kind_of?(TkSPACE) + @space_seen = tk.is_a?(TkSPACE) rescue SyntaxError abort if @exception_on_syntax_error tk = TkError.new(line_no, char_no) end - end while @skip_space and tk.kind_of?(TkSPACE) + end while @skip_space && tk.is_a?(TkSPACE) if @read_auto_clean_up get_read end # throw :eof unless tk p tk if $DEBUG @@ -571,15 +554,14 @@ end end ENINDENT_CLAUSE = [ "case", "class", "def", "do", "for", "if", - "module", "unless", "until", "while", "begin" #, "when" - ] + "module", "unless", "until", "while", "begin" + ] #, "when" ACCEPTS_COLON = ["if", "for", "unless", "until", "while"] - DEINDENT_CLAUSE = ["end" #, "when" - ] + DEINDENT_CLAUSE = ["end"] #, "when" PERCENT_LTYPE = { "q" => "\'", "Q" => "\"", "x" => "\`", @@ -605,41 +587,38 @@ Ltype2Token.default = TkSTRING DLtype2Token = { "\"" => TkDSTRING, "\`" => TkDXSTRING, - "/" => TkDREGEXP, + "/" => TkDREGEXP } def lex_init() @OP = SLex.new - @OP.def_rules("\0", "\004", "\032") do |chars, io| + @OP.def_rules("\0", "\004", "\032") do |chars, _io| Token(TkEND_OF_SCRIPT).set_text(chars) end - @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io| + @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, _io| @space_seen = TRUE while (ch = getc) =~ /[ \t\f\r\13]/ chars << ch end ungetc Token(TkSPACE).set_text(chars) end - @OP.def_rule("#") do - |op, io| + @OP.def_rule("#") do |_op, _io| identify_comment end - @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do - |op, io| - str = op + @OP.def_rule("=begin", proc { @prev_char_no == 0 && peek(0) =~ /\s/ }) do |op, _io| + str = String.new(op) @ltype = "=" - begin - line = "" + line = String.new begin ch = getc line << ch end until ch == "\n" str << line @@ -674,92 +653,92 @@ @OP.def_rules("*", "**", "!", "!=", "!~", "=", "==", "===", "=~", "<=>", "<", "<=", - ">", ">=", ">>") do |op, io| + ">", ">=", ">>") do |op, _io| @lex_state = EXPR_BEG Token(op).set_text(op) end - @OP.def_rules("<<") do |op, io| + @OP.def_rules("<<") do |op, _io| tk = nil if @lex_state != EXPR_END && @lex_state != EXPR_CLASS && (@lex_state != EXPR_ARG || @space_seen) c = peek(0) - tk = identify_here_document if /[-\w_\"\'\`]/ =~ c + tk = identify_here_document if /[-\w\"\'\`]/ =~ c end if !tk @lex_state = EXPR_BEG tk = Token(op).set_text(op) end tk end - @OP.def_rules("'", '"') do |op, io| + @OP.def_rules("'", '"') do |op, _io| identify_string(op) end - @OP.def_rules("`") do |op, io| + @OP.def_rules("`") do |op, _io| if @lex_state == EXPR_FNAME Token(op).set_text(op) else identify_string(op) end end - @OP.def_rules('?') do |op, io| + @OP.def_rules('?') do |op, _io| if @lex_state == EXPR_END @lex_state = EXPR_BEG Token(TkQUESTION).set_text(op) else ch = getc if @lex_state == EXPR_ARG && ch !~ /\s/ ungetc @lex_state = EXPR_BEG Token(TkQUESTION).set_text(op) else - str = op + str = String.new(op) str << ch - if (ch == '\\') #' + if ch == '\\' #' str << read_escape end @lex_state = EXPR_END Token(TkINTEGER).set_text(str) end end end - @OP.def_rules("&", "&&", "|", "||") do |op, io| + @OP.def_rules("&", "&&", "|", "||") do |op, _io| @lex_state = EXPR_BEG Token(op).set_text(op) end @OP.def_rules("+=", "-=", "*=", "**=", - "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, io| + "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do |op, _io| @lex_state = EXPR_BEG op =~ /^(.*)=$/ Token(TkOPASGN, $1).set_text(op) end - @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io| + @OP.def_rule("+@", proc { @lex_state == EXPR_FNAME }) do |op, _io| Token(TkUPLUS).set_text(op) end - @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io| + @OP.def_rule("-@", proc { @lex_state == EXPR_FNAME }) do |op, _io| Token(TkUMINUS).set_text(op) end - @OP.def_rules("+", "-") do |op, io| + @OP.def_rules("+", "-") do |op, _io| catch(:RET) do if @lex_state == EXPR_ARG - if @space_seen and peek(0) =~ /[0-9]/ + if @space_seen && peek(0) =~ /[0-9]/ throw :RET, identify_number(op) else @lex_state = EXPR_BEG end - elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/ + elsif @lex_state != EXPR_END && peek(0) =~ /[0-9]/ throw :RET, identify_number(op) else @lex_state = EXPR_BEG end Token(op).set_text(op) @@ -776,21 +755,20 @@ @lex_state = EXPR_DOT Token(TkDOT).set_text(".") end end - @OP.def_rules("..", "...") do |op, io| + @OP.def_rules("..", "...") do |op, _io| @lex_state = EXPR_BEG Token(op).set_text(op) end lex_int2 end def lex_int2 - @OP.def_rules("]", "}", ")") do - |op, io| + @OP.def_rules("]", "}", ")") do |op, _io| @lex_state = EXPR_END @indent -= 1 Token(op).set_text(op) end @@ -805,28 +783,28 @@ tk.set_text(":") end @OP.def_rule("::") do # p @lex_state.id2name, @space_seen - if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen + if @lex_state == EXPR_BEG || @lex_state == EXPR_ARG && @space_seen @lex_state = EXPR_BEG tk = Token(TkCOLON3) else @lex_state = EXPR_DOT tk = Token(TkCOLON2) end tk.set_text("::") end - @OP.def_rule("/") do |op, io| + @OP.def_rule("/") do |op, _io| if @lex_state == EXPR_BEG || @lex_state == EXPR_MID identify_string(op) elsif peek(0) == '=' getc @lex_state = EXPR_BEG Token(TkOPASGN, :/).set_text("/=") #") - elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/ + elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/ identify_string(op) else @lex_state = EXPR_BEG Token("/").set_text(op) end @@ -840,43 +818,43 @@ # @OP.def_rules("^=") do # @lex_state = EXPR_BEG # Token(TkOPASGN, :^) # end - @OP.def_rules(",", ";") do |op, io| + @OP.def_rules(",", ";") do |op, _io| @colonblock_seen = false @lex_state = EXPR_BEG Token(op).set_text(op) end @OP.def_rule("~") do @lex_state = EXPR_BEG Token("~").set_text("~") end - @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do + @OP.def_rule("~@", proc { @lex_state = EXPR_FNAME }) do @lex_state = EXPR_BEG Token("~").set_text("~@") end @OP.def_rule("(") do @indent += 1 - # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID - # @lex_state = EXPR_BEG - # tk = Token(TkfLPAREN) - # else + # if @lex_state == EXPR_BEG || @lex_state == EXPR_MID + # @lex_state = EXPR_BEG + # tk = Token(TkfLPAREN) + # else @lex_state = EXPR_BEG tk = Token(TkLPAREN) # end tk.set_text("(") end - @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do + @OP.def_rule("[]", proc { @lex_state == EXPR_FNAME }) do Token("[]").set_text("[]") end - @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do + @OP.def_rule("[]=", proc { @lex_state == EXPR_FNAME }) do Token("[]=").set_text("[]=") end @OP.def_rule("[") do @indent += 1 @@ -905,42 +883,41 @@ t = Token(TkLBRACE) @lex_state = EXPR_BEG t.set_text("{") end - @OP.def_rule('\\') do #' + @OP.def_rule('\\') do #' if getc == "\n" @space_seen = true @continue = true Token(TkSPACE).set_text("\\\n") else ungetc - Token("\\").set_text("\\") #" + Token("\\").set_text("\\") #" end end - @OP.def_rule('%') do - |op, io| + @OP.def_rule('%') do |_op, _io| if @lex_state == EXPR_BEG || @lex_state == EXPR_MID identify_quotation('%') elsif peek(0) == '=' getc Token(TkOPASGN, "%").set_text("%=") - elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/ + elsif @lex_state == EXPR_ARG && @space_seen && peek(0) !~ /\s/ identify_quotation('%') else @lex_state = EXPR_BEG Token("%").set_text("%") end end - @OP.def_rule('$') do #' + @OP.def_rule('$') do #' identify_gvar end @OP.def_rule('@') do - if peek(0) =~ /[@\w_]/ + if peek(0) =~ /[@\w]/ ungetc identify_identifier else Token("@").set_text("@") end @@ -954,19 +931,19 @@ # # until @rests[0] == "\n" or @rests[0] == ";" # # rests.shift # # end # end - @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do + @OP.def_rule("__END__", proc { @prev_char_no == 0 && peek(0) =~ /[\r\n]/ }) do throw :eof end @OP.def_rule("") do |op, io| printf "MATCH: start %s: %s\n", op, io.inspect if RubyLex.debug? if peek(0) =~ /[0-9]/ t = identify_number("") - elsif peek(0) =~ /[\w_]/ + elsif peek(0) =~ /[\w]/ t = identify_identifier end printf "MATCH: end %s: %s\n", op, io.inspect if RubyLex.debug? t end @@ -974,14 +951,14 @@ p @OP if RubyLex.debug? end def identify_gvar @lex_state = EXPR_END - str = "$" + str = String.new("$") tk = case ch = getc - when /[~_*$?!@\/\\;,=:<>".]/ #" + when %r{[~_*$?!@/\\;,=:<>".]} str << ch Token(TkGVAR, str) when "-" str << "-" << getc @@ -1018,11 +995,11 @@ print ":", ch, ":" if RubyLex.debug? token.concat ch end ungetc - if ch == "!" or ch == "?" + if ch == "!" || ch == "?" token.concat getc end # fix token # $stderr.puts "identifier - #{token}, state = #@lex_state" @@ -1040,13 +1017,13 @@ token_c, *trans = TkReading2Token[token] if token_c # reserved word? - if (@lex_state != EXPR_BEG && - @lex_state != EXPR_FNAME && - trans[1]) + if @lex_state != EXPR_BEG && + @lex_state != EXPR_FNAME && + trans[1] # modifiers token_c = TkSymbol2Token[trans[1]] @lex_state = trans[0] else if @lex_state != EXPR_FNAME @@ -1095,11 +1072,11 @@ ch = getc if ch == "-" ch = getc indent = true end - if /['"`]/ =~ ch # ' + if /['"`]/ =~ ch # ' lt = ch quoted = "" while (c = getc) && c != lt quoted.concat c end @@ -1111,23 +1088,23 @@ end ungetc end ltback, @ltype = @ltype, lt - reserve = "" + reserve = String.new while ch = getc reserve << ch - if ch == "\\" #" + if ch == "\\" #" ch = getc reserve << ch elsif ch == "\n" break end end - str = "" + str = String.new while (l = gets) l.chomp! l.strip! if indent break if l == quoted str << l.chomp << "\n" @@ -1160,11 +1137,11 @@ end def identify_number(start) str = start.dup - if start == "+" or start == "-" or start == "" + if start == "+" || start == "-" || start == "" start = getc str << start end @lex_state = EXPR_END @@ -1219,18 +1196,18 @@ end end Token(type).set_text(str) end - def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil) + def identify_string(ltype, quoted = ltype, opener = nil, initial_char = nil) @ltype = ltype @quoted = quoted subtype = nil - str = "" + str = String.new str << initial_char if initial_char - str << (opener||quoted) + str << (opener || quoted) nest = 0 begin while ch = getc str << ch @@ -1240,11 +1217,11 @@ else nest -= 1 end elsif opener == ch nest += 1 - elsif @ltype != "'" && @ltype != "]" and ch == "#" + elsif @ltype != "'" && @ltype != "]" && ch == "#" ch = getc if ch == "{" subtype = true str << ch << skip_inner_expression else @@ -1270,27 +1247,27 @@ @lex_state = EXPR_END end end def skip_inner_expression - res = "" + res = String.new nest = 0 while (ch = getc) res << ch if ch == '}' - break if nest.zero? + break if nest == 0 nest -= 1 elsif ch == '{' nest += 1 end end res end def identify_comment @ltype = "#" - comment = "#" + comment = String.new("#") while ch = getc if ch == "\\" ch = getc if ch == "\n" ch = " " @@ -1304,15 +1281,15 @@ break end end comment << ch end - return Token(TkCOMMENT).set_text(comment) + Token(TkCOMMENT).set_text(comment) end def read_escape - res = "" + res = String.new case ch = getc when /[0-7]/ ungetc ch 3.times do case ch = getc @@ -1322,11 +1299,11 @@ else ungetc break end res << ch - end + end when "x" res << ch 2.times do case ch = getc @@ -1354,10 +1331,10 @@ end end when "C", "c" #, "^" res << ch - if ch == "C" and (ch = getc) != "-" + if ch == "C" && (ch = getc) != "-" ungetc else res << ch if (ch = getc) == "\\" #" res << ch