#--
# irb/ruby-token.rb - ruby tokens
# $Release Version: 0.9.5$
# $Revision: 11708 $
# $Date: 2007-02-12 15:01:19 -0800 (Mon, 12 Feb 2007) $
# by Keiju ISHITSUKA(keiju@ruby-lang.org)
#++
# Definitions of all tokens involved in the lexical analysis.
#
# This class is not documented because it is so deep in the internals.
module RDoc::RubyToken
# :stopdoc:
EXPR_BEG = :EXPR_BEG
EXPR_MID = :EXPR_MID
EXPR_END = :EXPR_END
EXPR_ARG = :EXPR_ARG
EXPR_FNAME = :EXPR_FNAME
EXPR_DOT = :EXPR_DOT
EXPR_CLASS = :EXPR_CLASS
# for ruby 1.4X
if !defined?(Symbol)
Symbol = Integer
end
def set_token_position(line, char)
@prev_line_no = line
@prev_char_no = char
end
class Token
def initialize(seek, line_no, char_no, text = nil)
@seek = seek
@line_no = line_no
@char_no = char_no
@text = text
end
attr :seek
attr :line_no
attr :char_no
attr_accessor :text
def ==(other)
self.class == other.class and
other.line_no == @line_no and
other.char_no == @char_no and
other.text == @text
end
##
# Because we're used in contexts that expect to return a token, we set the
# text string and then return ourselves
def set_text(text)
@text = text
self
end
def inspect # :nodoc:
klass = self.class.name.split('::').last
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @text]
end
end
class TkNode < Token
def initialize(seek, line_no, char_no, node = nil)
super seek, line_no, char_no
@node = node
end
attr :node
def ==(other)
self.class == other.class and
other.line_no == @line_no and
other.char_no == @char_no and
other.node == @node
end
def set_text text
@node = text
self
end
alias text node
def inspect # :nodoc:
klass = self.class.name.split('::').last
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @node]
end
end
class TkId < Token
def initialize(seek, line_no, char_no, name)
super(seek, line_no, char_no)
@name = name
end
attr :name
def ==(other)
self.class == other.class and
other.line_no == @line_no and
other.char_no == @char_no and
other.name == @name
end
def set_text text
@name = text
self
end
alias text name
def inspect # :nodoc:
klass = self.class.name.split('::').last
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
end
end
class TkKW < TkId
end
class TkVal < Token
def initialize(seek, line_no, char_no, value = nil)
super(seek, line_no, char_no)
@value = value
end
attr_accessor :value
def ==(other)
self.class == other.class and
other.line_no == @line_no and
other.char_no == @char_no and
other.value == @value
end
def set_text text
@value = text
self
end
alias text value
def inspect # :nodoc:
klass = self.class.name.split('::').last
"{%s %s, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @value]
end
end
class TkOp < Token
def initialize(seek, line_no, char_no, name = nil)
super seek, line_no, char_no
@name = name
end
attr_accessor :name
def ==(other)
self.class == other.class and
other.line_no == @line_no and
other.char_no == @char_no and
other.name == @name
end
def set_text text
@name = text
self
end
alias text name
def inspect # :nodoc:
klass = self.class.name.split('::').last
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
end
end
class TkOPASGN < TkOp
def initialize(seek, line_no, char_no, op)
super(seek, line_no, char_no)
op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
@op = op
@text = nil
end
attr :op
def ==(other)
self.class == other.class and
other.line_no == @line_no and
other.char_no == @char_no and
other.op == @op
end
def text
@text ||= "#{TkToken2Reading[op]}="
end
def inspect # :nodoc:
klass = self.class.name.split('::').last
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @op]
end
end
class TkUnknownChar < Token
def initialize(seek, line_no, char_no, name)
super(seek, line_no, char_no)
@name = name
end
attr :name
def ==(other)
self.class == other.class and
other.line_no == @line_no and
other.char_no == @char_no and
other.name == @name
end
def set_text text
@name = text
self
end
alias text name
def inspect # :nodoc:
klass = self.class.name.split('::').last
"{%s %d, %d:%d %p}" % [klass, @seek, @line_no, @char_no, @name]
end
end
class TkError < Token
end
def Token(token, value = nil)
value ||= TkToken2Reading[token]
case token
when String
if (tk = TkReading2Token[token]).nil?
IRB.fail TkReading2TokenNoKey, token
end
tk = Token(tk[0], value)
if tk.kind_of?(TkOp) then
tk.name = token
end
when Symbol
if (tk = TkSymbol2Token[token]).nil?
IRB.fail TkSymbol2TokenNoKey, token
end
tk = Token(tk[0], value)
else
if token.instance_method(:initialize).arity == 3 then
tk = token.new(@prev_seek, @prev_line_no, @prev_char_no)
tk.set_text value
else
tk = token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
end
end
tk
end
TokenDefinitions = [
[:TkCLASS, TkKW, "class", :EXPR_CLASS],
[:TkMODULE, TkKW, "module", :EXPR_BEG],
[:TkDEF, TkKW, "def", :EXPR_FNAME],
[:TkUNDEF, TkKW, "undef", :EXPR_FNAME],
[:TkBEGIN, TkKW, "begin", :EXPR_BEG],
[:TkRESCUE, TkKW, "rescue", :EXPR_MID],
[:TkENSURE, TkKW, "ensure", :EXPR_BEG],
[:TkEND, TkKW, "end", :EXPR_END],
[:TkIF, TkKW, "if", :EXPR_BEG, :TkIF_MOD],
[:TkUNLESS, TkKW, "unless", :EXPR_BEG, :TkUNLESS_MOD],
[:TkTHEN, TkKW, "then", :EXPR_BEG],
[:TkELSIF, TkKW, "elsif", :EXPR_BEG],
[:TkELSE, TkKW, "else", :EXPR_BEG],
[:TkCASE, TkKW, "case", :EXPR_BEG],
[:TkWHEN, TkKW, "when", :EXPR_BEG],
[:TkWHILE, TkKW, "while", :EXPR_BEG, :TkWHILE_MOD],
[:TkUNTIL, TkKW, "until", :EXPR_BEG, :TkUNTIL_MOD],
[:TkFOR, TkKW, "for", :EXPR_BEG],
[:TkBREAK, TkKW, "break", :EXPR_MID],
[:TkNEXT, TkKW, "next", :EXPR_END],
[:TkREDO, TkKW, "redo", :EXPR_END],
[:TkRETRY, TkKW, "retry", :EXPR_END],
[:TkIN, TkKW, "in", :EXPR_BEG],
[:TkDO, TkKW, "do", :EXPR_BEG],
[:TkRETURN, TkKW, "return", :EXPR_MID],
[:TkYIELD, TkKW, "yield", :EXPR_END],
[:TkSUPER, TkKW, "super", :EXPR_END],
[:TkSELF, TkKW, "self", :EXPR_END],
[:TkNIL, TkKW, "nil", :EXPR_END],
[:TkTRUE, TkKW, "true", :EXPR_END],
[:TkFALSE, TkKW, "false", :EXPR_END],
[:TkAND, TkKW, "and", :EXPR_BEG],
[:TkOR, TkKW, "or", :EXPR_BEG],
[:TkNOT, TkKW, "not", :EXPR_BEG],
[:TkIF_MOD, TkKW],
[:TkUNLESS_MOD, TkKW],
[:TkWHILE_MOD, TkKW],
[:TkUNTIL_MOD, TkKW],
[:TkALIAS, TkKW, "alias", :EXPR_FNAME],
[:TkDEFINED, TkKW, "defined?", :EXPR_END],
[:TklBEGIN, TkKW, "BEGIN", :EXPR_END],
[:TklEND, TkKW, "END", :EXPR_END],
[:Tk__LINE__, TkKW, "__LINE__", :EXPR_END],
[:Tk__FILE__, TkKW, "__FILE__", :EXPR_END],
[:TkIDENTIFIER, TkId],
[:TkFID, TkId],
[:TkGVAR, TkId],
[:TkCVAR, TkId],
[:TkIVAR, TkId],
[:TkCONSTANT, TkId],
[:TkINTEGER, TkVal],
[:TkFLOAT, TkVal],
[:TkSTRING, TkVal],
[:TkHEREDOC, TkVal],
[:TkXSTRING, TkVal],
[:TkREGEXP, TkVal],
[:TkSYMBOL, TkVal],
[:TkCHAR, TkVal],
[:TkDSTRING, TkNode],
[:TkDXSTRING, TkNode],
[:TkDREGEXP, TkNode],
[:TkNTH_REF, TkNode],
[:TkBACK_REF, TkNode],
[:TkUPLUS, TkOp, "+@"],
[:TkUMINUS, TkOp, "-@"],
[:TkPOW, TkOp, "**"],
[:TkCMP, TkOp, "<=>"],
[:TkEQ, TkOp, "=="],
[:TkEQQ, TkOp, "==="],
[:TkNEQ, TkOp, "!="],
[:TkGEQ, TkOp, ">="],
[:TkLEQ, TkOp, "<="],
[:TkANDOP, TkOp, "&&"],
[:TkOROP, TkOp, "||"],
[:TkMATCH, TkOp, "=~"],
[:TkNMATCH, TkOp, "!~"],
[:TkDOT2, TkOp, ".."],
[:TkDOT3, TkOp, "..."],
[:TkAREF, TkOp, "[]"],
[:TkASET, TkOp, "[]="],
[:TkLSHFT, TkOp, "<<"],
[:TkRSHFT, TkOp, ">>"],
[:TkCOLON2, TkOp, '::'],
[:TkCOLON3, TkOp, '::'],
#[:OPASGN, TkOp], # +=, -= etc. #
[:TkASSOC, TkOp, "=>"],
[:TkQUESTION, TkOp, "?"], #?
[:TkCOLON, TkOp, ":"], #:
[:TkfLPAREN, Token, "("], # func( #
[:TkfLBRACK, Token, "["], # func[ #
[:TkfLBRACE, Token, "{"], # func{ #
[:TkSYMBEG, Token, ":"], # :SYMBOL
[:TkAMPER, TkOp, "&"],
[:TkGT, TkOp, ">"],
[:TkLT, TkOp, "<"],
[:TkPLUS, TkOp, "+"],
[:TkSTAR, TkOp, "*"],
[:TkMINUS, TkOp, "-"],
[:TkMULT, TkOp, "*"],
[:TkDIV, TkOp, "/"],
[:TkMOD, TkOp, "%"],
[:TkBITOR, TkOp, "|"],
[:TkBITXOR, TkOp, "^"],
[:TkBITAND, TkOp, "&"],
[:TkBITNOT, TkOp, "~"],
[:TkNOTOP, TkOp, "!"],
[:TkBACKQUOTE, TkOp, "`"],
[:TkASSIGN, Token, "="],
[:TkDOT, Token, "."],
[:TkLPAREN, Token, "("], #(exp)
[:TkLBRACK, Token, "["], #[arry]
[:TkLBRACE, Token, "{"], #{hash}
[:TkRPAREN, Token, ")"],
[:TkRBRACK, Token, "]"],
[:TkRBRACE, Token, "}"],
[:TkCOMMA, Token, ","],
[:TkSEMICOLON, Token, ";"],
[:TkCOMMENT, TkVal],
[:TkSPACE, Token, " "],
[:TkNL, Token, "\n"],
[:TkEND_OF_SCRIPT],
[:TkBACKSLASH, TkUnknownChar, "\\"],
[:TkAT, TkUnknownChar, "@"],
[:TkDOLLAR, TkUnknownChar, "$"],
]
# {reading => token_class}
# {reading => [token_class, *opt]}
TkReading2Token = {}
TkToken2Reading = {}
TkSymbol2Token = {}
def self.def_token(token_n, super_token = Token, reading = nil, *opts)
token_n = token_n.id2name if token_n.kind_of?(Symbol)
if const_defined?(token_n)
IRB.fail AlreadyDefinedToken, token_n
end
token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
if reading
TkToken2Reading[token_c] = reading
return if TkReading2Token[reading]
if opts.empty?
TkReading2Token[reading] = [token_c]
else
TkReading2Token[reading] = [token_c].concat(opts)
end
end
TkSymbol2Token[token_n.intern] = token_c
end
for defs in TokenDefinitions
def_token(*defs)
end
def_token :TkRD_COMMENT, TkCOMMENT
NEWLINE_TOKEN = TkNL.new 0, 0, 0, "\n"
class TkSYMBOL
def to_sym
@sym ||= text[1..-1].intern
end
end
# :startdoc:
end