class RDoc::RipperStateLex

def self.end?(token)
  (token[:state] & EXPR_END)
end

def self.parse(code)
  lex = self.new(code)
  tokens = []
  begin
    while tk = lex.get_squashed_tk
      tokens.push tk
    end
  rescue StopIteration
  end
  tokens
end

def get_embdoc_tk(tk)
 = tk[:text]
:on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do
ng = string + embdoc_tk[:text]
 = string + embdoc_tk[:text]
e_no => tk[:line_no],
r_no => tk[:char_no],
d => :on_embdoc,
t => string,
te => embdoc_tk[:state]

def get_heredoc_tk(heredoc_name, indent)
 = ''
tk = nil
k = nil
heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
t_tk = tk unless start_tk
prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
ring = string + (' ' * tk[:char_no])
ng = string + tk[:text]
_tk = tk
tk = tk unless start_tk
k = tk unless prev_tk
nshift tk # closing heredoc
c_tk = {
e_no => start_tk[:line_no],
r_no => start_tk[:char_no],
d => :on_heredoc,
t => string,
te => prev_tk[:state]
nshift heredoc_tk

def get_op_tk(tk)
nable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
efinable_operators.include?(tk[:text]) and tk[:state] == EXPR_ARG then
er_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
kind] = :on_ident
tk[:text] =~ /^[-+]$/ then
head = get_squashed_tk
 tk_ahead[:kind]
 :on_int, :on_float, :on_rational, :on_imaginary then
[:text] += tk_ahead[:text]
[:kind] = tk_ahead[:kind]
[:state] = tk_ahead[:state]

uf.unshift tk_ahead

def get_regexp_tk(tk)
 = tk[:text]
= nil
o
r_str_tk = get_squashed_tk
nner_str_tk.nil?
eak
f :on_regexp_end == inner_str_tk[:kind]
ring = string + inner_str_tk[:text]
ate = inner_str_tk[:state]
eak

ring = string + inner_str_tk[:text]
e_no => tk[:line_no],
r_no => tk[:char_no],
d => :on_regexp,
t => string,
te => state

def get_squashed_tk
  if @buf.empty?
    tk = @inner_lex_enumerator.next
  else
    tk = @buf.shift
  end
  case tk[:kind]
  when :on_symbeg then
    tk = get_symbol_tk(tk)
  when :on_tstring_beg then
    tk = get_string_tk(tk)
  when :on_backtick then
    if (tk[:state] & (EXPR_FNAME | EXPR_ENDFN)) != 0
      @inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
      tk[:kind] = :on_ident
      tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
    else
      tk = get_string_tk(tk)
    end
  when :on_regexp_beg then
    tk = get_regexp_tk(tk)
  when :on_embdoc_beg then
    tk = get_embdoc_tk(tk)
  when :on_heredoc_beg then
    @heredoc_queue << retrieve_heredoc_info(tk)
    @inner_lex.lex_state = EXPR_END unless RIPPER_HAS_LEX_STATE
  when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
    unless @heredoc_queue.empty?
      get_heredoc_tk(*@heredoc_queue.shift)
    end
  when :on_words_beg then
    tk = get_words_tk(tk)
  when :on_qwords_beg then
    tk = get_words_tk(tk)
  when :on_symbols_beg then
    tk = get_words_tk(tk)
  when :on_qsymbols_beg then
    tk = get_words_tk(tk)
  when :on_op then
    if '&.' == tk[:text]
      tk[:kind] = :on_period
    else
      tk = get_op_tk(tk)
    end
  end
  tk
end

def get_string_tk(tk)
 = tk[:text]
= nil
 :on_tstring
o
r_str_tk = get_squashed_tk
nner_str_tk.nil?
eak
f :on_tstring_end == inner_str_tk[:kind]
ring = string + inner_str_tk[:text]
ate = inner_str_tk[:state]
eak
f :on_label_end == inner_str_tk[:kind]
ring = string + inner_str_tk[:text]
ate = inner_str_tk[:state]
nd = :on_symbol
eak

ring = string + inner_str_tk[:text]
 :on_embexpr_beg == inner_str_tk[:kind] then
kind = :on_dstring if :on_tstring == kind
d
e_no => tk[:line_no],
r_no => tk[:char_no],
d => kind,
t => string,
te => state

def get_symbol_tk(tk)
bol = true
_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol }
" == tk[:text] or ':"' == tk[:text]
= get_string_tk(tk)
ol_tk[:text] = tk1[:text]
ol_tk[:state] = tk1[:state]
 (tk1 = get_squashed_tk)[:kind]
 :on_ident
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = tk1[:state]
 :on_tstring_content
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end
 :on_tstring_end
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = tk1[:state]
 :on_op
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = tk1[:state]
 :on_ivar
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = tk1[:state]
 :on_cvar
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = tk1[:state]
 :on_gvar
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = tk1[:state]
 :on_const
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = tk1[:state]
 :on_kw
mbol_tk[:text] = ":#{tk1[:text]}"
mbol_tk[:state] = tk1[:state]

_symbol = false
 = tk1
symbol
 symbol_tk

def get_words_tk(tk)
 = ''
token = tk[:text]
quote = tk[:text].rstrip[-1]
o = tk[:line_no]
o = tk[:char_no]
= tk[:state]
ote =
 start_quote
 ?( then ?)
 ?[ then ?]
 ?{ then ?}
 ?< then ?>
 start_quote
ken = nil
o
 get_squashed_tk
k.nil?
d_token = end_quote
eak
f :on_tstring_content == tk[:kind] then
ring += tk[:text]
f :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then
 end_quote == tk[:text].strip then
end_token = tk[:text]
break
se
string += tk[:text]
d

ring += tk[:text]
 "#{start_token}#{string}#{end_token}"
e_no => line_no,
r_no => char_no,
d => :on_dstring,
t => text,
te => state

def heredoc_end?(name, indent, tk)
 = false
_heredoc_end == tk[:kind] then
ame = (indent ? tk[:text].gsub(/^ *(.+)\n?$/, '\1') : tk[:text].gsub(/\n\z/, ''))
ame == tk_name
sult = true

def initialize(code)
  @buf = []
  @heredoc_queue = []
  @inner_lex = InnerStateLex.new(code)
  @inner_lex_enumerator = Enumerator.new do |y|
    @inner_lex.each do |tk|
      y << tk
    end
  end
end

def retrieve_heredoc_info(tk)
 tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2')
 = tk[:text] =~ /\A<<[-~]/
 indent]