module Regexp::Lexer

def self.scan(input, syntax = "ruby/#{RUBY_VERSION}", &block)
  syntax = Regexp::Syntax.new(syntax)
  @tokens = []
  @nesting, @set_nesting = 0, 0
  last = nil
  Regexp::Scanner.scan(input) do |type, token, text, ts, te|
    type, token = *syntax.normalize(type, token)
    syntax.check! type, token
    ascend(type, token)
    break_literal(last) if type == :quantifier and
      last and last.type == :literal
    current = Regexp::Token.new(type, token, text, ts, te,
                                @nesting, @set_nesting)
    current = merge_literal(current) if type == :literal and
      last and last.type == :literal
    last.next(current) if last
    current.previous(last) if last
    @tokens << current
    last = current
    descend(type, token)
  end
  if block_given?
    @tokens.each {|t| block.call(t)}
  else
    @tokens
  end
end