module Regexp::Lexer
def self.break_literal(token)
called by scan to break a literal run that is longer than one character
def self.break_literal(token) text = token.text if text.scan(/./mu).length > 1 lead = text.sub(/.\z/mu, "") last = text[/.\z/mu] || '' if RUBY_VERSION >= '1.9' lead_length = lead.bytesize last_length = last.bytesize else lead_length = lead.length last_length = last.length end @tokens.pop @tokens << Regexp::Token.new(:literal, :literal, lead, token.ts, (token.te - last_length), @nesting, @set_nesting, @conditional_nesting) @tokens << Regexp::Token.new(:literal, :literal, last, (token.ts + lead_length), token.te, @nesting, @set_nesting, @conditional_nesting) end end