class Crass::Scanner

CSS while preserving the original text.
Similar to a StringScanner, but with extra functionality needed to tokenize

def consume

an empty string if the end of the string has been reached.
Consumes the next character and returns it, advancing the pointer, or

def consume
  if @pos < @len
    @pos    += 1
    @current = @scanner.getch
  else
    ''
  end
end

def consume_rest

has already been reached.
the end of the string. Returns an empty string is the end of the string
Consumes the rest of the string and returns it, advancing the pointer to

def consume_rest
  result = @scanner.rest
  # `StringScanner#rest` does not advance the scan pointer, so move it to
  # the end of the input to keep the byte offset in sync with {#pos}. This
  # ensures a subsequent {#marked} extracts the correct substring.
  @scanner.terminate
  @current = result[-1]
  @pos     = @len
  result
end

def eos?

otherwise.
Returns `true` if the end of the string has been reached, `false`

def eos?
  @pos == @len
end

def initialize(input)

Creates a Scanner instance for the given _input_ string or IO instance.

def initialize(input)
  @string  = input.is_a?(IO) ? input.read : input.to_s
  @scanner = StringScanner.new(@string)
  reset
end

def mark

consumed.
Sets the marker to the position of the next character that will be

def mark
  @byte_marker = @scanner.pos
  @marker      = @pos
end

def marked

pointer.
Returns the substring between {#marker} and {#pos}, without altering the

def marked
  # Extract the marked text using byte offsets rather than character
  # offsets. Slicing the original string by character offset is O(n) on
  # multi-byte input (Ruby must translate the character index into a byte
  # index), which makes tokenizing non-ASCII input superlinear. Byte slicing
  # is O(length) regardless of how far into the string we are.
  @string.byteslice(@byte_marker, @scanner.pos - @byte_marker) || ''
end

def peek(length = 1)

_length_ if the end of the string is reached.
doesn't consume them. The number of characters returned may be less than
Returns up to _length_ characters starting at the current position, but

def peek(length = 1)
  # Grab the bytes for up to _length_ characters and then take the first
  # _length_ characters. A UTF-8 character is at most four bytes, so `length
  # * 4` bytes always contains at least _length_ whole characters when that
  # many remain. This avoids the O(n) character-offset slice that
  # `@string[pos, length]` would otherwise perform on multi-byte input.
  @string.byteslice(@scanner.pos, length * 4).slice(0, length) || ''
end

def reconsume

character.
{#current}. The next call to {#consume} will re-consume the current
Moves the pointer back one character without changing the value of

def reconsume
  @scanner.unscan
  @pos -= 1 if @pos > 0
end

def reset

Resets the pointer to the beginning of the string.

def reset
  @scanner.reset
  @byte_marker = 0
  @current     = nil
  @len         = @string.size
  @marker      = 0
  @pos         = 0
end

def scan(pattern)

Otherwise, `nil` will be returned.
matched substring will be returned and the pointer will be advanced.
Tries to match _pattern_ at the current position. If it matches, the

def scan(pattern)
  if match = @scanner.scan(pattern)
    @pos     += match.size
    @current  = match[-1]
  end
  match
end

def scan_until(pattern)

is no match, `nil` is returned and the pointer is not advanced.
to and including the end of the match, and advances the pointer. If there
Scans the string until the _pattern_ is matched. Returns the substring up

def scan_until(pattern)
  if match = @scanner.scan_until(pattern)
    @pos     += match.size
    @current  = match[-1]
  end
  match
end

Modules

Classes