module RDoc::TokenStream

def add_tokens(*tokens)
  tokens.flatten.each { |token| @token_stream << token }
end

def collect_tokens
  @token_stream = []
end

def pop_token
  @token_stream.pop
end

def token_stream
  @token_stream
end

def tokens_to_s
  token_stream.map { |token| token.text }.join ''
end