module GraphQL::Language::Lexer
def self.emit(token_name, ts, te, meta, token_value = nil)
def self.emit(token_name, ts, te, meta, token_value = nil) token_value ||= meta[:data][ts, te - ts].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING) meta[:tokens] << token = GraphQL::Language::Token.new( token_name, token_value, meta[:line], meta[:col], meta[:previous_token], ) meta[:previous_token] = token # Bump the column counter for the next token meta[:col] += te - ts end
def self.emit_string(ts, te, meta, block:)
def self.emit_string(ts, te, meta, block:) quotes_length = block ? 3 : 1 value = meta[:data][ts + quotes_length, te - ts - 2 * quotes_length].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING) || '' line_incr = 0 if block && !value.length.zero? line_incr = value.count("\n") value = GraphQL::Language::BlockString.trim_whitespace(value) end # TODO: replace with `String#match?` when we support only Ruby 2.4+ # (It's faster: https://bugs.ruby-lang.org/issues/8110) if !value.valid_encoding? || value !~ VALID_STRING meta[:tokens] << token = GraphQL::Language::Token.new( :BAD_UNICODE_ESCAPE, value, meta[:line], meta[:col], meta[:previous_token], ) else replace_escaped_characters_in_place(value) if !value.valid_encoding? meta[:tokens] << token = GraphQL::Language::Token.new( :BAD_UNICODE_ESCAPE, value, meta[:line], meta[:col], meta[:previous_token], ) else meta[:tokens] << token = GraphQL::Language::Token.new( :STRING, value, meta[:line], meta[:col], meta[:previous_token], ) end end meta[:previous_token] = token meta[:col] += te - ts meta[:line] += line_incr end
def self.record_comment(ts, te, meta)
def self.record_comment(ts, te, meta) token = GraphQL::Language::Token.new( :COMMENT, meta[:data][ts, te - ts].pack(PACK_DIRECTIVE).force_encoding(UTF_8_ENCODING), meta[:line], meta[:col], meta[:previous_token], ) meta[:previous_token] = token meta[:col] += te - ts end
def self.replace_escaped_characters_in_place(raw_string)
Replace any escaped unicode or whitespace with the _actual_ characters
def self.replace_escaped_characters_in_place(raw_string) raw_string.gsub!(ESCAPES, ESCAPES_REPLACE) raw_string.gsub!(UTF_8, &UTF_8_REPLACE) nil end
def self.run_lexer(query_string)
def self.run_lexer(query_string) data = query_string.unpack("c*") eof = data.length # Since `Lexer` is a module, store all lexer state # in this local variable: meta = { line: 1, col: 1, data: data, tokens: [], previous_token: nil, } p ||= 0 pe ||= data.length begin cs = graphql_lexer_start; ts = 0; te = 0; act = 0; end begin _trans = 0; _have = 0; _cont = 1; _keys = 0; _inds = 0; while ( _cont == 1 ) begin if ( cs == 0 ) _cont = 0; end _have = 0; if ( p == pe ) begin if ( p == eof ) begin if ( _graphql_lexer_eof_trans[cs] > 0 ) begin _trans = _graphql_lexer_eof_trans[cs] - 1; _have = 1; end end if ( _have == 0 ) begin end end end end if ( _have == 0 ) _cont = 0; end end end if ( _cont == 1 ) begin if ( _have == 0 ) begin case _graphql_lexer_from_state_actions[cs] when -2 then begin end when 10 then begin begin begin ts = p; end end end end _keys = (cs<<1) ; _inds = _graphql_lexer_index_offsets[cs] ; if ( ( data[p ].ord) <= 125 && ( data[p ].ord) >= 9 ) begin _ic = _graphql_lexer_char_class[( data[p ].ord) - 9]; if ( _ic <= _graphql_lexer_trans_keys[_keys+1 ]&& _ic >= _graphql_lexer_trans_keys[_keys ] ) _trans = _graphql_lexer_indicies[_inds + ( _ic - _graphql_lexer_trans_keys[_keys ]) ]; else _trans = _graphql_lexer_index_defaults[cs]; end end else begin _trans = _graphql_lexer_index_defaults[cs]; end end end end if ( _cont == 1 ) begin cs = _graphql_lexer_cond_targs[_trans]; case _graphql_lexer_cond_actions[_trans] when -2 then begin end when 14 then begin begin begin te = p+1; end end end when 8 then begin begin begin te = p+1; begin emit_string(ts, te, meta, block: false) end end end end when 28 then begin begin begin te = p+1; begin emit(:RCURLY, ts, te, meta, "}") end end end end when 26 then begin begin begin te = p+1; begin emit(:LCURLY, ts, te, meta, "{") end end end end when 18 then begin begin begin te = p+1; begin emit(:RPAREN, ts, te, meta, ")") end end end end when 17 then begin begin begin te = p+1; begin emit(:LPAREN, ts, te, meta, "(") end end end end when 25 then begin begin begin te = p+1; begin emit(:RBRACKET, ts, te, meta, "]") end end end end when 24 then begin begin begin te = p+1; begin emit(:LBRACKET, ts, te, meta, "[") end end end end when 20 then begin begin begin te = p+1; begin emit(:COLON, ts, te, meta, ":") end end end end when 2 then begin begin begin te = p+1; begin emit_string(ts, te, meta, block: false) end end end end when 34 then begin begin begin te = p+1; begin emit_string(ts, te, meta, block: true) end end end end when 15 then begin begin begin te = p+1; begin emit(:VAR_SIGN, ts, te, meta, "$") end end end end when 22 then begin begin begin te = p+1; begin emit(:DIR_SIGN, ts, te, meta, "@") end end end end when 7 then begin begin begin te = p+1; begin emit(:ELLIPSIS, ts, te, meta, "...") end end end end when 21 then begin begin begin te = p+1; begin emit(:EQUALS, ts, te, meta, "=") end end end end when 13 then begin begin begin te = p+1; begin emit(:BANG, ts, te, meta, "!") end end end end when 27 then begin begin begin te = p+1; begin emit(:PIPE, ts, te, meta, "|") end end end end when 16 then begin begin begin te = p+1; begin emit(:AMP, ts, te, meta, "&") end end end end when 12 then begin begin begin te = p+1; begin meta[:line] += 1 meta[:col] = 1 end end end end when 11 then begin begin begin te = p+1; begin emit(:UNKNOWN_CHAR, ts, te, meta) end end end end when 36 then begin begin begin te = p; p = p - 1; begin emit(:INT, ts, te, meta) end end end end when 37 then begin begin begin te = p; p = p - 1; begin emit(:FLOAT, ts, te, meta) end end end end when 32 then begin begin begin te = p; p = p - 1; begin emit_string(ts, te, meta, block: false) end end end end when 33 then begin begin begin te = p; p = p - 1; begin emit_string(ts, te, meta, block: true) end end end end when 38 then begin begin begin te = p; p = p - 1; begin emit(:IDENTIFIER, ts, te, meta) end end end end when 35 then begin begin begin te = p; p = p - 1; begin record_comment(ts, te, meta) end end end end when 29 then begin begin begin te = p; p = p - 1; begin meta[:col] += te - ts end end end end when 30 then begin begin begin te = p; p = p - 1; begin emit(:UNKNOWN_CHAR, ts, te, meta) end end end end when 5 then begin begin begin p = ((te))-1; begin emit(:INT, ts, te, meta) end end end end when 1 then begin begin begin p = ((te))-1; begin emit(:UNKNOWN_CHAR, ts, te, meta) end end end end when 3 then begin begin begin case act when -2 then begin end when 2 then begin p = ((te))-1; begin emit(:INT, ts, te, meta) end end when 3 then begin p = ((te))-1; begin emit(:FLOAT, ts, te, meta) end end when 4 then begin p = ((te))-1; begin emit(:ON, ts, te, meta, "on") end end when 5 then begin p = ((te))-1; begin emit(:FRAGMENT, ts, te, meta, "fragment") end end when 6 then begin p = ((te))-1; begin emit(:TRUE, ts, te, meta, "true") end end when 7 then begin p = ((te))-1; begin emit(:FALSE, ts, te, meta, "false") end end when 8 then begin p = ((te))-1; begin emit(:NULL, ts, te, meta, "null") end end when 9 then begin p = ((te))-1; begin emit(:QUERY, ts, te, meta, "query") end end when 10 then begin p = ((te))-1; begin emit(:MUTATION, ts, te, meta, "mutation") end end when 11 then begin p = ((te))-1; begin emit(:SUBSCRIPTION, ts, te, meta, "subscription") end end when 12 then begin p = ((te))-1; begin emit(:SCHEMA, ts, te, meta) end end when 13 then begin p = ((te))-1; begin emit(:SCALAR, ts, te, meta) end end when 14 then begin p = ((te))-1; begin emit(:TYPE, ts, te, meta) end end when 15 then begin p = ((te))-1; begin emit(:EXTEND, ts, te, meta) end end when 16 then begin p = ((te))-1; begin emit(:IMPLEMENTS, ts, te, meta) end end when 17 then begin p = ((te))-1; begin emit(:INTERFACE, ts, te, meta) end end when 18 then begin p = ((te))-1; begin emit(:UNION, ts, te, meta) end end when 19 then begin p = ((te))-1; begin emit(:ENUM, ts, te, meta) end end when 20 then begin p = ((te))-1; begin emit(:INPUT, ts, te, meta) end end when 21 then begin p = ((te))-1; begin emit(:DIRECTIVE, ts, te, meta) end end when 29 then begin p = ((te))-1; begin emit_string(ts, te, meta, block: false) end end when 30 then begin p = ((te))-1; begin emit_string(ts, te, meta, block: true) end end when 38 then begin p = ((te))-1; begin emit(:IDENTIFIER, ts, te, meta) end end end end end end when 19 then begin begin begin te = p+1; end end begin begin act = 2; end end end when 6 then begin begin begin te = p+1; end end begin begin act = 3; end end end when 49 then begin begin begin te = p+1; end end begin begin act = 4; end end end when 43 then begin begin begin te = p+1; end end begin begin act = 5; end end end when 54 then begin begin begin te = p+1; end end begin begin act = 6; end end end when 42 then begin begin begin te = p+1; end end begin begin act = 7; end end end when 48 then begin begin begin te = p+1; end end begin begin act = 8; end end end when 50 then begin begin begin te = p+1; end end begin begin act = 9; end end end when 47 then begin begin begin te = p+1; end end begin begin act = 10; end end end when 53 then begin begin begin te = p+1; end end begin begin act = 11; end end end when 52 then begin begin begin te = p+1; end end begin begin act = 12; end end end when 51 then begin begin begin te = p+1; end end begin begin act = 13; end end end when 55 then begin begin begin te = p+1; end end begin begin act = 14; end end end when 41 then begin begin begin te = p+1; end end begin begin act = 15; end end end when 44 then begin begin begin te = p+1; end end begin begin act = 16; end end end when 46 then begin begin begin te = p+1; end end begin begin act = 17; end end end when 56 then begin begin begin te = p+1; end end begin begin act = 18; end end end when 40 then begin begin begin te = p+1; end end begin begin act = 19; end end end when 45 then begin begin begin te = p+1; end end begin begin act = 20; end end end when 39 then begin begin begin te = p+1; end end begin begin act = 21; end end end when 31 then begin begin begin te = p+1; end end begin begin act = 29; end end end when 4 then begin begin begin te = p+1; end end begin begin act = 30; end end end when 23 then begin begin begin te = p+1; end end begin begin act = 38; end end end end case _graphql_lexer_to_state_actions[cs] when -2 then begin end when 9 then begin begin begin ts = 0; end end end end if ( cs == 0 ) _cont = 0; end if ( _cont == 1 ) p += 1; end end end end end end end end meta[:tokens] end
def self.tokenize(query_string)
def self.tokenize(query_string) run_lexer(query_string) end