# File lib/coderay/scanners/json.rb, line 21
21:     def scan_tokens encoder, options
22:       
23:       state = :initial
24:       stack = []
25:       key_expected = false
26:       
27:       until eos?
28:         
29:         case state
30:         
31:         when :initial
32:           if match = scan(/ \s+ /x)
33:             encoder.text_token match, :space
34:           elsif match = scan(/"/)
35:             state = key_expected ? :key : :string
36:             encoder.begin_group state
37:             encoder.text_token match, :delimiter
38:           elsif match = scan(/ [:,\[{\]}] /x)
39:             encoder.text_token match, :operator
40:             case match
41:             when ':' then key_expected = false
42:             when ',' then key_expected = true if stack.last == :object
43:             when '{' then stack << :object; key_expected = true
44:             when '[' then stack << :array
45:             when '}', ']' then stack.pop  # no error recovery, but works for valid JSON
46:             end
47:           elsif match = scan(/ true | false | null /x)
48:             encoder.text_token match, :value
49:           elsif match = scan(/ -? (?: 0 | [1-9]\d* ) /x)
50:             if scan(/ \.\d+ (?:[eE][-+]?\d+)? | [eE][-+]? \d+ /x)
51:               match << matched
52:               encoder.text_token match, :float
53:             else
54:               encoder.text_token match, :integer
55:             end
56:           else
57:             encoder.text_token getch, :error
58:           end
59:           
60:         when :string, :key
61:           if match = scan(/[^\\"]+/)
62:             encoder.text_token match, :content
63:           elsif match = scan(/"/)
64:             encoder.text_token match, :delimiter
65:             encoder.end_group state
66:             state = :initial
67:           elsif match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
68:             encoder.text_token match, :char
69:           elsif match = scan(/\\./m)
70:             encoder.text_token match, :content
71:           elsif match = scan(/ \\ | $ /x)
72:             encoder.end_group state
73:             encoder.text_token match, :error
74:             state = :initial
75:           else
76:             raise_inspect "else case \" reached; %p not handled." % peek(1), encoder
77:           end
78:           
79:         else
80:           raise_inspect 'Unknown state: %p' % [state], encoder
81:           
82:         end
83:       end
84:       
85:       if [:string, :key].include? state
86:         encoder.end_group state
87:       end
88:       
89:       encoder
90:     end