# File lib/coderay/scanners/diff.rb, line 21
 21:     def scan_tokens encoder, options
 22:       
 23:       line_kind = nil
 24:       state = :initial
 25:       deleted_lines = 0
 26:       scanners = Hash.new do |h, lang|
 27:         h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true
 28:       end
 29:       content_scanner = scanners[:plain]
 30:       content_scanner_entry_state = nil
 31:       
 32:       until eos?
 33:         
 34:         if match = scan(/\n/)
 35:           deleted_lines = 0 unless line_kind == :delete
 36:           if line_kind
 37:             encoder.end_line line_kind
 38:             line_kind = nil
 39:           end
 40:           encoder.text_token match, :space
 41:           next
 42:         end
 43:         
 44:         case state
 45:         
 46:         when :initial
 47:           if match = scan(/--- |\+\+\+ |=+|_+/)
 48:             encoder.begin_line line_kind = :head
 49:             encoder.text_token match, :head
 50:             if match = scan(/.*?(?=$|[\t\n\x00]|  \(revision)/)
 51:               encoder.text_token match, :filename
 52:               if options[:highlight_code]
 53:                 file_type = FileType.fetch(match, :text)
 54:                 file_type = :text if file_type == :diff
 55:                 content_scanner = scanners[file_type]
 56:                 content_scanner_entry_state = nil
 57:               end
 58:             end
 59:             next unless match = scan(/.+/)
 60:             encoder.text_token match, :plain
 61:           elsif match = scan(/Index: |Property changes on: /)
 62:             encoder.begin_line line_kind = :head
 63:             encoder.text_token match, :head
 64:             next unless match = scan(/.+/)
 65:             encoder.text_token match, :plain
 66:           elsif match = scan(/Added: /)
 67:             encoder.begin_line line_kind = :head
 68:             encoder.text_token match, :head
 69:             next unless match = scan(/.+/)
 70:             encoder.text_token match, :plain
 71:             state = :added
 72:           elsif match = scan(/\\ .*/)
 73:             encoder.text_token match, :comment
 74:           elsif match = scan(/@@(?>[^@\n]*)@@/)
 75:             content_scanner.state = :initial unless match?(/\n\+/)
 76:             content_scanner_entry_state = nil
 77:             if check(/\n|$/)
 78:               encoder.begin_line line_kind = :change
 79:             else
 80:               encoder.begin_group :change
 81:             end
 82:             encoder.text_token match[0,2], :change
 83:             encoder.text_token match[2...-2], :plain
 84:             encoder.text_token match[-2,2], :change
 85:             encoder.end_group :change unless line_kind
 86:             next unless match = scan(/.+/)
 87:             if options[:highlight_code]
 88:               content_scanner.tokenize match, :tokens => encoder
 89:             else
 90:               encoder.text_token match, :plain
 91:             end
 92:             next
 93:           elsif match = scan(/\+/)
 94:             encoder.begin_line line_kind = :insert
 95:             encoder.text_token match, :insert
 96:             next unless match = scan(/.+/)
 97:             if options[:highlight_code]
 98:               content_scanner.tokenize match, :tokens => encoder
 99:             else
100:               encoder.text_token match, :plain
101:             end
102:             next
103:           elsif match = scan(/-/)
104:             deleted_lines += 1
105:             encoder.begin_line line_kind = :delete
106:             encoder.text_token match, :delete
107:             if options[:inline_diff] && deleted_lines == 1 && check(/(?>.*)\n\+(?>.*)$(?!\n\+)/)
108:               content_scanner_entry_state = content_scanner.state
109:               skip(/(.*)\n\+(.*)$/)
110:               head, deletion, insertion, tail = diff self[1], self[2]
111:               pre, deleted, post = content_scanner.tokenize [head, deletion, tail], :tokens => Tokens.new
112:               encoder.tokens pre
113:               unless deleted.empty?
114:                 encoder.begin_group :eyecatcher
115:                 encoder.tokens deleted
116:                 encoder.end_group :eyecatcher
117:               end
118:               encoder.tokens post
119:               encoder.end_line line_kind
120:               encoder.text_token "\n", :space
121:               encoder.begin_line line_kind = :insert
122:               encoder.text_token '+', :insert
123:               content_scanner.state = content_scanner_entry_state || :initial
124:               pre, inserted, post = content_scanner.tokenize [head, insertion, tail], :tokens => Tokens.new
125:               encoder.tokens pre
126:               unless inserted.empty?
127:                 encoder.begin_group :eyecatcher
128:                 encoder.tokens inserted
129:                 encoder.end_group :eyecatcher
130:               end
131:               encoder.tokens post
132:             elsif match = scan(/.*/)
133:               if options[:highlight_code]
134:                 if deleted_lines == 1
135:                   content_scanner_entry_state = content_scanner.state
136:                 end
137:                 content_scanner.tokenize match, :tokens => encoder unless match.empty?
138:                 if !match?(/\n-/)
139:                   if match?(/\n\+/)
140:                     content_scanner.state = content_scanner_entry_state || :initial
141:                   end
142:                   content_scanner_entry_state = nil
143:                 end
144:               else
145:                 encoder.text_token match, :plain
146:               end
147:             end
148:             next
149:           elsif match = scan(/ .*/)
150:             if options[:highlight_code]
151:               content_scanner.tokenize match, :tokens => encoder
152:             else
153:               encoder.text_token match, :plain
154:             end
155:             next
156:           elsif match = scan(/.+/)
157:             encoder.begin_line line_kind = :comment
158:             encoder.text_token match, :plain
159:           else
160:             raise_inspect 'else case rached'
161:           end
162:         
163:         when :added
164:           if match = scan(/   \+/)
165:             encoder.begin_line line_kind = :insert
166:             encoder.text_token match, :insert
167:             next unless match = scan(/.+/)
168:             encoder.text_token match, :plain
169:           else
170:             state = :initial
171:             next
172:           end
173:         end
174:         
175:       end
176:       
177:       encoder.end_line line_kind if line_kind
178:       
179:       encoder
180:     end