# File lib/coderay/scanners/diff.rb, line 19
 19:     def scan_tokens encoder, options
 20:       
 21:       line_kind = nil
 22:       state = :initial
 23:       deleted_lines = 0
 24:       scanners = Hash.new do |h, lang|
 25:         h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true
 26:       end
 27:       content_scanner = scanners[:plain]
 28:       content_scanner_entry_state = nil
 29:       
 30:       until eos?
 31:         
 32:         if match = scan(/\n/)
 33:           deleted_lines = 0 unless line_kind == :delete
 34:           if line_kind
 35:             encoder.end_line line_kind
 36:             line_kind = nil
 37:           end
 38:           encoder.text_token match, :space
 39:           next
 40:         end
 41:         
 42:         case state
 43:         
 44:         when :initial
 45:           if match = scan(/--- |\+\+\+ |=+|_+/)
 46:             encoder.begin_line line_kind = :head
 47:             encoder.text_token match, :head
 48:             if match = scan(/.*?(?=$|[\t\n\x00]|  \(revision)/)
 49:               encoder.text_token match, :filename
 50:               if options[:highlight_code] && match != '/dev/null'
 51:                 file_type = CodeRay::FileType.fetch(match, :text)
 52:                 file_type = :text if file_type == :diff
 53:                 content_scanner = scanners[file_type]
 54:                 content_scanner_entry_state = nil
 55:               end
 56:             end
 57:             next unless match = scan(/.+/)
 58:             encoder.text_token match, :plain
 59:           elsif match = scan(/Index: |Property changes on: /)
 60:             encoder.begin_line line_kind = :head
 61:             encoder.text_token match, :head
 62:             next unless match = scan(/.+/)
 63:             encoder.text_token match, :plain
 64:           elsif match = scan(/Added: /)
 65:             encoder.begin_line line_kind = :head
 66:             encoder.text_token match, :head
 67:             next unless match = scan(/.+/)
 68:             encoder.text_token match, :plain
 69:             state = :added
 70:           elsif match = scan(/\\ .*/)
 71:             encoder.text_token match, :comment
 72:           elsif match = scan(/@@(?>[^@\n]*)@@/)
 73:             content_scanner.state = :initial unless match?(/\n\+/)
 74:             content_scanner_entry_state = nil
 75:             if check(/\n|$/)
 76:               encoder.begin_line line_kind = :change
 77:             else
 78:               encoder.begin_group :change
 79:             end
 80:             encoder.text_token match[0,2], :change
 81:             encoder.text_token match[2...-2], :plain
 82:             encoder.text_token match[-2,2], :change
 83:             encoder.end_group :change unless line_kind
 84:             next unless match = scan(/.+/)
 85:             if options[:highlight_code]
 86:               content_scanner.tokenize match, :tokens => encoder
 87:             else
 88:               encoder.text_token match, :plain
 89:             end
 90:             next
 91:           elsif match = scan(/\+/)
 92:             encoder.begin_line line_kind = :insert
 93:             encoder.text_token match, :insert
 94:             next unless match = scan(/.+/)
 95:             if options[:highlight_code]
 96:               content_scanner.tokenize match, :tokens => encoder
 97:             else
 98:               encoder.text_token match, :plain
 99:             end
100:             next
101:           elsif match = scan(/-/)
102:             deleted_lines += 1
103:             encoder.begin_line line_kind = :delete
104:             encoder.text_token match, :delete
105:             if options[:inline_diff] && deleted_lines == 1 && check(/(?>.*)\n\+(?>.*)$(?!\n\+)/)
106:               content_scanner_entry_state = content_scanner.state
107:               skip(/(.*)\n\+(.*)$/)
108:               head, deletion, insertion, tail = diff self[1], self[2]
109:               pre, deleted, post = content_scanner.tokenize [head, deletion, tail], :tokens => Tokens.new
110:               encoder.tokens pre
111:               unless deleted.empty?
112:                 encoder.begin_group :eyecatcher
113:                 encoder.tokens deleted
114:                 encoder.end_group :eyecatcher
115:               end
116:               encoder.tokens post
117:               encoder.end_line line_kind
118:               encoder.text_token "\n", :space
119:               encoder.begin_line line_kind = :insert
120:               encoder.text_token '+', :insert
121:               content_scanner.state = content_scanner_entry_state || :initial
122:               pre, inserted, post = content_scanner.tokenize [head, insertion, tail], :tokens => Tokens.new
123:               encoder.tokens pre
124:               unless inserted.empty?
125:                 encoder.begin_group :eyecatcher
126:                 encoder.tokens inserted
127:                 encoder.end_group :eyecatcher
128:               end
129:               encoder.tokens post
130:             elsif match = scan(/.*/)
131:               if options[:highlight_code]
132:                 if deleted_lines == 1
133:                   content_scanner_entry_state = content_scanner.state
134:                 end
135:                 content_scanner.tokenize match, :tokens => encoder unless match.empty?
136:                 if !match?(/\n-/)
137:                   if match?(/\n\+/)
138:                     content_scanner.state = content_scanner_entry_state || :initial
139:                   end
140:                   content_scanner_entry_state = nil
141:                 end
142:               else
143:                 encoder.text_token match, :plain
144:               end
145:             end
146:             next
147:           elsif match = scan(/ .*/)
148:             if options[:highlight_code]
149:               content_scanner.tokenize match, :tokens => encoder
150:             else
151:               encoder.text_token match, :plain
152:             end
153:             next
154:           elsif match = scan(/.+/)
155:             encoder.begin_line line_kind = :comment
156:             encoder.text_token match, :plain
157:           else
158:             raise_inspect 'else case rached'
159:           end
160:         
161:         when :added
162:           if match = scan(/   \+/)
163:             encoder.begin_line line_kind = :insert
164:             encoder.text_token match, :insert
165:             next unless match = scan(/.+/)
166:             encoder.text_token match, :plain
167:           else
168:             state = :initial
169:             next
170:           end
171:         end
172:         
173:       end
174:       
175:       encoder.end_line line_kind if line_kind
176:       
177:       encoder
178:     end