Fix tokenzing empty strings

1 files changed, 10 insertions, 2 deletions

diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb
index bf05dcd..ed06a9f 100644
--- a/lib/linguist/tokenizer.rb
+++ b/lib/linguist/tokenizer.rb
@@ -64,9 +64,17 @@ module Linguist
 
         # Skip single or double quoted strings
         elsif s.scan(/"/)
-          s.skip_until(/[^\\]"/)
+          if s.peek(1) == "\""
+            s.getch
+          else
+            s.skip_until(/[^\\]"/)
+          end
         elsif s.scan(/'/)
-          s.skip_until(/[^\\]'/)
+          if s.peek(1) == "'"
+            s.getch
+          else
+            s.skip_until(/[^\\]'/)
+          end
 
         # Skip number literals
         elsif s.scan(/(0x)?\d(\d|\.)*/)