diff options
| author | Joshua Peek <josh@joshpeek.com> | 2012-06-20 11:26:14 -0500 |
|---|---|---|
| committer | Joshua Peek <josh@joshpeek.com> | 2012-06-20 11:26:14 -0500 |
| commit | f68e94f181dc6a2a41fa0825817d712c4f20ccd0 (patch) | |
| tree | daf7c2eaa3a2ae954118216389eb1987598e29ba /lib | |
| parent | cb7057216324dc9b25857cfa25e5dd27814926df (diff) | |
Skip number literals
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/linguist/tokenizer.rb | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index 4e64166..9c6779a 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -73,12 +73,15 @@ module Linguist elsif s.scan(/'/) s.skip_until(/[^\\]'/) + # Skip number literals + elsif s.scan(/\d+/) + # SGML style brackets elsif token = s.scan(/<[^>]+>/) extract_sgml_tokens(token).each { |t| tokens << t } # Common programming punctuation - elsif token = s.scan(/;|\{|\}|\(|\)|<<?/) + elsif token = s.scan(/;|\{|\}|\(|\)|<<?|\+/) tokens << token # Regular token |