diff options
| -rw-r--r-- | lib/linguist/tokenizer.rb | 2 | ||||
| -rw-r--r-- | test/test_tokenizer.rb | 1 |
2 files changed, 2 insertions, 1 deletions
diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index 520ed5c..e591214 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -69,7 +69,7 @@ module Linguist s.skip_until(/[^\\]'/) # Skip number literals - elsif s.scan(/(0x)?\d+/) + elsif s.scan(/(0x)?\d(\d|\.)*/) # SGML style brackets elsif token = s.scan(/<[^\s<>][^<>]*>/) diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 644a2c3..55271ba 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -26,6 +26,7 @@ class TestTokenizer < Test::Unit::TestCase assert_equal %w(+), tokenize('1 + 1') assert_equal %w(add \( \)), tokenize('add(123, 456)') assert_equal %w(|), tokenize('0x01 | 0x10') + assert_equal %w(*), tokenize('500.42 * 1.0') end def test_skip_comments |