diff options
| author | Joshua Peek <josh@joshpeek.com> | 2012-07-24 11:23:06 -0500 |
|---|---|---|
| committer | Joshua Peek <josh@joshpeek.com> | 2012-07-24 11:23:06 -0500 |
| commit | f5705eaf3872fa5872dd7500483f8c01e53d9768 (patch) | |
| tree | 84e23a1f224c966b71052a92b88de085085953f5 | |
| parent | e2a91bba3e114227e80beef5b62b53be3dab183c (diff) | |
Parse float tokens
| -rw-r--r-- | lib/linguist/tokenizer.rb | 2 | ||||
| -rw-r--r-- | test/test_tokenizer.rb | 1 |
2 files changed, 2 insertions, 1 deletions
diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index 520ed5c..e591214 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -69,7 +69,7 @@ module Linguist s.skip_until(/[^\\]'/) # Skip number literals - elsif s.scan(/(0x)?\d+/) + elsif s.scan(/(0x)?\d(\d|\.)*/) # SGML style brackets elsif token = s.scan(/<[^\s<>][^<>]*>/) diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 644a2c3..55271ba 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -26,6 +26,7 @@ class TestTokenizer < Test::Unit::TestCase assert_equal %w(+), tokenize('1 + 1') assert_equal %w(add \( \)), tokenize('add(123, 456)') assert_equal %w(|), tokenize('0x01 | 0x10') + assert_equal %w(*), tokenize('500.42 * 1.0') end def test_skip_comments |