From 059f661eb6a72526de1f1fccbf6efca5ebbcf2d9 Mon Sep 17 00:00:00 2001 From: Joshua Peek Date: Tue, 24 Jul 2012 11:03:09 -0500 Subject: Rename Max/MSP to Max --- test/test_language.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'test') diff --git a/test/test_language.rb b/test/test_language.rb index 1ca7116..f94acf3 100644 --- a/test/test_language.rb +++ b/test/test_language.rb @@ -165,7 +165,7 @@ class TestLanguage < Test::Unit::TestCase assert_equal 'ruby', Language['Ruby'].search_term assert_equal 'common-lisp', Language['Common Lisp'].search_term assert_equal 'html+erb', Language['HTML+ERB'].search_term - assert_equal 'max/msp', Language['Max/MSP'].search_term + assert_equal 'max/msp', Language['Max'].search_term assert_equal 'puppet', Language['Puppet'].search_term assert_equal 'pure-data', Language['Pure Data'].search_term @@ -310,7 +310,6 @@ class TestLanguage < Test::Unit::TestCase assert_equal 'C%2B%2B', Language['C++'].escaped_name assert_equal 'Objective-C', Language['Objective-C'].escaped_name assert_equal 'Common%20Lisp', Language['Common Lisp'].escaped_name - assert_equal 'Max%2FMSP', Language['Max/MSP'].escaped_name end def test_error_without_name -- cgit v1.2.3 From f5705eaf3872fa5872dd7500483f8c01e53d9768 Mon Sep 17 00:00:00 2001 From: Joshua Peek Date: Tue, 24 Jul 2012 11:23:06 -0500 Subject: Parse float tokens --- lib/linguist/tokenizer.rb | 2 +- test/test_tokenizer.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'test') diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index 520ed5c..e591214 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -69,7 +69,7 @@ module Linguist s.skip_until(/[^\\]'/) # Skip number literals - elsif s.scan(/(0x)?\d+/) + elsif s.scan(/(0x)?\d(\d|\.)*/) # SGML style brackets elsif token = s.scan(/<[^\s<>][^<>]*>/) diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 644a2c3..55271ba 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -26,6 +26,7 @@ class TestTokenizer < Test::Unit::TestCase assert_equal %w(+), tokenize('1 + 1') assert_equal %w(add \( \)), tokenize('add(123, 456)') assert_equal %w(|), tokenize('0x01 | 0x10') + assert_equal %w(*), tokenize('500.42 * 1.0') end def test_skip_comments -- cgit v1.2.3 From 53300ca5812a3f19b4ea1c670e286d68b900c7cf Mon Sep 17 00:00:00 2001 From: Joshua Peek Date: Tue, 24 Jul 2012 11:28:27 -0500 Subject: Add brackets to tokens --- lib/linguist/tokenizer.rb | 2 +- test/test_tokenizer.rb | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index e591214..bf05dcd 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -76,7 +76,7 @@ module Linguist extract_sgml_tokens(token).each { |t| tokens << t } # Common programming punctuation - elsif token = s.scan(/;|\{|\}|\(|\)/) + elsif token = s.scan(/;|\{|\}|\(|\)|\[|\]/) tokens << token # Regular token diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 55271ba..81654a0 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -78,13 +78,17 @@ class TestTokenizer < Test::Unit::TestCase def test_objective_c_tokens assert_equal %w(#import @interface Foo NSObject { } @end), tokenize(:"Objective-C/Foo.h") assert_equal %w(#import @implementation Foo @end), tokenize(:"Objective-C/Foo.m") - assert_equal %w(#import int main \( int argc char *argv \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m") + assert_equal %w(#import int main \( int argc char *argv [ ] \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m") end def test_javascript_tokens assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize(:"JavaScript/hello.js") end + def test_json_tokens + assert_equal %w( { [ ] { } } ), tokenize(:"JSON/product.json") + end + def test_ruby_tokens assert_equal %w(module Foo end), tokenize(:"Ruby/foo.rb") assert_equal %w(# /usr/bin/env ruby puts), tokenize(:"Ruby/script.rb") -- cgit v1.2.3 From e5d302459fd718ca26a1d9477a1c5220092a8fd1 Mon Sep 17 00:00:00 2001 From: Joshua Peek Date: Tue, 24 Jul 2012 11:49:29 -0500 Subject: Fix tokenzing empty strings --- lib/linguist/tokenizer.rb | 12 ++++++++++-- test/test_tokenizer.rb | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb index bf05dcd..ed06a9f 100644 --- a/lib/linguist/tokenizer.rb +++ b/lib/linguist/tokenizer.rb @@ -64,9 +64,17 @@ module Linguist # Skip single or double quoted strings elsif s.scan(/"/) - s.skip_until(/[^\\]"/) + if s.peek(1) == "\"" + s.getch + else + s.skip_until(/[^\\]"/) + end elsif s.scan(/'/) - s.skip_until(/[^\\]'/) + if s.peek(1) == "'" + s.getch + else + s.skip_until(/[^\\]'/) + end # Skip number literals elsif s.scan(/(0x)?\d(\d|\.)*/) diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb index 81654a0..d57726c 100644 --- a/test/test_tokenizer.rb +++ b/test/test_tokenizer.rb @@ -20,6 +20,10 @@ class TestTokenizer < Test::Unit::TestCase assert_equal %w(print), tokenize("print 'Josh'") assert_equal %w(print), tokenize('print "Hello \"Josh\""') assert_equal %w(print), tokenize("print 'Hello \\'Josh\\''") + assert_equal %w(print), tokenize("print \"Hello\", \"Josh\"") + assert_equal %w(print), tokenize("print 'Hello', 'Josh'") + assert_equal %w(print), tokenize("print \"Hello\", \"\", \"Josh\"") + assert_equal %w(print), tokenize("print 'Hello', '', 'Josh'") end def test_skip_number_literals -- cgit v1.2.3