aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--lib/linguist/blob_helper.rb88
-rw-r--r--lib/linguist/language.rb27
-rw-r--r--lib/linguist/languages.yml17
-rw-r--r--lib/linguist/samples.json214
-rw-r--r--lib/linguist/tokenizer.rb36
-rwxr-xr-xsamples/Groovy/groovy.script! (renamed from samples/Groovy/script.groovy)0
-rwxr-xr-xsamples/JavaScript/js.script! (renamed from samples/JavaScript/script.js)0
-rw-r--r--samples/JavaScript/js2.script!7
-rwxr-xr-xsamples/Nu/nu.script! (renamed from samples/Nu/script.nu)0
-rwxr-xr-xsamples/Perl/perl.script!2
-rwxr-xr-xsamples/Python/python.script! (renamed from samples/Python/script.py)0
-rwxr-xr-xsamples/Racket/rkt.script! (renamed from samples/Racket/script.rkt)0
-rwxr-xr-xsamples/Ruby/macruby.script! (renamed from samples/Ruby/macruby-script)0
-rwxr-xr-xsamples/Ruby/ruby.script! (renamed from samples/Ruby/script.rb)0
-rw-r--r--samples/Ruby/ruby2.script! (renamed from samples/Ruby/script2.rb)2
-rwxr-xr-xsamples/Scala/scala.script! (renamed from samples/Scala/script.scala)0
-rwxr-xr-xsamples/Shell/bash.script!2
-rwxr-xr-xsamples/Shell/sh.script!2
-rwxr-xr-xsamples/Shell/zsh.script!2
-rwxr-xr-xsamples/Text/script.foo2
-rw-r--r--test/test_blob.rb23
-rw-r--r--test/test_classifier.rb7
-rw-r--r--test/test_language.rb10
-rw-r--r--test/test_tokenizer.rb12
24 files changed, 178 insertions, 275 deletions
diff --git a/lib/linguist/blob_helper.rb b/lib/linguist/blob_helper.rb
index ee9f32f..af0150c 100644
--- a/lib/linguist/blob_helper.rb
+++ b/lib/linguist/blob_helper.rb
@@ -129,15 +129,6 @@ module Linguist
['.png', '.jpg', '.jpeg', '.gif'].include?(extname)
end
- # Public: Is the blob likely to have a shebang?
- #
- # Return true or false
- def shebang_extname?
- extname.empty? &&
- mode &&
- (mode.to_i(8) & 05) == 05
- end
-
MEGABYTE = 1024 * 1024
# Public: Is the blob too big to load?
@@ -410,14 +401,23 @@ module Linguist
def guess_language
return if binary_mime_type?
- possible_languages = Language.find_by_filename(name.to_s)
+ name = self.name.to_s
+
+ # A bit of an elegant hack. If the file is exectable but extensionless,
+ # append a "magic" extension so it can be classified with other
+ # languages that have shebang scripts.
+ if extname.empty? && mode && (mode.to_i(8) & 05) == 05
+ name += ".script!"
+ end
+
+ possible_languages = Language.find_by_filename(name)
if possible_languages.length > 1
if result = Classifier.classify(Samples::DATA, data, possible_languages.map(&:name)).first
Language[result[0]]
end
else
- possible_languages.first || shebang_language
+ possible_languages.first
end
end
@@ -428,72 +428,6 @@ module Linguist
language ? language.lexer : Pygments::Lexer.find_by_name('Text only')
end
- # Internal: Extract the script name from the shebang line
- #
- # Requires Blob#data
- #
- # Examples
- #
- # '#!/usr/bin/ruby'
- # # => 'ruby'
- #
- # '#!/usr/bin/env ruby'
- # # => 'ruby'
- #
- # '#!/usr/bash/python2.4'
- # # => 'python'
- #
- # Please add additional test coverage to
- # `test/test_blob.rb#test_shebang_script` if you make any changes.
- #
- # Returns a script name String or nil
- def shebang_script
- # Fail fast if blob isn't viewable?
- return unless viewable?
-
- if lines.any? && (match = lines[0].match(/(.+)\n?/)) && (bang = match[0]) =~ /^#!/
- bang.sub!(/^#! /, '#!')
- tokens = bang.split(' ')
- pieces = tokens.first.split('/')
- if pieces.size > 1
- script = pieces.last
- else
- script = pieces.first.sub('#!', '')
- end
-
- script = script == 'env' ? tokens[1] : script
-
- # python2.4 => python
- if script =~ /((?:\d+\.?)+)/
- script.sub! $1, ''
- end
-
- # Check for multiline shebang hacks that exec themselves
- #
- # #!/bin/sh
- # exec foo "$0" "$@"
- #
- if script == 'sh' &&
- lines[0...5].any? { |l| l.match(/exec (\w+).+\$0.+\$@/) }
- script = $1
- end
-
- script
- end
- end
-
- # Internal: Get Language for shebang script
- #
- # Returns the Language or nil
- def shebang_language
- # Skip file extensions unlikely to have shebangs
- return unless shebang_extname?
-
- if script = shebang_script
- Language[script]
- end
- end
-
# Public: Highlight syntax of blob
#
# options - A Hash of options (defaults to {})
diff --git a/lib/linguist/language.rb b/lib/linguist/language.rb
index 8586fa7..643a523 100644
--- a/lib/linguist/language.rb
+++ b/lib/linguist/language.rb
@@ -11,7 +11,6 @@ module Linguist
# Languages are defined in `lib/linguist/languages.yml`.
class Language
@languages = []
- @overrides = {}
@index = {}
@name_index = {}
@alias_index = {}
@@ -21,13 +20,6 @@ module Linguist
# Valid Languages types
TYPES = [:data, :markup, :programming]
- # Internal: Test if extension maps to multiple Languages.
- #
- # Returns true or false.
- def self.ambiguous?(extension)
- @overrides.include?(extension)
- end
-
# Internal: Create a new Language object
#
# attributes - A hash of attributes
@@ -63,18 +55,6 @@ module Linguist
@extension_index[extension] << language
end
- language.overrides.each do |extension|
- if extension !~ /^\./
- raise ArgumentError, "Extension is missing a '.': #{extension.inspect}"
- end
-
- if l = @overrides[extension]
- raise ArgumentError, "#{extension} is already overridden by #{l.name}"
- end
-
- @overrides[extension] = language
- end
-
language.filenames.each do |filename|
@filename_index[filename] << language
end
@@ -216,7 +196,6 @@ module Linguist
# Set extensions or default to [].
@extensions = attributes[:extensions] || []
- @overrides = attributes[:overrides] || []
@filenames = attributes[:filenames] || []
unless @primary_extension = attributes[:primary_extension]
@@ -324,11 +303,6 @@ module Linguist
# Returns the extension String.
attr_reader :primary_extension
- # Internal: Get overridden extensions.
- #
- # Returns the extensions Array.
- attr_reader :overrides
-
# Public: Get filenames
#
# Examples
@@ -461,7 +435,6 @@ module Linguist
:search_term => options['search_term'],
:extensions => options['extensions'].sort,
:primary_extension => options['primary_extension'],
- :overrides => options['overrides'],
:filenames => options['filenames'],
:popular => popular.include?(name)
)
diff --git a/lib/linguist/languages.yml b/lib/linguist/languages.yml
index a9bdeef..2a1bdef 100644
--- a/lib/linguist/languages.yml
+++ b/lib/linguist/languages.yml
@@ -15,7 +15,6 @@
# the language. Must be unique. Used when a Language is picked
# from a dropdown and we need to automatically choose an
# extension.
-# overrides - An Array of extensions that takes precedence over conflicts
# searchable - Boolean flag to enable searching (defaults to true)
# search_term - Deprecated: Some languages maybe indexed under a
# different alias. Avoid defining new exceptions.
@@ -67,8 +66,6 @@ Apex:
type: programming
lexer: Text only
primary_extension: .cls
- overrides:
- - .cls
AppleScript:
aliases:
@@ -157,8 +154,6 @@ Bro:
C:
type: programming
color: "#555"
- overrides:
- - .h
primary_extension: .c
extensions:
- .w
@@ -533,8 +528,6 @@ Groovy:
Groovy Server Pages:
group: Groovy
lexer: Java Server Page
- overrides:
- - .gsp
aliases:
- gsp
primary_extension: .gsp
@@ -841,8 +834,6 @@ ObjDump:
Objective-C:
type: programming
color: "#438eff"
- overrides:
- - .m
primary_extension: .m
extensions:
- .mm
@@ -915,8 +906,6 @@ Perl:
ace_mode: perl
color: "#0298c3"
primary_extension: .pl
- overrides:
- - .pl
extensions:
- .PL
- .perl
@@ -983,8 +972,6 @@ R:
type: programming
color: "#198ce7"
lexer: S
- overrides:
- - .r
primary_extension: .r
extensions:
- .r
@@ -1208,8 +1195,6 @@ Turing:
color: "#45f715"
lexer: Text only
primary_extension: .t
- overrides:
- - .t
extensions:
- .tu
@@ -1241,8 +1226,6 @@ Verilog:
type: programming
lexer: verilog
color: "#848bf3"
- overrides:
- - .v
primary_extension: .v
VimL:
diff --git a/lib/linguist/samples.json b/lib/linguist/samples.json
index 471917e..8318b7d 100644
--- a/lib/linguist/samples.json
+++ b/lib/linguist/samples.json
@@ -57,7 +57,7 @@
],
"Groovy": [
".gradle",
- ".groovy"
+ ".script!"
],
"Groovy Server Pages": [
".gsp"
@@ -72,7 +72,8 @@
".java"
],
"JavaScript": [
- ".js"
+ ".js",
+ ".script!"
],
"JSON": [
".maxhelp",
@@ -104,7 +105,7 @@
".nim"
],
"Nu": [
- ".nu"
+ ".script!"
],
"Objective-C": [
".h",
@@ -132,7 +133,8 @@
"Perl": [
".pm",
".pl",
- ".t"
+ ".t",
+ ".script!"
],
"PHP": [
".php",
@@ -146,20 +148,22 @@
".pl"
],
"Python": [
- ".py"
+ ".py",
+ ".script!"
],
"R": [
".R"
],
"Racket": [
- ".scrbl",
- ".rkt"
+ ".script!",
+ ".scrbl"
],
"Rebol": [
".r"
],
"Ruby": [
".rb",
+ ".script!",
".rabl",
".rake"
],
@@ -171,7 +175,7 @@
],
"Scala": [
".sbt",
- ".scala"
+ ".script!"
],
"Scheme": [
".sps"
@@ -185,6 +189,7 @@
".scss"
],
"Shell": [
+ ".script!",
".bash",
".sh",
".zsh"
@@ -235,8 +240,8 @@
"PKGBUILD"
]
},
- "tokens_total": 331511,
- "languages_total": 250,
+ "tokens_total": 331500,
+ "languages_total": 255,
"tokens": {
"Apex": {
"/*": 15,
@@ -6682,9 +6687,7 @@
"println": 2,
"it.toString": 1,
"-": 1,
- "#": 1,
- "/usr/bin/env": 1,
- "groovy": 1
+ "SHEBANG#!groovy": 1
},
"Groovy Server Pages": {
"<html>": 4,
@@ -6744,9 +6747,7 @@
"josh@github.com": 1
},
"Ioke": {
- "#": 1,
- "/usr/bin/env": 1,
- "ioke": 1,
+ "SHEBANG#!ioke": 1,
"println": 1
},
"Java": {
@@ -7528,13 +7529,13 @@
"JavaScript": {
"/*": 134,
"*/": 138,
- "function": 2319,
- "(": 18005,
- ")": 18021,
- "{": 6174,
- ";": 8412,
+ "function": 2320,
+ "(": 18012,
+ ")": 18028,
+ "{": 6176,
+ ";": 8417,
"//": 2853,
- "var": 1528,
+ "var": 1529,
"Modal": 2,
"content": 5,
"options": 112,
@@ -7545,7 +7546,7 @@
".proxy": 1,
"this.hide": 1,
"this": 1045,
- "}": 6180,
+ "}": 6182,
"Modal.prototype": 1,
"constructor": 4,
"toggle": 16,
@@ -7643,9 +7644,9 @@
"_super": 4,
"Snake.name": 1,
"Horse.name": 1,
- "console.log": 2,
+ "console.log": 3,
"util": 1,
- "require": 8,
+ "require": 9,
"net": 1,
"stream": 1,
"url": 77,
@@ -7924,7 +7925,7 @@
"OutgoingMessage.prototype._flush": 1,
"this.socket.writable": 2,
"this.socket.write": 1,
- "req": 36,
+ "req": 37,
"OutgoingMessage.call": 2,
"req.method": 5,
"req.httpVersionMajor": 2,
@@ -8057,7 +8058,7 @@
"req.res": 9,
"req.res.readable": 1,
"req.res.emit": 1,
- "res": 16,
+ "res": 17,
"req.res._emitPending": 1,
"res._emitEnd": 1,
"res.emit": 1,
@@ -8318,7 +8319,7 @@
"Ta": 1,
"<[\\w\\W]+>": 4,
"|": 343,
- "#": 23,
+ "#": 22,
"Ua": 1,
".": 67,
"Va": 1,
@@ -9761,7 +9762,7 @@
"results.splice": 2,
"Sizzle.matches": 2,
"Sizzle.matchesSelector": 2,
- "node": 46,
+ "node": 45,
"isXML": 34,
"Expr.order.length": 2,
"Expr.order": 2,
@@ -11635,7 +11636,7 @@
"url=": 1,
"dataTypes=": 1,
"crossDomain=": 2,
- "http": 2,
+ "http": 3,
"80": 2,
"443": 2,
"s=": 14,
@@ -12250,6 +12251,11 @@
"define": 2,
"define.amd": 1,
"define.amd.jQuery": 1,
+ "SHEBANG#!node": 2,
+ "http.createServer": 1,
+ "res.writeHead": 1,
+ "res.end": 1,
+ ".listen": 1,
"JSON": 3,
"Date.prototype.toJSON": 2,
"this.valueOf": 2,
@@ -12807,7 +12813,6 @@
"this.column": 1,
"result.SyntaxError.prototype": 1,
"Error.prototype": 1,
- "/usr/bin/env": 1,
"steelseries": 13,
"n.charAt": 1,
"n.substring": 1,
@@ -16148,9 +16153,7 @@
"echo": 1
},
"Nu": {
- "#": 1,
- "/usr/bin/env": 1,
- "nush": 1,
+ "SHEBANG#!nush": 1,
"(": 1,
"puts": 1,
")": 1
@@ -19406,27 +19409,22 @@
"lcPostBase64Data.": 1
},
"Parrot Assembly": {
- "#": 1,
- "/usr/bin/env": 1,
- "parrot": 1,
+ "SHEBANG#!parrot": 1,
".pcc_sub": 1,
"main": 2,
"say": 1,
"end": 1
},
"Parrot Internal Representation": {
- "#": 1,
- "/usr/bin/env": 1,
- "parrot": 1,
+ "SHEBANG#!parrot": 1,
".sub": 1,
"main": 1,
"say": 1,
".end": 1
},
"Perl": {
- "#": 258,
- "/usr/bin/env": 1,
- "perl": 13,
+ "SHEBANG#!perl": 4,
+ "#": 249,
"use": 70,
"warnings": 15,
";": 1152,
@@ -19434,14 +19432,14 @@
"our": 34,
"VERSION": 15,
"MAIN": 1,
- "{": 1102,
+ "{": 1100,
"if": 267,
"(": 895,
"App": 129,
"Ack": 134,
"ne": 11,
"main": 3,
- ")": 895,
+ ")": 893,
"die": 37,
"}": 1113,
"my": 395,
@@ -19451,7 +19449,7 @@
"last": 15,
"_": 100,
"eq": 31,
- "/": 70,
+ "/": 68,
"-": 843,
"th": 1,
"[": 154,
@@ -19734,7 +19732,7 @@
"unless": 34,
"explicitly": 1,
"<NUM>": 2,
- "print": 29,
+ "print": 30,
"file.": 2,
"Multiple": 1,
"with": 25,
@@ -19867,6 +19865,7 @@
"skipped": 2,
"make": 3,
"binary": 3,
+ "perl": 8,
"ruby": 3,
"php": 2,
"python": 1,
@@ -20390,6 +20389,7 @@
"lc": 5,
"r": 10,
"header": 17,
+ "SHEBANG#!#!": 2,
"lua": 2,
"erl": 2,
"hp": 2,
@@ -20669,6 +20669,7 @@
"number": 1,
"handed": 1,
"argument.": 1,
+ "SHEBANG#!#! perl": 4,
"examples/benchmarks/fib.pl": 1,
"Fibonacci": 2,
"Benchmark": 1,
@@ -21066,9 +21067,7 @@
"formats": 1,
"<+3M>": 1,
"reference.": 1,
- "AUTHOR": 1,
- "/usr/local/bin/perl": 1,
- "/usr/bin/perl": 1
+ "AUTHOR": 1
},
"PHP": {
"<": 9,
@@ -22178,7 +22177,7 @@
"future_builtins": 1,
"zip": 3,
"django.db.models.manager": 1,
- "#": 177,
+ "#": 175,
"django.conf": 1,
"settings": 1,
"django.core.exceptions": 1,
@@ -22707,10 +22706,8 @@
"meth": 5,
"request.method.lower": 1,
"request.method": 1,
- "/usr/bin/env": 2,
- "python2.4": 1,
+ "SHEBANG#!python": 2,
"print": 1,
- "python": 1,
"absolute_import": 1,
"division": 1,
"with_statement": 1,
@@ -22899,13 +22896,29 @@
"}": 1
},
"Racket": {
+ "SHEBANG#!sh": 1,
+ "#": 2,
+ "|": 2,
+ "-": 95,
+ "*": 2,
+ "scheme": 1,
+ "exec": 1,
+ "racket": 1,
+ "um": 1,
+ "(": 7,
+ "require": 2,
+ "racket/file": 1,
+ "racket/path": 1,
+ "racket/list": 1,
+ "racket/string": 1,
+ "for": 2,
+ "syntax": 1,
+ "racket/base": 1,
+ ")": 7,
"#lang": 1,
"scribble/manual": 1,
"@": 3,
- "(": 7,
- "require": 2,
"scribble/bnf": 1,
- ")": 7,
"@title": 1,
"{": 2,
"Scribble": 3,
@@ -22922,11 +22935,9 @@
"collection": 1,
"of": 3,
"tools": 1,
- "for": 2,
"creating": 1,
"prose": 2,
"documents": 1,
- "-": 95,
"papers": 1,
"books": 1,
"library": 1,
@@ -22986,21 +22997,7 @@
";": 1,
"@include": 8,
"section": 9,
- "@index": 1,
- "#": 3,
- "/bin/sh": 1,
- "|": 2,
- "*": 2,
- "scheme": 1,
- "exec": 1,
- "racket": 1,
- "um": 1,
- "racket/file": 1,
- "racket/path": 1,
- "racket/list": 1,
- "racket/string": 1,
- "syntax": 1,
- "racket/base": 1
+ "@index": 1
},
"Rebol": {
"REBOL": 1,
@@ -23025,12 +23022,12 @@
"task": 2,
"default": 2,
"do": 36,
- "puts": 20,
+ "puts": 21,
"end": 248,
"module": 8,
"Foo": 1,
"require": 58,
- "#": 481,
+ "#": 476,
"class": 7,
"Formula": 2,
"include": 3,
@@ -23173,7 +23170,7 @@
"e": 8,
"ARGV.debug": 1,
"%": 11,
- "w": 8,
+ "w": 7,
"config.log": 1,
"CMakeCache.txt": 1,
".select": 1,
@@ -23197,7 +23194,7 @@
"to_s": 2,
"std_cmake_args": 1,
"W": 1,
- "-": 33,
+ "-": 31,
"DCMAKE_INSTALL_PREFIX": 1,
"DCMAKE_BUILD_TYPE": 1,
"None": 1,
@@ -23643,8 +23640,7 @@
"err.to_s": 1,
"DEFAULTS.deep_merge": 1,
".deep_merge": 1,
- "/usr/bin/env": 5,
- "macruby": 1,
+ "SHEBANG#!macruby": 1,
"object": 2,
"@user": 1,
"person": 1,
@@ -23784,11 +23780,8 @@
"keys": 6,
"redis.keys": 1,
"key.sub": 1,
- "rake": 1,
- "ruby": 2,
- "Ilib": 1,
- "test": 6,
- "echo": 1,
+ "SHEBANG#!ruby": 2,
+ "SHEBANG#!rake": 1,
"Sinatra": 2,
"Request": 2,
"<": 2,
@@ -23824,6 +23817,7 @@
".to_sym": 1,
"raise_errors": 1,
"Proc.new": 11,
+ "test": 5,
"dump_errors": 1,
"show_exceptions": 1,
"sessions": 1,
@@ -23974,7 +23968,7 @@
"Delegator.target.helpers": 1,
"self.use": 1,
"Delegator.target.use": 1,
- "python": 1
+ "SHEBANG#!python": 1
},
"Rust": {
"fn": 1,
@@ -24122,10 +24116,10 @@
"Credentials": 2,
"Path.userHome": 1,
"/": 2,
- "#": 2,
- "/bin/sh": 1,
+ "SHEBANG#!sh": 1,
"exec": 1,
"scala": 1,
+ "#": 1,
"object": 1,
"HelloWorld": 1,
"def": 1,
@@ -24385,7 +24379,9 @@
"Shell": {
"export": 6,
"PATH": 5,
- "#": 10,
+ "SHEBANG#!bash": 4,
+ "echo": 14,
+ "#": 5,
"pkgname": 1,
"stud": 4,
"-": 23,
@@ -24439,13 +24435,10 @@
"init.stud": 1,
"mkdir": 1,
"p": 1,
- "/usr/bin/env": 2,
- "bash": 2,
"set": 2,
"e": 1,
"n": 2,
"x": 1,
- "echo": 11,
"unset": 3,
"system": 1,
"exec": 1,
@@ -24479,9 +24472,8 @@
"rvm_is_not_a_shell_function": 2,
"rvm_path/scripts": 1,
"rvm": 1,
- "/bin/bash": 1,
- "/bin/sh": 1,
- "/bin/zsh": 1
+ "SHEBANG#!sh": 2,
+ "SHEBANG#!zsh": 2
},
"Standard ML": {
"signature": 2,
@@ -26469,13 +26461,13 @@
"Emacs Lisp": 3,
"GAS": 133,
"Gosu": 422,
- "Groovy": 71,
+ "Groovy": 69,
"Groovy Server Pages": 91,
"Haml": 4,
"INI": 8,
- "Ioke": 4,
+ "Ioke": 2,
"Java": 7515,
- "JavaScript": 150260,
+ "JavaScript": 150293,
"JSON": 619,
"Julia": 202,
"Kotlin": 155,
@@ -26485,30 +26477,30 @@
"Max": 58,
"Nemerle": 17,
"Nimrod": 2,
- "Nu": 6,
+ "Nu": 4,
"Objective-C": 38749,
"OCaml": 273,
"Opa": 32,
"OpenCL": 88,
"OpenEdge ABL": 3072,
- "Parrot Assembly": 8,
- "Parrot Internal Representation": 7,
- "Perl": 17087,
+ "Parrot Assembly": 6,
+ "Parrot Internal Representation": 5,
+ "Perl": 17075,
"PHP": 23550,
"PowerShell": 14,
"Prolog": 61,
- "Python": 4084,
+ "Python": 4080,
"R": 14,
- "Racket": 270,
+ "Racket": 269,
"Rebol": 11,
- "Ruby": 4339,
+ "Ruby": 4324,
"Rust": 8,
"Sass": 28,
- "Scala": 319,
+ "Scala": 318,
"Scheme": 3484,
"Scilab": 72,
"SCSS": 39,
- "Shell": 315,
+ "Shell": 314,
"Standard ML": 247,
"SuperCollider": 141,
"Tea": 3,
@@ -26545,7 +26537,7 @@
"INI": 1,
"Ioke": 1,
"Java": 5,
- "JavaScript": 19,
+ "JavaScript": 20,
"JSON": 5,
"Julia": 1,
"Kotlin": 1,
@@ -26563,7 +26555,7 @@
"OpenEdge ABL": 5,
"Parrot Assembly": 1,
"Parrot Internal Representation": 1,
- "Perl": 12,
+ "Perl": 13,
"PHP": 6,
"PowerShell": 2,
"Prolog": 1,
@@ -26578,7 +26570,7 @@
"Scheme": 1,
"Scilab": 3,
"SCSS": 1,
- "Shell": 11,
+ "Shell": 14,
"Standard ML": 2,
"SuperCollider": 1,
"Tea": 1,
@@ -26593,5 +26585,5 @@
"XSLT": 1,
"YAML": 1
},
- "md5": "34a5b1ab9d3cf845a0603cef9e9f6509"
+ "md5": "c1a765b2d321e1a0fe84a6f1624b6663"
} \ No newline at end of file
diff --git a/lib/linguist/tokenizer.rb b/lib/linguist/tokenizer.rb
index ed06a9f..5682173 100644
--- a/lib/linguist/tokenizer.rb
+++ b/lib/linguist/tokenizer.rb
@@ -1,3 +1,5 @@
+require 'strscan'
+
module Linguist
# Generic programming language tokenizer.
#
@@ -50,8 +52,13 @@ module Linguist
tokens = []
until s.eos?
+ if token = s.scan(/^#!.+$/)
+ if name = extract_shebang(token)
+ tokens << "SHEBANG#!#{name}"
+ end
+
# Single line comment
- if token = s.scan(START_SINGLE_LINE_COMMENT)
+ elsif token = s.scan(START_SINGLE_LINE_COMMENT)
tokens << token.strip
s.skip_until(/\n|\Z/)
@@ -103,6 +110,33 @@ module Linguist
tokens
end
+ # Internal: Extract normalized shebang command token.
+ #
+ # Examples
+ #
+ # extract_shebang("#!/usr/bin/ruby")
+ # # => "ruby"
+ #
+ # extract_shebang("#!/usr/bin/env node")
+ # # => "node"
+ #
+ # Returns String token or nil it couldn't be parsed.
+ def extract_shebang(data)
+ s = StringScanner.new(data)
+
+ if path = s.scan(/^#!\s*\S+/)
+ script = path.split('/').last
+ if script == 'env'
+ s.scan(/\s+/)
+ script = s.scan(/\S+/)
+ end
+ script = script[/[^\d]+/, 0]
+ return script
+ end
+
+ nil
+ end
+
# Internal: Extract tokens from inside SGML tag.
#
# data - SGML tag String.
diff --git a/samples/Groovy/script.groovy b/samples/Groovy/groovy.script!
index 1ee357a..1ee357a 100755
--- a/samples/Groovy/script.groovy
+++ b/samples/Groovy/groovy.script!
diff --git a/samples/JavaScript/script.js b/samples/JavaScript/js.script!
index a399ce3..a399ce3 100755
--- a/samples/JavaScript/script.js
+++ b/samples/JavaScript/js.script!
diff --git a/samples/JavaScript/js2.script! b/samples/JavaScript/js2.script!
new file mode 100644
index 0000000..d4d290a
--- /dev/null
+++ b/samples/JavaScript/js2.script!
@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+var http = require('http');
+http.createServer(function (req, res) {
+ res.writeHead(200, {'Content-Type': 'text/plain'});
+ res.end('Hello World\n');
+}).listen(1337, '127.0.0.1');
+console.log('Server running at http://127.0.0.1:1337/');
diff --git a/samples/Nu/script.nu b/samples/Nu/nu.script!
index 4e02da8..4e02da8 100755
--- a/samples/Nu/script.nu
+++ b/samples/Nu/nu.script!
diff --git a/samples/Perl/perl.script! b/samples/Perl/perl.script!
new file mode 100755
index 0000000..bb20fbc
--- /dev/null
+++ b/samples/Perl/perl.script!
@@ -0,0 +1,2 @@
+#!/usr/local/bin/perl
+print "Perl\n"
diff --git a/samples/Python/script.py b/samples/Python/python.script!
index bcebcd4..bcebcd4 100755
--- a/samples/Python/script.py
+++ b/samples/Python/python.script!
diff --git a/samples/Racket/script.rkt b/samples/Racket/rkt.script!
index bc5a8ca..bc5a8ca 100755
--- a/samples/Racket/script.rkt
+++ b/samples/Racket/rkt.script!
diff --git a/samples/Ruby/macruby-script b/samples/Ruby/macruby.script!
index 491be11..491be11 100755
--- a/samples/Ruby/macruby-script
+++ b/samples/Ruby/macruby.script!
diff --git a/samples/Ruby/script.rb b/samples/Ruby/ruby.script!
index 01a30f8..01a30f8 100755
--- a/samples/Ruby/script.rb
+++ b/samples/Ruby/ruby.script!
diff --git a/samples/Ruby/script2.rb b/samples/Ruby/ruby2.script!
index 40aeb7c..4b86064 100644
--- a/samples/Ruby/script2.rb
+++ b/samples/Ruby/ruby2.script!
@@ -1,2 +1,2 @@
#! /usr/bin/env ruby -w -Ilib:test
-echo "Ruby"
+puts "Ruby"
diff --git a/samples/Scala/script.scala b/samples/Scala/scala.script!
index b026d24..b026d24 100755
--- a/samples/Scala/script.scala
+++ b/samples/Scala/scala.script!
diff --git a/samples/Shell/bash.script! b/samples/Shell/bash.script!
new file mode 100755
index 0000000..0c2172c
--- /dev/null
+++ b/samples/Shell/bash.script!
@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "bash"
diff --git a/samples/Shell/sh.script! b/samples/Shell/sh.script!
new file mode 100755
index 0000000..e5ed467
--- /dev/null
+++ b/samples/Shell/sh.script!
@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "sh"
diff --git a/samples/Shell/zsh.script! b/samples/Shell/zsh.script!
new file mode 100755
index 0000000..ed274ab
--- /dev/null
+++ b/samples/Shell/zsh.script!
@@ -0,0 +1,2 @@
+#!/bin/zsh
+echo "zsh"
diff --git a/samples/Text/script.foo b/samples/Text/script.foo
deleted file mode 100755
index f94d740..0000000
--- a/samples/Text/script.foo
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/foo
-???
diff --git a/test/test_blob.rb b/test/test_blob.rb
index 079554a..c8f103e 100644
--- a/test/test_blob.rb
+++ b/test/test_blob.rb
@@ -45,10 +45,6 @@ class TestBlob < Test::Unit::TestCase
assert_equal "application/pdf", blob("Binary/foo.pdf").content_type
assert_equal "image/png", blob("Binary/foo.png").content_type
assert_equal "text/plain; charset=iso-8859-2", blob("Text/README").content_type
- assert_equal "text/plain; charset=iso-8859-1", blob("Perl/script.pl").content_type
- assert_equal "text/plain; charset=iso-8859-1", blob("Python/script.py").content_type
- assert_equal "text/plain; charset=iso-8859-1", blob("Ruby/script.rb").content_type
- assert_equal "text/plain; charset=iso-8859-1", blob("Shell/script.sh").content_type
end
def test_disposition
@@ -280,25 +276,6 @@ class TestBlob < Test::Unit::TestCase
assert_equal Lexer['Ruby'], blob("Ruby/foo.rb").lexer
end
- def test_shebang_script
- assert_equal 'sh', script_blob("Shell/script.sh").shebang_script
- assert_equal 'bash', script_blob("Shell/script.bash").shebang_script
- assert_equal 'zsh', script_blob("Shell/script.zsh").shebang_script
- assert_equal 'perl', script_blob("Perl/script.pl").shebang_script
- assert_equal 'ruby', script_blob("Ruby/script.rb").shebang_script
- assert_equal 'ruby', script_blob("Ruby/script2.rb").shebang_script
- assert_equal 'python', script_blob("Python/script.py").shebang_script
- assert_equal 'node', script_blob("JavaScript/script.js").shebang_script
- assert_equal 'groovy', script_blob("Groovy/script.groovy").shebang_script
- assert_equal 'macruby', script_blob("Ruby/macruby-script").shebang_script
- assert_equal 'rake', script_blob("Ruby/script.rake").shebang_script
- assert_equal 'foo', script_blob("Text/script.foo").shebang_script
- assert_equal 'nush', script_blob("Nu/script.nu").shebang_script
- assert_equal 'scala', script_blob("Scala/script.scala").shebang_script
- assert_equal 'racket', script_blob("Racket/script.rkt").shebang_script
- assert_equal nil, script_blob("Ruby/foo.rb").shebang_script
- end
-
def test_colorize
assert_equal <<-HTML, blob("Ruby/foo.rb").colorize
<div class="highlight"><pre><span class="k">module</span> <span class="nn">Foo</span>
diff --git a/test/test_classifier.rb b/test/test_classifier.rb
index a24c5ba..0a47783 100644
--- a/test/test_classifier.rb
+++ b/test/test_classifier.rb
@@ -54,11 +54,8 @@ class TestClassifier < Test::Unit::TestCase
def test_classify_ambiguous_languages
Samples.each do |sample|
- language = Linguist::Language.find_by_name(sample[:language])
- next unless language.overrides.any?
-
- extname = File.extname(sample[:path])
- languages = Language.all.select { |l| l.extensions.include?(extname) }.map(&:name)
+ language = Linguist::Language.find_by_name(sample[:language])
+ languages = Language.find_by_filename(sample[:path]).map(&:name)
next unless languages.length > 1
results = Classifier.classify(Samples::DATA, File.read(sample[:path]), languages)
diff --git a/test/test_language.rb b/test/test_language.rb
index 757b917..47483b0 100644
--- a/test/test_language.rb
+++ b/test/test_language.rb
@@ -8,16 +8,6 @@ class TestLanguage < Test::Unit::TestCase
Lexer = Pygments::Lexer
- def test_ambiguous_extensions
- assert Language.ambiguous?('.cls')
- assert Language.ambiguous?('.h')
- assert Language.ambiguous?('.m')
- assert Language.ambiguous?('.pl')
- assert Language.ambiguous?('.r')
- assert Language.ambiguous?('.t')
- assert Language.ambiguous?('.v')
- end
-
def test_lexer
assert_equal Lexer['ActionScript 3'], Language['ActionScript'].lexer
assert_equal Lexer['Bash'], Language['Gentoo Ebuild'].lexer
diff --git a/test/test_tokenizer.rb b/test/test_tokenizer.rb
index d57726c..4fb49a4 100644
--- a/test/test_tokenizer.rb
+++ b/test/test_tokenizer.rb
@@ -85,6 +85,17 @@ class TestTokenizer < Test::Unit::TestCase
assert_equal %w(#import <Cocoa/Cocoa.h> int main \( int argc char *argv [ ] \) { NSLog \( @ \) ; return ; }), tokenize(:"Objective-C/hello.m")
end
+ def test_shebang
+ assert_equal "SHEBANG#!sh", tokenize(:"Shell/sh.script!")[0]
+ assert_equal "SHEBANG#!bash", tokenize(:"Shell/bash.script!")[0]
+ assert_equal "SHEBANG#!zsh", tokenize(:"Shell/zsh.script!")[0]
+ assert_equal "SHEBANG#!perl", tokenize(:"Perl/perl.script!")[0]
+ assert_equal "SHEBANG#!python", tokenize(:"Python/python.script!")[0]
+ assert_equal "SHEBANG#!ruby", tokenize(:"Ruby/ruby.script!")[0]
+ assert_equal "SHEBANG#!ruby", tokenize(:"Ruby/ruby2.script!")[0]
+ assert_equal "SHEBANG#!node", tokenize(:"JavaScript/js.script!")[0]
+ end
+
def test_javascript_tokens
assert_equal %w( \( function \( \) { console.log \( \) ; } \) .call \( this \) ;), tokenize(:"JavaScript/hello.js")
end
@@ -95,7 +106,6 @@ class TestTokenizer < Test::Unit::TestCase
def test_ruby_tokens
assert_equal %w(module Foo end), tokenize(:"Ruby/foo.rb")
- assert_equal %w(# /usr/bin/env ruby puts), tokenize(:"Ruby/script.rb")
assert_equal %w(task default do puts end), tokenize(:"Ruby/filenames/Rakefile")
end
end
generated by cgit v1.2.3 (git 2.39.1) at 2026-08-07 10:25:50 +0000