From 80e8ee7ce6b5c221feecbf490f83fadd0d7626ba Mon Sep 17 00:00:00 2001 From: Joshua Peek Date: Mon, 23 Jul 2012 13:15:27 -0500 Subject: Rename Sample -> Samples --- lib/linguist/blob_helper.rb | 4 +- lib/linguist/language.rb | 6 +- lib/linguist/sample.rb | 138 -------------------------------------------- lib/linguist/samples.rb | 138 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 143 deletions(-) delete mode 100644 lib/linguist/sample.rb create mode 100644 lib/linguist/samples.rb (limited to 'lib') diff --git a/lib/linguist/blob_helper.rb b/lib/linguist/blob_helper.rb index b932612..39d62c0 100644 --- a/lib/linguist/blob_helper.rb +++ b/lib/linguist/blob_helper.rb @@ -2,7 +2,7 @@ require 'linguist/classifier' require 'linguist/language' require 'linguist/mime' require 'linguist/pathname' -require 'linguist/sample' +require 'linguist/samples' require 'charlock_holmes' require 'escape_utils' @@ -442,7 +442,7 @@ module Linguist if Language.ambiguous?(extname) possible_languages = Language.all.select { |l| l.extensions.include?(extname) }.map(&:name) if possible_languages.any? - if result = Classifier.new(Sample::DATA).classify(data, possible_languages).first + if result = Classifier.new(Samples::DATA).classify(data, possible_languages).first Language[result[0]] end end diff --git a/lib/linguist/language.rb b/lib/linguist/language.rb index 88ccb9f..4317936 100644 --- a/lib/linguist/language.rb +++ b/lib/linguist/language.rb @@ -2,7 +2,7 @@ require 'escape_utils' require 'pygments' require 'yaml' -require 'linguist/sample' +require 'linguist/samples' module Linguist # Language names that are recognizable by GitHub. Defined languages @@ -441,8 +441,8 @@ module Linguist end end - extensions = Sample.extensions - filenames = Sample.filenames + extensions = Samples.extensions + filenames = Samples.filenames popular = YAML.load_file(File.expand_path("../popular.yml", __FILE__)) YAML.load_file(File.expand_path("../languages.yml", __FILE__)).each do |name, options| diff --git a/lib/linguist/sample.rb b/lib/linguist/sample.rb deleted file mode 100644 index 4c3f5e9..0000000 --- a/lib/linguist/sample.rb +++ /dev/null @@ -1,138 +0,0 @@ -require 'set' -require 'yaml' - -module Linguist - # Model for accessing classifier training data. - module Sample - # Samples live in test/ for now, we'll eventually move them out - PATH = File.expand_path("../../../samples", __FILE__) - - YML = File.expand_path('../samples.yml', __FILE__) - if File.exist?(YML) - DATA = YAML.load_file(YML) - else - DATA = nil - end - - # Public: Iterate over each sample. - # - # &block - Yields Sample to block - # - # Returns nothing. - def self.each(&block) - Dir.entries(PATH).each do |category| - next if category == '.' || category == '..' - - # Skip text and binary for now - # Possibly reconsider this later - next if category == 'text' || category == 'binary' - - dirname = File.join(PATH, category) - Dir.entries(dirname).each do |filename| - next if filename == '.' || filename == '..' - - if filename == 'filenames' - Dir.entries(File.join(dirname, filename)).each do |subfilename| - next if subfilename == '.' || subfilename == '..' - - yield({ - :path => File.join(dirname, filename, subfilename), - :language => category, - :filename => subfilename - }) - end - else - yield({ - :path => File.join(dirname, filename), - :language => category, - :extname => File.extname(filename) - }) - end - end - end - - nil - end - - # Get all extensions listed in samples/ - # - # Returns Hash of sample language keys with a Set of extension - # Strings. - def self.extensions - extensions = {} - each do |sample| - # TODO: For now skip empty extnames - next if sample[:extname].nil? || sample[:extname] == "" - extensions[sample[:language]] ||= Set.new - extensions[sample[:language]] << sample[:extname] - end - extensions - end - - # Get all filenames listed in samples/ - # - # Returns Hash of sample language keys with a Set of filename - # Strings. - def self.filenames - filenames = {} - each do |sample| - # TODO: For now skip empty extnames - next if sample[:filename].nil? - filenames[sample[:language]] ||= Set.new - filenames[sample[:language]] << sample[:filename] - end - filenames - end - - # Public: Build Classifier from all samples. - # - # Returns trained Classifier. - def self.classifier - require 'linguist/classifier' - require 'linguist/language' - - classifier = Classifier.new - each { |sample| - language = Language.find_by_alias(sample[:language]) - data = File.read(sample[:path]) - classifier.train(language.name, data) - } - classifier - end - - # Public: Serialize samples data to YAML. - # - # data - Hash - # io - IO object to write to - # - # Returns nothing. - def self.serialize_to_yaml(data, io) - data = "" - escape = lambda { |s| s.inspect.gsub(/\\#/, "\#") } - - data << "languages_total: #{data['languages_total']}\n" - data << "tokens_total: #{data['tokens_total']}\n" - - data << "languages:\n" - data['languages'].sort.each do |language, count| - data << " #{escape.call(language)}: #{count}\n" - end - - data << "language_tokens:\n" - data['language_tokens'].sort.each do |language, count| - data << " #{escape.call(language)}: #{count}\n" - end - - data << "tokens:\n" - data['tokens'].sort.each do |language, tokens| - data << " #{escape.call(language)}:\n" - tokens.sort.each do |token, count| - data << " #{escape.call(token)}: #{count}\n" - end - end - - io.write data - nil - end - end -end diff --git a/lib/linguist/samples.rb b/lib/linguist/samples.rb new file mode 100644 index 0000000..f85f9ae --- /dev/null +++ b/lib/linguist/samples.rb @@ -0,0 +1,138 @@ +require 'set' +require 'yaml' + +module Linguist + # Model for accessing classifier training data. + module Samples + # Samples live in test/ for now, we'll eventually move them out + PATH = File.expand_path("../../../samples", __FILE__) + + YML = File.expand_path('../samples.yml', __FILE__) + if File.exist?(YML) + DATA = YAML.load_file(YML) + else + DATA = nil + end + + # Public: Iterate over each sample. + # + # &block - Yields Sample to block + # + # Returns nothing. + def self.each(&block) + Dir.entries(PATH).each do |category| + next if category == '.' || category == '..' + + # Skip text and binary for now + # Possibly reconsider this later + next if category == 'text' || category == 'binary' + + dirname = File.join(PATH, category) + Dir.entries(dirname).each do |filename| + next if filename == '.' || filename == '..' + + if filename == 'filenames' + Dir.entries(File.join(dirname, filename)).each do |subfilename| + next if subfilename == '.' || subfilename == '..' + + yield({ + :path => File.join(dirname, filename, subfilename), + :language => category, + :filename => subfilename + }) + end + else + yield({ + :path => File.join(dirname, filename), + :language => category, + :extname => File.extname(filename) + }) + end + end + end + + nil + end + + # Get all extensions listed in samples/ + # + # Returns Hash of sample language keys with a Set of extension + # Strings. + def self.extensions + extensions = {} + each do |sample| + # TODO: For now skip empty extnames + next if sample[:extname].nil? || sample[:extname] == "" + extensions[sample[:language]] ||= Set.new + extensions[sample[:language]] << sample[:extname] + end + extensions + end + + # Get all filenames listed in samples/ + # + # Returns Hash of sample language keys with a Set of filename + # Strings. + def self.filenames + filenames = {} + each do |sample| + # TODO: For now skip empty extnames + next if sample[:filename].nil? + filenames[sample[:language]] ||= Set.new + filenames[sample[:language]] << sample[:filename] + end + filenames + end + + # Public: Build Classifier from all samples. + # + # Returns trained Classifier. + def self.classifier + require 'linguist/classifier' + require 'linguist/language' + + classifier = Classifier.new + each { |sample| + language = Language.find_by_alias(sample[:language]) + data = File.read(sample[:path]) + classifier.train(language.name, data) + } + classifier + end + + # Public: Serialize samples data to YAML. + # + # data - Hash + # io - IO object to write to + # + # Returns nothing. + def self.serialize_to_yaml(data, io) + data = "" + escape = lambda { |s| s.inspect.gsub(/\\#/, "\#") } + + data << "languages_total: #{data['languages_total']}\n" + data << "tokens_total: #{data['tokens_total']}\n" + + data << "languages:\n" + data['languages'].sort.each do |language, count| + data << " #{escape.call(language)}: #{count}\n" + end + + data << "language_tokens:\n" + data['language_tokens'].sort.each do |language, count| + data << " #{escape.call(language)}: #{count}\n" + end + + data << "tokens:\n" + data['tokens'].sort.each do |language, tokens| + data << " #{escape.call(language)}:\n" + tokens.sort.each do |token, count| + data << " #{escape.call(token)}: #{count}\n" + end + end + + io.write data + nil + end + end +end -- cgit v1.2.3