From e5107da4f9491fa7516e6c2a903c075a388abfe0 Mon Sep 17 00:00:00 2001 From: "Glenn Y. Rolland" Date: Mon, 8 Aug 2016 10:40:43 +0200 Subject: [PATCH] Add parameters for output & match string. --- bin/ewoga-fetch | 9 ++- bin/ewoga-test | 178 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 145 insertions(+), 42 deletions(-) diff --git a/bin/ewoga-fetch b/bin/ewoga-fetch index 70ff09c..d41589a 100755 --- a/bin/ewoga-fetch +++ b/bin/ewoga-fetch @@ -68,7 +68,7 @@ module Ewoga m = Mail.read_from_string message.attr[@saved_key] return if m.from.nil? return if m.to.nil? - return unless m.subject =~ /RUBY\s+NEXTFORMATION/ + return unless m.subject =~ Regexp.new(@config[:match]) begin puts "\n### #{m.subject}" @@ -76,7 +76,7 @@ module Ewoga print "#{m.from.first} --> #{m.to.join(',')} " attach = m.attachments.first - fn = "extract/%s/projet.tar" % [m.from.first, attach.filename] + fn = "%s/%s/projet-%s.tar" % [@config[:output], m.from.first, m.date.to_date.to_s] puts "- #{fn}" begin FileUtils.mkdir_p File.dirname(fn) @@ -122,14 +122,17 @@ module Ewoga default_task :crawl + option :match, required: true + option :output, required: true desc 'crawl', 'Crawls email to save mails' def crawl #saved_info = [] parse_configuration + @config['match'] = options[:match] + @config['output'] = options[:match] ## Run application app = CrawlerApp.new @config - app.connect! app.examine_all app.disconnect! diff --git a/bin/ewoga-test b/bin/ewoga-test index 8c6b223..ce36c37 100755 --- a/bin/ewoga-test +++ b/bin/ewoga-test @@ -21,15 +21,18 @@ class Project @name = File.basename path @score = 0 @score_max = 0 + @errors = 0 end class ExtractionError < Exception ; end def extract Dir.chdir(@path) - print indent + "Extracting project data ... " - tarfile = @path + '/projet.tar' + tarfile = Dir.glob('projet-*.tar').sort.last + # tarfile = @path + '/projet.tar' + print indent + "Extracting project data from #{tarfile}... " + system "tar xavf #{tarfile} > extractlog" raise ExtractionError unless $?.success? puts "success".green @@ -41,29 +44,27 @@ class Project end FileUtils.rm 'extractlog' puts "" + FileUtils.rm_f ENV['HOME'] + '/.taskman.yml' + FileUtils.rm_f ENV['HOME'] + '/.taskman' end - def patch + def test_patch + res = { log: [], errors: [], score: 0, score_max: 0 } + Dir.chdir(@path) - print indent + "Patching project data ... " - $INDENT += 2 - if File.exist? 'patch.d' then - log = [] - Dir.glob('patch.d/*.patch').sort.each do |patchfile| - patchname = File.basename(patchfile).gsub(/\.patch$/,'') - IO.popen("patch -p0 < #{patchfile}") do |fh| - log.concat fh.readlines.map(&:strip).map { |line| - patchname + ': ' + line - } - end - end - puts "success".green - puts log.map {|line| indent + line } - else - puts 'skipping' + FileUtils.rm_f '.pc/applied-patches' + if File.exist? 'patches' then + Dir.chdir(@path + '/taskman') + res[:log].concat %x{quilt push -a -v 2>&1 }.split(/\n/).map(&:strip) + + # We generate an error for each patch + res[:score_max] += File.readlines('../patches/series').length + res[:log] << "" + res[:log] << "Applied #{res[:score_max]} patch(es)" end - $INDENT -= 2 - puts "" + + + res end def test_structure @@ -187,6 +188,7 @@ class Project no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? res[:score] += 1 if not $?.success? and not no_error res[:score_max] += 1 + res[:errors] << "Error not detected in output" if no_error or $?.success? res end @@ -196,13 +198,26 @@ class Project command = bundle_prefix + './bin/taskman list 2>&1' res[:log] << "Running command: #{command}" + output = [] IO.popen(command) do |fh| - res[:log].concat fh.readlines.map(&:strip) + output.concat fh.readlines.map(&:strip) end - no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? - res[:score] += 1 if $?.success? and no_error + res[:log].concat output + res[:score] += 1 if $?.success? res[:score_max] += 1 + # detect errors in output + no_error = output.select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if no_error + res[:score_max] += 1 + res[:errors] << "Detected error in output" unless no_error + + # detect bad output formating + bad_formatted = output.select{|line| not line =~ /^\d+:/ } + res[:score] += 1 if bad_formatted.empty? + res[:score_max] += 1 + res[:errors] << "Lines are badly formatted" unless bad_formatted.empty? + res end @@ -215,7 +230,8 @@ class Project before.concat fh.readlines.map(&:strip) end - command = bundle_prefix + './bin/taskman add "Tester taskman 1" 2>&1' + testid = Random.rand(10000) + command = bundle_prefix + "./bin/taskman add \"Tester taskman #{testid}\" 2>&1" res[:log] << "Running command: #{command}" IO.popen(command) do |fh| res[:log].concat fh.readlines.map(&:strip) @@ -230,18 +246,94 @@ class Project after.concat fh.readlines.map(&:strip) end + res[:log] << "List difference :" + diff = (after-before).map{|x| '+' + x} + res[:log].concat diff + res[:score] += 1 if diff.length == 1 + res[:score_max] += 1 + + res[:errors] << "Save was not implemented" if diff.length < 1 + res[:errors] << "Listing called everywhere" if diff.length > 1 + res end def test_taskman_command_del res = { log: [], errors: [], score: 0, score_max: 0 } + command = bundle_prefix + './bin/taskman list 2>&1' + before = [] + IO.popen(command) do |fh| + before.concat fh.readlines.map(&:strip) + end + + testid = ((before.first || "").match(/^(\d+):/) || [])[1] + command = bundle_prefix + "./bin/taskman del #{testid} 2>&1" + res[:log] << "Running command: #{command}" + IO.popen(command) do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if $?.success? and no_error + res[:score_max] += 1 + + command = bundle_prefix + './bin/taskman list 2>&1' + after = [] + IO.popen(command) do |fh| + after.concat fh.readlines.map(&:strip) + end + + res[:log] << "List difference :" + diff = (before - after).map{|x| '-' + x} + res[:log].concat diff + res[:score] += 1 if diff.length == 1 + res[:score_max] += 1 + + res[:errors] << "Save was not implemented" if diff.length < 1 + res[:errors] << "Listing called everywhere" if diff.length > 1 + res end def test_taskman_command_mod res = { log: [], errors: [], score: 0, score_max: 0 } + command = bundle_prefix + './bin/taskman list 2>&1' + before = [] + IO.popen(command) do |fh| + before.concat fh.readlines.map(&:strip) + end + + testid = ((before.first || "").match(/^(\d+):/) || [])[1] + command = bundle_prefix + "./bin/taskman mod #{testid} Modified 2>&1" + res[:log] << "Running command: #{command}" + IO.popen(command) do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if $?.success? and no_error + res[:score_max] += 1 + + command = bundle_prefix + './bin/taskman list 2>&1' + after = [] + IO.popen(command) do |fh| + after.concat fh.readlines.map(&:strip) + end + + mod_found = after.select{|line| line =~ /^\s*#{testid}:.*Modified/ } + res[:score] += 1 unless mod_found.empty? + res[:errors] << "Title was not changed" if mod_found.empty? + res[:score_max] += 1 + + + res[:log] << "List difference :" + diff = (before - after).map{|x| '-' + x} + (after - before).map{|x| '+' + x} + res[:log].concat diff + res[:score] += 1 if diff.length == 2 + res[:score_max] += 1 + + # res[:errors] << "Save was not implemented" if diff.length < 1 + # res[:errors] << "Listing called everywhere" if diff.length > 1 res end @@ -259,16 +351,14 @@ class Project else success = res[:score].to_f / res[:score_max].to_f success_pcent = (success * 100).to_i - if success == 1.0 then - # success case - puts ("%d%% success" % success_pcent).green - elsif success > 0.5 then - puts ("%d%% error" % success_pcent).yellow - else - puts ("%d%% error" % success_pcent).red + case (10 * success).to_i + when 10 then puts ("%d%% success" % success_pcent).green + when 3..9 then puts ("%d%% error" % success_pcent).yellow + else puts ("%d%% error" % success_pcent).red end - @score += res[:score] + @score += res[:score] @score_max += res[:score_max] + @errors += res[:errors].size end unless res[:log].empty? then @@ -276,7 +366,12 @@ class Project puts "" end unless res[:errors].empty? then - puts res[:errors].map{|line| indent + line.red }.join("\n") + puts ( + res[:errors] + .map{|line| "EWOGA_ERROR: " + line } + .map{|line| indent + line.red } + .join("\n") + ) puts "" end @@ -284,8 +379,9 @@ class Project $INDENT = prev_INDENT end - def score - puts indent + "[ #{@score} / #{@score_max} ]" + def score out + puts indent + "[ passed #{@score} / total #{@score_max} / errors #{@errors} ]" + out.puts "#{File.basename @path}, #{@score}, #{@score_max}, #{@errors}" end private @@ -304,13 +400,14 @@ else projects = ARGV end +out = File.open('result.csv','w+') projects.each do |name| projectpath = Pathname.new(name).realpath.to_s puts "[#{File.basename(projectpath).yellow.on_blue}] #{projectpath}" project = Project.new(projectpath) project.extract - project.patch + project.test :patch, "Apply corrective patches" project.test :structure, "Testing project structure" project.test :bundle_install, "Installing bundled Gems" project.test :taskman_help_short, "Testing taskman short help" @@ -318,11 +415,14 @@ projects.each do |name| project.test :taskman_command_wrong, "Testing taskman wrong command" project.test :taskman_command_list, "Testing taskman command: list" project.test :taskman_command_add, "Testing taskman command: add" - project.test :taskman_command_del, "Testing taskman command: del" project.test :taskman_command_mod, "Testing taskman command: mod" - project.score + project.test :taskman_command_del, "Testing taskman command: del" + project.score out + puts "" end +out.close + exit 0