From 97d76be0638019493d8281926a58a2b7528ae393 Mon Sep 17 00:00:00 2001 From: "Glenn Y. Rolland" Date: Fri, 17 Jun 2016 01:32:08 +0200 Subject: [PATCH] Update tests. --- .gitignore | 1 + bin/ewoga | 366 ------------------------------------------------ bin/ewoga-fetch | 195 ++++++++++++++++++++++++++ bin/ewoga-test | 328 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 524 insertions(+), 366 deletions(-) delete mode 100755 bin/ewoga create mode 100755 bin/ewoga-fetch create mode 100755 bin/ewoga-test diff --git a/.gitignore b/.gitignore index 1a57f63..3e10125 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ vendor/bundle* /pkg/ /spec/reports/ /tmp/ +/extract diff --git a/bin/ewoga b/bin/ewoga deleted file mode 100755 index 5cafe52..0000000 --- a/bin/ewoga +++ /dev/null @@ -1,366 +0,0 @@ -#!/usr/bin/env ruby - -#require 'bundler/setup' -#Bundler.require - -require 'pry' -require 'zlib' -require 'net/imap' -require 'pp' -require 'mechanize' -require 'yaml' -require 'hash_validator' -require 'uri' -require 'thor' -require 'json' -require 'mail' -require 'colorize' - -#Net::IMAP.debug = true - -class Hash - #take keys of hash and transform those to a symbols - def self.transform_keys_to_symbols(value) - return value if not value.is_a?(Hash) - hash = value.inject({}) do |memo,(k,v)| - memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo - end - return hash - end -end - -module Ewoga - EPAFI_CONFIG_FILE = File.join(ENV['HOME'],'.ewoga','config.yml') - EPAFI_IGNORE_FILE = File.join(ENV['HOME'],'.ewoga','ignore.yml') - - class ContactManager - - CRM_LOGIN_URL = '/login' - CRM_LEADS_URL = '/leads.json' - CRM_CONTACTS_URL = '/contacts.json' - - - def initialize config - @config = config - - @browser = Mechanize.new { |agent| - agent.user_agent_alias = 'Mac Safari' - } - @ignore_list = Set.new - @keep_list = Set.new - - ## Load configuration file - # - - unless File.exist? EPAFI_CONFIG_FILE then - raise "Unable to find configuration file #{EPAFI_CONFIG_FILE}" - end - @config = config - - - connect! - load_contacts - load_leads - load_ignore - #puts @keep_list.to_a - rescue RuntimeError => e - STDERR.puts e.message - end - - def connect! - @browser.get(@config[:crm][:baseurl] + CRM_LOGIN_URL) do |page| - page.form_with(action: '/authentication') do |f| - f['authentication[username]'] = @config[:crm][:login] - f['authentication[password]'] = @config[:crm][:password] - end.click_button - end - - rescue Mechanize::ResponseCodeError - raise "Authentication error. Verify your credentials." - end - - def load_ignore - if File.exist? EPAFI_IGNORE_FILE - ignore_list = YAML.load_file(EPAFI_IGNORE_FILE) - ignore_list.each do |email| - @ignore_list << email.strip.downcase - end - end - end - - def load_leads page=1 - crm_leads_page = @browser.get(@config[:crm][:baseurl] + CRM_LEADS_URL + "?page=#{page}") - crm_leads = JSON.parse crm_leads_page.body - crm_leads.each do |lead_obj| - keep_contact lead_obj['lead']['email'].split(',') - keep_contact lead_obj['lead']['alt_email'].split(',') - end - - if crm_leads.size > 0 then - load_leads (page + 1) - end - end - - def load_contacts page=1 - crm_contacts_page = @browser.get(@config[:crm][:baseurl] + CRM_CONTACTS_URL + "?page=#{page}") - crm_contacts = JSON.parse crm_contacts_page.body - crm_contacts.each do |contact_obj| - keep_contact contact_obj['contact']['email'].split(',') - keep_contact contact_obj['contact']['alt_email'].split(',') - end - - if crm_contacts.size > 0 then - load_contacts (page + 1) - end - #contacts.to_a.sort.join(', ') - end - - def keep_contact emails - emails = emails.to_a if emails.is_a? Set - [emails].flatten.each do |mail| - @keep_list << mail.strip.downcase - end - end - - def ignore_contact emails - emails = emails.to_a if emails.is_a? Set - [emails].flatten.each do |mail| - @ignore_list << mail.strip.downcase - end - File.open(EPAFI_IGNORE_FILE, 'w') do |f| - f.write @ignore_list.to_a.to_yaml - end - end - - def include? mail - return ( - (@ignore_list.include? mail.strip.downcase) or - (@keep_list.include? mail.strip.downcase) - ) - end - end - - class CrawlerApp - attr_reader :imap - attr_reader :contacts - - TMPMAIL_FILE = '.tmpmail' - - def initialize config - @saved_key = 'RFC822' - @filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase - @config = config - @imap = nil - @contact_manager = ContactManager.new config - end - - - def connect! - @imap = Net::IMAP.new( - @config[:imap][:server], - ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE}, - port: 993 - ) - @imap.login(@config[:imap][:login], @config[:imap][:password]) - #@imap.select(SOURCE_MAILBOX) - end - - def disconnect! - imap.logout - imap.disconnect - end - - MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/ - - def examine_message message - m = Mail.read_from_string message.attr[@saved_key] - return if m.from.nil? - return if m.to.nil? - - - emails = Set.new - begin - emails.merge m.from - emails.merge [m.to].flatten if m.to - emails.merge [m.cc].flatten if m.cc - rescue => e - binding.pry - end - - body_emails = Set.new - m.body.parts.each do |part| - next if part.content_type != 'text/plain' - - #body_emails = m.body.decoded.scan MAIL_REGEXP - part_emails = part.decoded.scan MAIL_REGEXP - #pp body_emails - if not part_emails.empty? then - body_emails.merge part_emails - end - end - emails.merge body_emails - - # puts emails.to_a.join(' , ') - remaining_emails = ( - emails - .map{ |e| [e, (@contact_manager.include? e)] } - .select{ |e,t| !t } - ) - seen_emails = ( - remaining_emails - .empty? - ) - # puts @contacts.to_a.join(', ') - if seen_emails then - print "." - return - else - puts "" - all_addr = { - from: (m.from || []), - to: (m.to || []), - cc: (m.cc || []), - body: (body_emails || []) - } - all_addr.each do |key, list| - list.each do |addr| - addr_str = if remaining_emails.map{|e,t| e}.include? addr then - addr.yellow.on_black - else addr - end - str = "%4s: %s" % [key.to_s.upcase, addr_str] - puts str - end - end - puts "" - #puts " ORIGINAL EMAILS: #{emails.to_a.join(', ')}" - #puts "REMAINING EMAILS: #{remaining_emails.map{|e,t| e}.join(', ')}".yellow.on_black - #puts " SEEN EMAILS: #{seen_emails}" - end - - while true - begin - puts "\n### #{m.subject}" - print "#{m.from.join(',')} --> #{m.to.join(',')} " - puts "[Ignore/Add/Skip/Detail] ?" - - i = STDIN.gets - case i.strip - when /^[iI]$/ then # ignore - @contact_manager.ignore_contact remaining_emails.map{|e,t| e} - break - when /^[aA]$/ then # add - @contact_manager.keep_contact remaining_emails.map{|e,t| e} - break - when /^[sS]$/ then #skip - break - when /^[dD]$/ then # decode - # puts m.body.decoded - File.open(TMPMAIL_FILE + ".2", 'w') do |f| - f.write message.attr[@saved_key] - end - system "formail < #{TMPMAIL_FILE}.2 > #{TMPMAIL_FILE}" - system "mutt -R -f #{TMPMAIL_FILE}" - end - rescue Encoding::ConverterNotFoundError - STDERR.puts "ERROR: encoding problem in email. Unable to convert." - end - end - - return - end - - def examine_all - @imap.list('', '*').each do |mailbox| - puts "\nMAILBOX #{mailbox.name}".yellow - next unless mailbox.name =~ /#{@config[:imap][:pattern]}/ - @imap.examine mailbox.name - - puts "Searching #{mailbox.name}" - messages_in_mailbox = @imap.responses['EXISTS'][0] - if not messages_in_mailbox then - say "#{mailbox.name} does not have any messages" - next - end - - @imap.select mailbox.name #GYR: TEST - ids = @imap.search('SINCE 1-Jan-2001') - # NOT OR TO "@agilefant.org" CC "@agilefant.org"') - if ids.empty? - puts "\tFound no messages" - else - examine_message_list mailbox.name, ids - end - end - end - - def examine_message_list mailbox_name, ids - ids.each do |id| - @imap.select mailbox_name #GYR: TEST - message = imap.fetch(id, [@saved_key])[0] - examine_message message - end - rescue IOError - # re-connect and try again - connect! - retry - end - end - - class Crawler < Thor - - CONFIG_FILE = 'config/secrey.yml' - - include Thor::Actions - default_task :crawl - - - desc 'crawl', 'Crawls email to save mails' - def crawl - #saved_info = [] - parse_configuration - - ## Run application - app = CrawlerApp.new @config - - app.connect! - app.examine_all - #pp saved_info - app.disconnect! - end - - def initialize *args - @config = {} - super - end - - private - - - def parse_configuration - ## Load configuration - @config.merge! Hash.transform_keys_to_symbols( - YAML::load( File.open( EPAFI_CONFIG_FILE ) ) - ) - - ## Validate configuration structure - validations = { - crm: { - baseurl: lambda { |url| url =~ URI::regexp }, - login: 'string', - password: 'string' - }, - imap: { - server: 'string', - login: 'string', - password: 'string' - } - } - validator = HashValidator.validate(@config, validations) - raise "Configuration is not valid: #{validator.errors.inspect}" unless validator.valid? - end - end -end - -Ewoga::Crawler.start - diff --git a/bin/ewoga-fetch b/bin/ewoga-fetch new file mode 100755 index 0000000..70ff09c --- /dev/null +++ b/bin/ewoga-fetch @@ -0,0 +1,195 @@ +#!/usr/bin/env ruby + +# set ts=2 sw=2 et + +require 'pry' +require 'zlib' +require 'net/imap' +require 'pp' +require 'mechanize' +require 'yaml' +require 'hash_validator' +require 'uri' +require 'thor' +require 'json' +require 'mail' +require 'colorize' + +#Net::IMAP.debug = true + +class Hash + #take keys of hash and transform those to a symbols + def self.transform_keys_to_symbols(value) + return value if not value.is_a?(Hash) + hash = value.inject({}) do |memo,(k,v)| + memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo + end + return hash + end +end + +module Ewoga + EWOGA_CONFIG_FILE = File.join(ENV['HOME'],'.ewoga','config.yml') + EWOGA_IGNORE_FILE = File.join(ENV['HOME'],'.ewoga','ignore.yml') + + class InvalidConfiguration < Exception ; end + + class CrawlerApp + attr_reader :imap + attr_reader :contacts + + TMPMAIL_FILE = '.tmpmail' + + def initialize config + @saved_key = 'RFC822' + @filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase + @config = config + @imap = nil + end + + + def connect! + @imap = Net::IMAP.new( + @config[:imap][:server], + ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE}, + port: 993 + ) + @imap.login(@config[:imap][:login], @config[:imap][:password]) + end + + def disconnect! + imap.logout + imap.disconnect + end + + MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/ + + def examine_message message + m = Mail.read_from_string message.attr[@saved_key] + return if m.from.nil? + return if m.to.nil? + return unless m.subject =~ /RUBY\s+NEXTFORMATION/ + + begin + puts "\n### #{m.subject}" + puts "### #{m.date}" + print "#{m.from.first} --> #{m.to.join(',')} " + + attach = m.attachments.first + fn = "extract/%s/projet.tar" % [m.from.first, attach.filename] + puts "- #{fn}" + begin + FileUtils.mkdir_p File.dirname(fn) + File.open( fn, "w+b", 0644 ) { |f| f.write attach.decoded} + rescue Exception => e + puts "Error : Unable to save data for #{fn} because #{e.message}" + end + rescue Encoding::ConverterNotFoundError + STDERR.puts "ERROR: encoding problem in email. Unable to convert." + end + + return + end + + def examine_all + @imap.select "INBOX" + # ids = @imap.search('SUBJECT NEXTFORMATION') + ids = @imap.sort(['DATE'], ['SUBJECT', 'NEXTFORMATION'], 'US-ASCII') + if ids.empty? + puts "\tFound no messages" + else + examine_message_list "INBOX", ids + end + end + + def examine_message_list mailbox_name, ids + ids.each do |id| + @imap.select mailbox_name #GYR: TEST + message = imap.fetch(id, [@saved_key])[0] + examine_message message + end + rescue IOError + # re-connect and try again + connect! + retry + end + end + + class Crawler < Thor + CONFIG_FILE = 'config/secrey.yml' + + include Thor::Actions + default_task :crawl + + + desc 'crawl', 'Crawls email to save mails' + def crawl + #saved_info = [] + parse_configuration + + ## Run application + app = CrawlerApp.new @config + + app.connect! + app.examine_all + app.disconnect! + end + + def initialize *args + @config = {} + super + end + + private + + + def parse_configuration + ## Load configuration + # + unless File.exist? EWOGA_CONFIG_FILE then + puts "Creating sample configuration file #{EWOGA_CONFIG_FILE}" + FileUtils.mkdir_p File.dirname(EWOGA_CONFIG_FILE) + File.open(EWOGA_CONFIG_FILE, "w") do |fh| + fh.puts "imap:" + fh.puts " server: EXAMPLE.COM" + fh.puts " login: FOO" + fh.puts " password: BAR" + end + exit 1 + end + + @config.merge! Hash.transform_keys_to_symbols( + YAML::load( File.open( EWOGA_CONFIG_FILE ) ) + ) + + ## Validate configuration structure + validations = { + imap: { + server: 'string', + login: 'string', + password: 'string' + } + } + validator = HashValidator.validate(@config, validations) + + raise InvalidConfiguration.new( + "Configuration is not valid: #{validator.errors.inspect}" + ) unless validator.valid? + + raise InvalidConfiguration.new( + "Configuration is not valid: please modify #{EWOGA_CONFIG_FILE}" + ) if @config[:imap][:server] == "EXAMPLE.COM" + + end + end +end + +begin + Ewoga::Crawler.start ARGV + exit 0 +rescue SystemExit => e + raise e +rescue Exception => e + puts "ERROR: #{e.class} #{e}" + exit 1 +end diff --git a/bin/ewoga-test b/bin/ewoga-test new file mode 100755 index 0000000..8c6b223 --- /dev/null +++ b/bin/ewoga-test @@ -0,0 +1,328 @@ +#!/usr/bin/env ruby + +require 'pathname' +require 'fileutils' +require 'colorize' + +ROOTDIR=Pathname.new(__FILE__).dirname.parent.realpath.to_s +DATADIR=(Pathname.new(ROOTDIR) + 'extract').to_s + +# puts "ROOTDIR = #{ROOTDIR}" +# puts "DATA = #{DATADIR}" + +$INDENT = 2 +def indent + " " * $INDENT +end + +class Project + def initialize path + @path = path + @name = File.basename path + @score = 0 + @score_max = 0 + end + + class ExtractionError < Exception ; end + + def extract + Dir.chdir(@path) + print indent + "Extracting project data ... " + + tarfile = @path + '/projet.tar' + system "tar xavf #{tarfile} > extractlog" + raise ExtractionError unless $?.success? + puts "success".green + + dir = %x{head -n1 extractlog}.strip + if dir != 'taskman/' then + FileUtils.rm_rf 'taskman' + FileUtils.mv dir, 'taskman' + end + FileUtils.rm 'extractlog' + puts "" + end + + def patch + Dir.chdir(@path) + print indent + "Patching project data ... " + $INDENT += 2 + if File.exist? 'patch.d' then + log = [] + Dir.glob('patch.d/*.patch').sort.each do |patchfile| + patchname = File.basename(patchfile).gsub(/\.patch$/,'') + IO.popen("patch -p0 < #{patchfile}") do |fh| + log.concat fh.readlines.map(&:strip).map { |line| + patchname + ': ' + line + } + end + end + puts "success".green + puts log.map {|line| indent + line } + else + puts 'skipping' + end + $INDENT -= 2 + puts "" + end + + def test_structure + score = 0 + score_max = 0 + errors = [] + ['bin', 'lib', 'lib/taskman'].each do |dir| + score_max += 2 + if File.exist? dir then score += 1 + else errors << "Missing directory #{dir}" + end + + if File.directory? dir then score +=1 + else errors << "#{dir} must be a directory" + end + end + + ['Gemfile', 'bin/taskman', 'lib/taskman.rb'].each do |file| + score_max += 1 + if File.exist? file then score += 1 + else errors << "Missing file #{file}" + end + end + + log = IO.popen('find').readlines + .map{|line| line.strip } + .reject do |line| + # hide some files we're not interested in + case line.strip + when /\/ruby\/2\.3\.0\// then true + when /.swp/ then true + when /^\.$/ then true + when /~$/ then true + else false + end + end + .map{ |line| line.strip[2..-1] } + + { log: log, + score: score, + score_max: score_max, + errors: errors + } + end + + + def test_bundle_install + res = { log: [], errors: [], score: 0, score_max: 0 } + + if File.exist? 'Gemfile' then + IO.popen('bundle install --path vendor/bundler') do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + res[:score] += 1 if $?.success? + res[:score_max] += 1 + end + + res + end + + + def test_taskman_help + res = { log: [], errors: [], score: 0, score_max: 0 } + + command = bundle_prefix + './bin/taskman -h 2>&1' + res[:log] << "Running command: #{command}" + IO.popen(command) do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + + no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if $?.success? and no_error + res[:score_max] += 1 + + res + end + + # help (-h) + def test_taskman_help_short + res = { log: [], errors: [], score: 0, score_max: 0 } + + command = bundle_prefix + './bin/taskman -h 2>&1' + res[:log] << "Running command: #{command}" + IO.popen(command) do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + + no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if $?.success? and no_error + res[:score_max] += 1 + res + end + + + # help long (--help) + def test_taskman_help_long + res = { log: [], errors: [], score: 0, score_max: 0 } + + command = bundle_prefix + './bin/taskman --help 2>&1' + res[:log] << "Running command: #{command}" + IO.popen(command) do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + + no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if $?.success? and no_error + res[:score_max] += 1 + res + end + + # wrong command + def test_taskman_command_wrong + res = { log: [], errors: [], score: 0, score_max: 0 } + + command = bundle_prefix + './bin/taskman wrong-command 2>&1' + res[:log] << "Running command: #{command}" + IO.popen(command) do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + + no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if not $?.success? and not no_error + res[:score_max] += 1 + + res + end + + def test_taskman_command_list + res = { log: [], errors: [], score: 0, score_max: 0 } + + command = bundle_prefix + './bin/taskman list 2>&1' + res[:log] << "Running command: #{command}" + IO.popen(command) do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if $?.success? and no_error + res[:score_max] += 1 + + res + end + + def test_taskman_command_add + res = { log: [], errors: [], score: 0, score_max: 0 } + + command = bundle_prefix + './bin/taskman list 2>&1' + before = [] + IO.popen(command) do |fh| + before.concat fh.readlines.map(&:strip) + end + + command = bundle_prefix + './bin/taskman add "Tester taskman 1" 2>&1' + res[:log] << "Running command: #{command}" + IO.popen(command) do |fh| + res[:log].concat fh.readlines.map(&:strip) + end + no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty? + res[:score] += 1 if $?.success? and no_error + res[:score_max] += 1 + + command = bundle_prefix + './bin/taskman list 2>&1' + after = [] + IO.popen(command) do |fh| + after.concat fh.readlines.map(&:strip) + end + + res + end + + def test_taskman_command_del + res = { log: [], errors: [], score: 0, score_max: 0 } + + res + end + + def test_taskman_command_mod + res = { log: [], errors: [], score: 0, score_max: 0 } + + res + end + + def test name, desc + print indent + desc + " ... " + prev_INDENT = $INDENT + $INDENT += 2 + @score_max += 1 + Dir.chdir(@path + '/taskman') + res = self.send(('test_' + name.to_s).to_sym) + + if res[:score_max] == 0 then + puts "skipping" + puts "" + else + success = res[:score].to_f / res[:score_max].to_f + success_pcent = (success * 100).to_i + if success == 1.0 then + # success case + puts ("%d%% success" % success_pcent).green + elsif success > 0.5 then + puts ("%d%% error" % success_pcent).yellow + else + puts ("%d%% error" % success_pcent).red + end + @score += res[:score] + @score_max += res[:score_max] + end + + unless res[:log].empty? then + puts res[:log].map{|line| indent + line }.join("\n") + puts "" + end + unless res[:errors].empty? then + puts res[:errors].map{|line| indent + line.red }.join("\n") + puts "" + end + + ensure + $INDENT = prev_INDENT + end + + def score + puts indent + "[ #{@score} / #{@score_max} ]" + end + + private + + def bundle_prefix + prefix = '' + prefix = 'bundle exec ' if File.exist? 'Gemfile' + prefix + end +end + +projects = [] +if ARGV.empty? then + projects = Dir.glob(DATADIR + '/*') +else + projects = ARGV +end + +projects.each do |name| + projectpath = Pathname.new(name).realpath.to_s + puts "[#{File.basename(projectpath).yellow.on_blue}] #{projectpath}" + + project = Project.new(projectpath) + project.extract + project.patch + project.test :structure, "Testing project structure" + project.test :bundle_install, "Installing bundled Gems" + project.test :taskman_help_short, "Testing taskman short help" + project.test :taskman_help_long, "Testing taskman long help" + project.test :taskman_command_wrong, "Testing taskman wrong command" + project.test :taskman_command_list, "Testing taskman command: list" + project.test :taskman_command_add, "Testing taskman command: add" + project.test :taskman_command_del, "Testing taskman command: del" + project.test :taskman_command_mod, "Testing taskman command: mod" + project.score + puts "" +end + +exit 0 +