glenux/ewoga
glenux/ewoga
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Update tests.

Browse source
This commit is contained in:
Glenn Y. Rolland 2016-06-17 01:32:08 +02:00
parent 866b079a98
commit 97d76be063
4 changed files with 524 additions and 366 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -9,3 +9,4 @@ vendor/bundle*
/pkg/ /pkg/
/spec/reports/ /spec/reports/
/tmp/ /tmp/
/extract

366
bin/ewoga
View file

@ -1,366 +0,0 @@
#!/usr/bin/env ruby
#require 'bundler/setup'
#Bundler.require
require 'pry'
require 'zlib'
require 'net/imap'
require 'pp'
require 'mechanize'
require 'yaml'
require 'hash_validator'
require 'uri'
require 'thor'
require 'json'
require 'mail'
require 'colorize'
#Net::IMAP.debug = true
class Hash
#take keys of hash and transform those to a symbols
def self.transform_keys_to_symbols(value)
return value if not value.is_a?(Hash)
hash = value.inject({}) do |memo,(k,v)|
memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo
end
return hash
end
end
module Ewoga
EPAFI_CONFIG_FILE = File.join(ENV['HOME'],'.ewoga','config.yml')
EPAFI_IGNORE_FILE = File.join(ENV['HOME'],'.ewoga','ignore.yml')
class ContactManager
CRM_LOGIN_URL = '/login'
CRM_LEADS_URL = '/leads.json'
CRM_CONTACTS_URL = '/contacts.json'
def initialize config
@config = config
@browser = Mechanize.new { |agent|
agent.user_agent_alias = 'Mac Safari'
}
@ignore_list = Set.new
@keep_list = Set.new
## Load configuration file
#
unless File.exist? EPAFI_CONFIG_FILE then
raise "Unable to find configuration file #{EPAFI_CONFIG_FILE}"
end
@config = config
connect!
load_contacts
load_leads
load_ignore
#puts @keep_list.to_a
rescue RuntimeError => e
STDERR.puts e.message
end
def connect!
@browser.get(@config[:crm][:baseurl] + CRM_LOGIN_URL) do |page|
page.form_with(action: '/authentication') do |f|
f['authentication[username]'] = @config[:crm][:login]
f['authentication[password]'] = @config[:crm][:password]
end.click_button
end
rescue Mechanize::ResponseCodeError
raise "Authentication error. Verify your credentials."
end
def load_ignore
if File.exist? EPAFI_IGNORE_FILE
ignore_list = YAML.load_file(EPAFI_IGNORE_FILE)
ignore_list.each do |email|
@ignore_list << email.strip.downcase
end
end
end
def load_leads page=1
crm_leads_page = @browser.get(@config[:crm][:baseurl] + CRM_LEADS_URL + "?page=#{page}")
crm_leads = JSON.parse crm_leads_page.body
crm_leads.each do |lead_obj|
keep_contact lead_obj['lead']['email'].split(',')
keep_contact lead_obj['lead']['alt_email'].split(',')
end
if crm_leads.size > 0 then
load_leads (page + 1)
end
end
def load_contacts page=1
crm_contacts_page = @browser.get(@config[:crm][:baseurl] + CRM_CONTACTS_URL + "?page=#{page}")
crm_contacts = JSON.parse crm_contacts_page.body
crm_contacts.each do |contact_obj|
keep_contact contact_obj['contact']['email'].split(',')
keep_contact contact_obj['contact']['alt_email'].split(',')
end
if crm_contacts.size > 0 then
load_contacts (page + 1)
end
#contacts.to_a.sort.join(', ')
end
def keep_contact emails
emails = emails.to_a if emails.is_a? Set
[emails].flatten.each do |mail|
@keep_list << mail.strip.downcase
end
end
def ignore_contact emails
emails = emails.to_a if emails.is_a? Set
[emails].flatten.each do |mail|
@ignore_list << mail.strip.downcase
end
File.open(EPAFI_IGNORE_FILE, 'w') do |f|
f.write @ignore_list.to_a.to_yaml
end
end
def include? mail
return (
(@ignore_list.include? mail.strip.downcase) or
(@keep_list.include? mail.strip.downcase)
)
end
end
class CrawlerApp
attr_reader :imap
attr_reader :contacts
TMPMAIL_FILE = '.tmpmail'
def initialize config
@saved_key = 'RFC822'
@filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase
@config = config
@imap = nil
@contact_manager = ContactManager.new config
end
def connect!
@imap = Net::IMAP.new(
@config[:imap][:server],
ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE},
port: 993
)
@imap.login(@config[:imap][:login], @config[:imap][:password])
#@imap.select(SOURCE_MAILBOX)
end
def disconnect!
imap.logout
imap.disconnect
end
MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/
def examine_message message
m = Mail.read_from_string message.attr[@saved_key]
return if m.from.nil?
return if m.to.nil?
emails = Set.new
begin
emails.merge m.from
emails.merge [m.to].flatten if m.to
emails.merge [m.cc].flatten if m.cc
rescue => e
binding.pry
end
body_emails = Set.new
m.body.parts.each do |part|
next if part.content_type != 'text/plain'
#body_emails = m.body.decoded.scan MAIL_REGEXP
part_emails = part.decoded.scan MAIL_REGEXP
#pp body_emails
if not part_emails.empty? then
body_emails.merge part_emails
end
end
emails.merge body_emails
# puts emails.to_a.join(' , ')
remaining_emails = (
emails
.map{ |e| [e, (@contact_manager.include? e)] }
.select{ |e,t| !t }
)
seen_emails = (
remaining_emails
.empty?
)
# puts @contacts.to_a.join(', ')
if seen_emails then
print "."
return
else
puts ""
all_addr = {
from: (m.from || []),
to: (m.to || []),
cc: (m.cc || []),
body: (body_emails || [])
}
all_addr.each do |key, list|
list.each do |addr|
addr_str = if remaining_emails.map{|e,t| e}.include? addr then
addr.yellow.on_black
else addr
end
str = "%4s: %s" % [key.to_s.upcase, addr_str]
puts str
end
end
puts ""
#puts " ORIGINAL EMAILS: #{emails.to_a.join(', ')}"
#puts "REMAINING EMAILS: #{remaining_emails.map{|e,t| e}.join(', ')}".yellow.on_black
#puts " SEEN EMAILS: #{seen_emails}"
end
while true
begin
puts "\n### #{m.subject}"
print "#{m.from.join(',')} --> #{m.to.join(',')} "
puts "[Ignore/Add/Skip/Detail] ?"
i = STDIN.gets
case i.strip
when /^[iI]$/ then # ignore
@contact_manager.ignore_contact remaining_emails.map{|e,t| e}
break
when /^[aA]$/ then # add
@contact_manager.keep_contact remaining_emails.map{|e,t| e}
break
when /^[sS]$/ then #skip
break
when /^[dD]$/ then # decode
# puts m.body.decoded
File.open(TMPMAIL_FILE + ".2", 'w') do |f|
f.write message.attr[@saved_key]
end
system "formail < #{TMPMAIL_FILE}.2 > #{TMPMAIL_FILE}"
system "mutt -R -f #{TMPMAIL_FILE}"
end
rescue Encoding::ConverterNotFoundError
STDERR.puts "ERROR: encoding problem in email. Unable to convert."
end
end
return
end
def examine_all
@imap.list('', '*').each do |mailbox|
puts "\nMAILBOX #{mailbox.name}".yellow
next unless mailbox.name =~ /#{@config[:imap][:pattern]}/
@imap.examine mailbox.name
puts "Searching #{mailbox.name}"
messages_in_mailbox = @imap.responses['EXISTS'][0]
if not messages_in_mailbox then
say "#{mailbox.name} does not have any messages"
next
end
@imap.select mailbox.name #GYR: TEST
ids = @imap.search('SINCE 1-Jan-2001')
# NOT OR TO "@agilefant.org" CC "@agilefant.org"')
if ids.empty?
puts "\tFound no messages"
else
examine_message_list mailbox.name, ids
end
end
end
def examine_message_list mailbox_name, ids
ids.each do |id|
@imap.select mailbox_name #GYR: TEST
message = imap.fetch(id, [@saved_key])[0]
examine_message message
end
rescue IOError
# re-connect and try again
connect!
retry
end
end
class Crawler < Thor
CONFIG_FILE = 'config/secrey.yml'
include Thor::Actions
default_task :crawl
desc 'crawl', 'Crawls email to save mails'
def crawl
#saved_info = []
parse_configuration
## Run application
app = CrawlerApp.new @config
app.connect!
app.examine_all
#pp saved_info
app.disconnect!
end
def initialize *args
@config = {}
super
end
private
def parse_configuration
## Load configuration
@config.merge! Hash.transform_keys_to_symbols(
YAML::load( File.open( EPAFI_CONFIG_FILE ) )
)
## Validate configuration structure
validations = {
crm: {
baseurl: lambda { |url| url =~ URI::regexp },
login: 'string',
password: 'string'
},
imap: {
server: 'string',
login: 'string',
password: 'string'
}
}
validator = HashValidator.validate(@config, validations)
raise "Configuration is not valid: #{validator.errors.inspect}" unless validator.valid?
end
end
end
Ewoga::Crawler.start

195
bin/ewoga-fetch Executable file
View file

@ -0,0 +1,195 @@
#!/usr/bin/env ruby
# set ts=2 sw=2 et
require 'pry'
require 'zlib'
require 'net/imap'
require 'pp'
require 'mechanize'
require 'yaml'
require 'hash_validator'
require 'uri'
require 'thor'
require 'json'
require 'mail'
require 'colorize'
#Net::IMAP.debug = true
class Hash
#take keys of hash and transform those to a symbols
def self.transform_keys_to_symbols(value)
return value if not value.is_a?(Hash)
hash = value.inject({}) do |memo,(k,v)|
memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo
end
return hash
end
end
module Ewoga
EWOGA_CONFIG_FILE = File.join(ENV['HOME'],'.ewoga','config.yml')
EWOGA_IGNORE_FILE = File.join(ENV['HOME'],'.ewoga','ignore.yml')
class InvalidConfiguration < Exception ; end
class CrawlerApp
attr_reader :imap
attr_reader :contacts
TMPMAIL_FILE = '.tmpmail'
def initialize config
@saved_key = 'RFC822'
@filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase
@config = config
@imap = nil
end
def connect!
@imap = Net::IMAP.new(
@config[:imap][:server],
ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE},
port: 993
)
@imap.login(@config[:imap][:login], @config[:imap][:password])
end
def disconnect!
imap.logout
imap.disconnect
end
MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/
def examine_message message
m = Mail.read_from_string message.attr[@saved_key]
return if m.from.nil?
return if m.to.nil?
return unless m.subject =~ /RUBY\s+NEXTFORMATION/
begin
puts "\n### #{m.subject}"
puts "### #{m.date}"
print "#{m.from.first} --> #{m.to.join(',')} "
attach = m.attachments.first
fn = "extract/%s/projet.tar" % [m.from.first, attach.filename]
puts "- #{fn}"
begin
FileUtils.mkdir_p File.dirname(fn)
File.open( fn, "w+b", 0644 ) { |f| f.write attach.decoded}
rescue Exception => e
puts "Error : Unable to save data for #{fn} because #{e.message}"
end
rescue Encoding::ConverterNotFoundError
STDERR.puts "ERROR: encoding problem in email. Unable to convert."
end
return
end
def examine_all
@imap.select "INBOX"
# ids = @imap.search('SUBJECT NEXTFORMATION')
ids = @imap.sort(['DATE'], ['SUBJECT', 'NEXTFORMATION'], 'US-ASCII')
if ids.empty?
puts "\tFound no messages"
else
examine_message_list "INBOX", ids
end
end
def examine_message_list mailbox_name, ids
ids.each do |id|
@imap.select mailbox_name #GYR: TEST
message = imap.fetch(id, [@saved_key])[0]
examine_message message
end
rescue IOError
# re-connect and try again
connect!
retry
end
end
class Crawler < Thor
CONFIG_FILE = 'config/secrey.yml'
include Thor::Actions
default_task :crawl
desc 'crawl', 'Crawls email to save mails'
def crawl
#saved_info = []
parse_configuration
## Run application
app = CrawlerApp.new @config
app.connect!
app.examine_all
app.disconnect!
end
def initialize *args
@config = {}
super
end
private
def parse_configuration
## Load configuration
#
unless File.exist? EWOGA_CONFIG_FILE then
puts "Creating sample configuration file #{EWOGA_CONFIG_FILE}"
FileUtils.mkdir_p File.dirname(EWOGA_CONFIG_FILE)
File.open(EWOGA_CONFIG_FILE, "w") do |fh|
fh.puts "imap:"
fh.puts " server: EXAMPLE.COM"
fh.puts " login: FOO"
fh.puts " password: BAR"
end
exit 1
end
@config.merge! Hash.transform_keys_to_symbols(
YAML::load( File.open( EWOGA_CONFIG_FILE ) )
)
## Validate configuration structure
validations = {
imap: {
server: 'string',
login: 'string',
password: 'string'
}
}
validator = HashValidator.validate(@config, validations)
raise InvalidConfiguration.new(
"Configuration is not valid: #{validator.errors.inspect}"
) unless validator.valid?
raise InvalidConfiguration.new(
"Configuration is not valid: please modify #{EWOGA_CONFIG_FILE}"
) if @config[:imap][:server] == "EXAMPLE.COM"
end
end
end
begin
Ewoga::Crawler.start ARGV
exit 0
rescue SystemExit => e
raise e
rescue Exception => e
puts "ERROR: #{e.class} #{e}"
exit 1
end

328
bin/ewoga-test Executable file
View file

@ -0,0 +1,328 @@
#!/usr/bin/env ruby
require 'pathname'
require 'fileutils'
require 'colorize'
ROOTDIR=Pathname.new(__FILE__).dirname.parent.realpath.to_s
DATADIR=(Pathname.new(ROOTDIR) + 'extract').to_s
# puts "ROOTDIR = #{ROOTDIR}"
# puts "DATA = #{DATADIR}"
$INDENT = 2
def indent
" " * $INDENT
end
class Project
def initialize path
@path = path
@name = File.basename path
@score = 0
@score_max = 0
end
class ExtractionError < Exception ; end
def extract
Dir.chdir(@path)
print indent + "Extracting project data ... "
tarfile = @path + '/projet.tar'
system "tar xavf #{tarfile} > extractlog"
raise ExtractionError unless $?.success?
puts "success".green
dir = %x{head -n1 extractlog}.strip
if dir != 'taskman/' then
FileUtils.rm_rf 'taskman'
FileUtils.mv dir, 'taskman'
end
FileUtils.rm 'extractlog'
puts ""
end
def patch
Dir.chdir(@path)
print indent + "Patching project data ... "
$INDENT += 2
if File.exist? 'patch.d' then
log = []
Dir.glob('patch.d/*.patch').sort.each do |patchfile|
patchname = File.basename(patchfile).gsub(/\.patch$/,'')
IO.popen("patch -p0 < #{patchfile}") do |fh|
log.concat fh.readlines.map(&:strip).map { |line|
patchname + ': ' + line
}
end
end
puts "success".green
puts log.map {|line| indent + line }
else
puts 'skipping'
end
$INDENT -= 2
puts ""
end
def test_structure
score = 0
score_max = 0
errors = []
['bin', 'lib', 'lib/taskman'].each do |dir|
score_max += 2
if File.exist? dir then score += 1
else errors << "Missing directory #{dir}"
end
if File.directory? dir then score +=1
else errors << "#{dir} must be a directory"
end
end
['Gemfile', 'bin/taskman', 'lib/taskman.rb'].each do |file|
score_max += 1
if File.exist? file then score += 1
else errors << "Missing file #{file}"
end
end
log = IO.popen('find').readlines
.map{|line| line.strip }
.reject do |line|
# hide some files we're not interested in
case line.strip
when /\/ruby\/2\.3\.0\// then true
when /.swp/ then true
when /^\.$/ then true
when /~$/ then true
else false
end
end
.map{ |line| line.strip[2..-1] }
{ log: log,
score: score,
score_max: score_max,
errors: errors
}
end
def test_bundle_install
res = { log: [], errors: [], score: 0, score_max: 0 }
if File.exist? 'Gemfile' then
IO.popen('bundle install --path vendor/bundler') do |fh|
res[:log].concat fh.readlines.map(&:strip)
end
res[:score] += 1 if $?.success?
res[:score_max] += 1
end
res
end
def test_taskman_help
res = { log: [], errors: [], score: 0, score_max: 0 }
command = bundle_prefix + './bin/taskman -h 2>&1'
res[:log] << "Running command: #{command}"
IO.popen(command) do |fh|
res[:log].concat fh.readlines.map(&:strip)
end
no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty?
res[:score] += 1 if $?.success? and no_error
res[:score_max] += 1
res
end
# help (-h)
def test_taskman_help_short
res = { log: [], errors: [], score: 0, score_max: 0 }
command = bundle_prefix + './bin/taskman -h 2>&1'
res[:log] << "Running command: #{command}"
IO.popen(command) do |fh|
res[:log].concat fh.readlines.map(&:strip)
end
no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty?
res[:score] += 1 if $?.success? and no_error
res[:score_max] += 1
res
end
# help long (--help)
def test_taskman_help_long
res = { log: [], errors: [], score: 0, score_max: 0 }
command = bundle_prefix + './bin/taskman --help 2>&1'
res[:log] << "Running command: #{command}"
IO.popen(command) do |fh|
res[:log].concat fh.readlines.map(&:strip)
end
no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty?
res[:score] += 1 if $?.success? and no_error
res[:score_max] += 1
res
end
# wrong command
def test_taskman_command_wrong
res = { log: [], errors: [], score: 0, score_max: 0 }
command = bundle_prefix + './bin/taskman wrong-command 2>&1'
res[:log] << "Running command: #{command}"
IO.popen(command) do |fh|
res[:log].concat fh.readlines.map(&:strip)
end
no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty?
res[:score] += 1 if not $?.success? and not no_error
res[:score_max] += 1
res
end
def test_taskman_command_list
res = { log: [], errors: [], score: 0, score_max: 0 }
command = bundle_prefix + './bin/taskman list 2>&1'
res[:log] << "Running command: #{command}"
IO.popen(command) do |fh|
res[:log].concat fh.readlines.map(&:strip)
end
no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty?
res[:score] += 1 if $?.success? and no_error
res[:score_max] += 1
res
end
def test_taskman_command_add
res = { log: [], errors: [], score: 0, score_max: 0 }
command = bundle_prefix + './bin/taskman list 2>&1'
before = []
IO.popen(command) do |fh|
before.concat fh.readlines.map(&:strip)
end
command = bundle_prefix + './bin/taskman add "Tester taskman 1" 2>&1'
res[:log] << "Running command: #{command}"
IO.popen(command) do |fh|
res[:log].concat fh.readlines.map(&:strip)
end
no_error = res[:log].select{|line| line =~ /ERROR/i or line =~ /Erreur/i }.empty?
res[:score] += 1 if $?.success? and no_error
res[:score_max] += 1
command = bundle_prefix + './bin/taskman list 2>&1'
after = []
IO.popen(command) do |fh|
after.concat fh.readlines.map(&:strip)
end
res
end
def test_taskman_command_del
res = { log: [], errors: [], score: 0, score_max: 0 }
res
end
def test_taskman_command_mod
res = { log: [], errors: [], score: 0, score_max: 0 }
res
end
def test name, desc
print indent + desc + " ... "
prev_INDENT = $INDENT
$INDENT += 2
@score_max += 1
Dir.chdir(@path + '/taskman')
res = self.send(('test_' + name.to_s).to_sym)
if res[:score_max] == 0 then
puts "skipping"
puts ""
else
success = res[:score].to_f / res[:score_max].to_f
success_pcent = (success * 100).to_i
if success == 1.0 then
# success case
puts ("%d%% success" % success_pcent).green
elsif success > 0.5 then
puts ("%d%% error" % success_pcent).yellow
else
puts ("%d%% error" % success_pcent).red
end
@score += res[:score]
@score_max += res[:score_max]
end
unless res[:log].empty? then
puts res[:log].map{|line| indent + line }.join("\n")
puts ""
end
unless res[:errors].empty? then
puts res[:errors].map{|line| indent + line.red }.join("\n")
puts ""
end
ensure
$INDENT = prev_INDENT
end
def score
puts indent + "[ #{@score} / #{@score_max} ]"
end
private
def bundle_prefix
prefix = ''
prefix = 'bundle exec ' if File.exist? 'Gemfile'
prefix
end
end
projects = []
if ARGV.empty? then
projects = Dir.glob(DATADIR + '/*')
else
projects = ARGV
end
projects.each do |name|
projectpath = Pathname.new(name).realpath.to_s
puts "[#{File.basename(projectpath).yellow.on_blue}] #{projectpath}"
project = Project.new(projectpath)
project.extract
project.patch
project.test :structure, "Testing project structure"
project.test :bundle_install, "Installing bundled Gems"
project.test :taskman_help_short, "Testing taskman short help"
project.test :taskman_help_long, "Testing taskman long help"
project.test :taskman_command_wrong, "Testing taskman wrong command"
project.test :taskman_command_list, "Testing taskman command: list"
project.test :taskman_command_add, "Testing taskman command: add"
project.test :taskman_command_del, "Testing taskman command: del"
project.test :taskman_command_mod, "Testing taskman command: mod"
project.score
puts ""
end
exit 0