glenux/epafh
glenux/epafh
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Compare commits

base: glenux:develop
Branches Tags
glenux:master
glenux:develop
..
compare: glenux:master
Branches Tags
glenux:develop
glenux:master

No commits in common. "develop" and "master" have entirely different histories.
develop ... master

11 changed files with 364 additions and 436 deletions
Show stats Expand all files Collapse all files

4
Gemfile.lock
View file

@ -4,13 +4,11 @@ PATH
epafh (0.1.0)
colorize
hash_validator
highline
json
mail (~> 2.6.3)
mechanize
pry
pry-rescue
ruby-progressbar
thor
GEM
@ -21,7 +19,6 @@ GEM
domain_name (0.5.25)
unf (>= 0.0.5, < 1.0.0)
hash_validator (0.4.0)
highline (1.7.8)
http-cookie (1.0.2)
domain_name (~> 0.5)
interception (0.5)
@ -53,7 +50,6 @@ GEM
interception (>= 0.5)
pry
rake (10.4.2)
ruby-progressbar (1.7.5)
slop (3.6.0)
thor (0.19.1)
unf (0.1.4)

6
TODO.md
View file

@ -3,12 +3,6 @@ TODO
## Features
* Add support for mailbox exclude filter
* Add support for email exclude filter
support@.*
no-reply.*@.*
* Add POP3 server support

351
bin/epafh
View file

@ -15,13 +15,352 @@ require 'thor'
require 'json'
require 'mail'
require 'colorize'
require 'epafh'
#Net::IMAP.debug = true
begin
Epafh::App.start(ARGV)
rescue Interrupt
puts " Interrupt! Exiting."
exit 1
class Hash
#take keys of hash and transform those to a symbols
def self.transform_keys_to_symbols(value)
return value if not value.is_a?(Hash)
hash = value.inject({}) do |memo,(k,v)|
memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo
end
return hash
end
end
module Epafh
EPAFI_CONFIG_FILE = File.join(ENV['HOME'],'.epafh','config.yml')
EPAFI_IGNORE_FILE = File.join(ENV['HOME'],'.epafh','ignore.yml')
class ContactManager
CRM_LOGIN_URL = '/login'
CRM_LEADS_URL = '/leads.json'
CRM_CONTACTS_URL = '/contacts.json'
def initialize config
@config = config
@browser = Mechanize.new { |agent|
agent.user_agent_alias = 'Mac Safari'
}
@ignore_list = Set.new
@keep_list = Set.new
## Load configuration file
#
unless File.exist? EPAFI_CONFIG_FILE then
raise "Unable to find configuration file #{EPAFI_CONFIG_FILE}"
end
@config = config
connect!
load_contacts
load_leads
load_ignore
#puts @keep_list.to_a
rescue RuntimeError => e
STDERR.puts e.message
end
def connect!
@browser.get(@config[:crm][:baseurl] + CRM_LOGIN_URL) do |page|
page.form_with(action: '/authentication') do |f|
f['authentication[username]'] = @config[:crm][:login]
f['authentication[password]'] = @config[:crm][:password]
end.click_button
end
rescue Mechanize::ResponseCodeError
raise "Authentication error. Verify your credentials."
end
def load_ignore
if File.exist? EPAFI_IGNORE_FILE
ignore_list = YAML.load_file(EPAFI_IGNORE_FILE)
ignore_list.each do |email|
@ignore_list << email.strip.downcase
end
end
end
def load_leads page=1
crm_leads_page = @browser.get(@config[:crm][:baseurl] + CRM_LEADS_URL + "?page=#{page}")
crm_leads = JSON.parse crm_leads_page.body
crm_leads.each do |lead_obj|
keep_contact lead_obj['lead']['email'].split(',')
keep_contact lead_obj['lead']['alt_email'].split(',')
end
if crm_leads.size > 0 then
load_leads (page + 1)
end
end
def load_contacts page=1
crm_contacts_page = @browser.get(@config[:crm][:baseurl] + CRM_CONTACTS_URL + "?page=#{page}")
crm_contacts = JSON.parse crm_contacts_page.body
crm_contacts.each do |contact_obj|
keep_contact contact_obj['contact']['email'].split(',')
keep_contact contact_obj['contact']['alt_email'].split(',')
end
if crm_contacts.size > 0 then
load_contacts (page + 1)
end
#contacts.to_a.sort.join(', ')
end
def keep_contact emails
emails = emails.to_a if emails.is_a? Set
[emails].flatten.each do |mail|
@keep_list << mail.strip.downcase
end
end
def ignore_contact emails
emails = emails.to_a if emails.is_a? Set
[emails].flatten.each do |mail|
@ignore_list << mail.strip.downcase
end
File.open(EPAFI_IGNORE_FILE, 'w') do |f|
f.write @ignore_list.to_a.to_yaml
end
end
def include? mail
return (
(@ignore_list.include? mail.strip.downcase) or
(@keep_list.include? mail.strip.downcase)
)
end
end
class CrawlerApp
attr_reader :imap
attr_reader :contacts
TMPMAIL_FILE = '.tmpmail'
def initialize config
@saved_key = 'RFC822'
@filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase
@config = config
@imap = nil
@contact_manager = ContactManager.new config
end
def connect!
@imap = Net::IMAP.new(
@config[:imap][:server],
ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE},
port: 993
)
@imap.login(@config[:imap][:login], @config[:imap][:password])
#@imap.select(SOURCE_MAILBOX)
end
def disconnect!
imap.logout
imap.disconnect
end
MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/
def examine_message message
m = Mail.read_from_string message.attr[@saved_key]
return if m.from.nil?
return if m.to.nil?
emails = Set.new
begin
emails.merge m.from
emails.merge [m.to].flatten if m.to
emails.merge [m.cc].flatten if m.cc
rescue => e
binding.pry
end
body_emails = Set.new
m.body.parts.each do |part|
next if part.content_type != 'text/plain'
#body_emails = m.body.decoded.scan MAIL_REGEXP
part_emails = part.decoded.scan MAIL_REGEXP
#pp body_emails
if not part_emails.empty? then
body_emails.merge part_emails
end
end
emails.merge body_emails
# puts emails.to_a.join(' , ')
remaining_emails = (
emails
.map{ |e| [e, (@contact_manager.include? e)] }
.select{ |e,t| !t }
)
seen_emails = (
remaining_emails
.empty?
)
# puts @contacts.to_a.join(', ')
if seen_emails then
print "."
return
else
puts ""
all_addr = {
from: (m.from || []),
to: (m.to || []),
cc: (m.cc || []),
body: (body_emails || [])
}
all_addr.each do |key, list|
list.each do |addr|
addr_str = if remaining_emails.map{|e,t| e}.include? addr then
addr.yellow.on_black
else addr
end
str = "%4s: %s" % [key.to_s.upcase, addr_str]
puts str
end
end
puts ""
#puts " ORIGINAL EMAILS: #{emails.to_a.join(', ')}"
#puts "REMAINING EMAILS: #{remaining_emails.map{|e,t| e}.join(', ')}".yellow.on_black
#puts " SEEN EMAILS: #{seen_emails}"
end
while true
begin
puts "\n### #{m.subject}"
print "#{m.from.join(',')} --> #{m.to.join(',')} "
puts "[Ignore/Add/Skip/Detail] ?"
i = STDIN.gets
case i.strip
when /^[iI]$/ then # ignore
@contact_manager.ignore_contact remaining_emails.map{|e,t| e}
break
when /^[aA]$/ then # add
@contact_manager.keep_contact remaining_emails.map{|e,t| e}
break
when /^[sS]$/ then #skip
break
when /^[dD]$/ then # decode
# puts m.body.decoded
File.open(TMPMAIL_FILE + ".2", 'w') do |f|
f.write message.attr[@saved_key]
end
system "formail < #{TMPMAIL_FILE}.2 > #{TMPMAIL_FILE}"
system "mutt -R -f #{TMPMAIL_FILE}"
end
rescue Encoding::ConverterNotFoundError
STDERR.puts "ERROR: encoding problem in email. Unable to convert."
end
end
return
end
def examine_all
@imap.list('', '*').each do |mailbox|
puts "\nMAILBOX #{mailbox.name}".yellow
next unless mailbox.name =~ /#{@config[:imap][:pattern]}/
@imap.examine mailbox.name
puts "Searching #{mailbox.name}"
messages_in_mailbox = @imap.responses['EXISTS'][0]
if not messages_in_mailbox then
say "#{mailbox.name} does not have any messages"
next
end
@imap.select mailbox.name #GYR: TEST
ids = @imap.search('SINCE 1-Jan-2001')
# NOT OR TO "@agilefant.org" CC "@agilefant.org"')
if ids.empty?
puts "\tFound no messages"
else
examine_message_list mailbox.name, ids
end
end
end
def examine_message_list mailbox_name, ids
ids.each do |id|
@imap.select mailbox_name #GYR: TEST
message = imap.fetch(id, [@saved_key])[0]
examine_message message
end
rescue IOError
# re-connect and try again
connect!
retry
end
end
class Crawler < Thor
CONFIG_FILE = 'config/secrey.yml'
include Thor::Actions
default_task :crawl
desc 'crawl', 'Crawls email to save mails'
def crawl
#saved_info = []
parse_configuration
## Run application
app = CrawlerApp.new @config
app.connect!
app.examine_all
#pp saved_info
app.disconnect!
end
def initialize *args
@config = {}
super
end
private
def parse_configuration
## Load configuration
@config.merge! Hash.transform_keys_to_symbols(
YAML::load( File.open( EPAFI_CONFIG_FILE ) )
)
## Validate configuration structure
validations = {
crm: {
baseurl: lambda { |url| url =~ URI::regexp },
login: 'string',
password: 'string'
},
imap: {
server: 'string',
login: 'string',
password: 'string'
}
}
validator = HashValidator.validate(@config, validations)
raise "Configuration is not valid: #{validator.errors.inspect}" unless validator.valid?
end
end
end
Epafh::Crawler.start

23
epafh.gemspec
View file

@ -1,19 +1,27 @@
# coding: utf-8
lib = File.expand_path('../lib', __FILE__)
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
require 'epafh/constants'
require 'epafh/version'
Gem::Specification.new do |spec|
spec.name = "epafh"
spec.version = Epafh::VERSION
spec.authors = ["Glenn Y. Rolland"]
spec.email = ["glenux@glenux.net"]
spec.authors = ["@@@No user configured@@@"]
spec.email = ["@@@No user configured@@@"]
spec.summary = %q{A handy tool to extract emails and URLs from an IMAP account.}
spec.description = %q{A handy tool to extract emails and URLs from an IMAP account.}
spec.homepage = "https://github.com/glenux/epafh"
spec.summary = %q{TODO: Write a short summary, because Rubygems requires one.}
spec.description = %q{TODO: Write a longer description or delete this line.}
spec.homepage = "TODO: Put your gem's website or public repo URL here."
spec.license = "LGPL-3"
# Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
# delete this section to allow pushing this gem to any host.
if spec.respond_to?(:metadata)
spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
else
raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
end
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
spec.bindir = "bin"
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@ -29,7 +37,4 @@ Gem::Specification.new do |spec|
spec.add_runtime_dependency "hash_validator"
spec.add_runtime_dependency "pry"
spec.add_runtime_dependency "pry-rescue"
spec.add_runtime_dependency "highline"
spec.add_runtime_dependency "ruby-progressbar"
end

9
lib/epafh.rb
View file

@ -1,10 +1,5 @@
require "epafh/version"
module Epafh
# Your code goes here...
end
require "epafh/hash"
require "epafh/constants"
require "epafh/contact_manager"
require "epafh/crawler"
require "epafh/app"

120
lib/epafh/app.rb
View file

@ -1,120 +0,0 @@
require 'highline'
class Epafh::App < Thor
class InvalidConfiguration < RuntimeError ; end
CONFIG_FILE = 'config/secrey.yml'
CONFIG_DEFAULT = {
'imap' => {
'server' => '',
'login' => '',
'password' => ''
},
'crm' => {
'baseurl' => '',
'login' => '',
'password' => ''
}
}
include Thor::Actions
default_task :crawl
desc 'config', 'Initialize configuration'
def config
puts "Welcome to Epafh !".green
config = CONFIG_DEFAULT
if File.exist? Epafh::EPAFI_CONFIG_FILE then
config = config.merge(YAML::load( File.open( Epafh::EPAFI_CONFIG_FILE ) ) || {})
end
imap_params = {
server: {desc: 'IMAP hostname ? ' },
login: {desc: 'IMAP username ? ' },
password: {desc: 'IMAP password ? ', hidden: true}
}
crm_params = {
baseurl: {desc: 'CRM base url ? ' },
login: {desc: 'CRM username ? ' },
password: {desc: 'CRM password ? ', hidden: true}
}
config['imap'] = config_with_rules config['imap'], imap_params
config['crm'] = config_with_rules config['crm'], crm_params
FileUtils.mkdir_p(Epafh::EPAFI_CONFIG_DIR)
File.open(Epafh::EPAFI_CONFIG_FILE, 'w'){|f| f.write(config.to_yaml)}
end
desc 'crawl', 'Crawls email to save mails'
def crawl
parse_configuration
## Run application
app = Epafh::Crawler.new @config
app.connect!
app.examine_all
app.disconnect!
end
def initialize *args
@config = {}
super
end
private
# Ask parameters, with rule constraints
#
def config_with_rules cfg_in, rules
cli = ::HighLine.new
cfg_out = cfg_in.clone
# Loop parameter rules
rules.each.map {|param,values| [param.to_s,values] }
.each do |param, values|
backup = cfg_out[param]
backup_hidden = cfg_out[param].gsub(/./,'*')
cfg_out[param] = cli.ask(values[:desc]) do |q|
# Disable echo if hidden enabled
q.echo = '*' if values[:hidden]
# Replace default value by stars if hidden
if not cfg_out[param].empty? then
q.default =
if (values[:hidden]) then backup_hidden
else cfg_out[param]
end
end
end
# When RETURN is pressed, Highline uses default (starred)
# We have to replace it with the real value
if values[:hidden] and cfg_out[param] == backup_hidden then
cfg_out[param] = backup
end
end
cfg_out
end
def parse_configuration
## Load configuration
@config.merge! Hash.transform_keys_to_symbols(
YAML::load( File.open( Epafh::EPAFI_CONFIG_FILE ) )
)
## Validate configuration structure
validations = {
crm: {
baseurl: lambda { |url| url =~ URI::regexp },
login: 'string',
password: 'string'
},
imap: {
server: 'string',
login: 'string',
password: 'string'
}
}
validator = HashValidator.validate(@config, validations)
raise InvalidConfiguration, "Configuration is not valid: #{validator.errors.inspect}" unless validator.valid?
end
end

7
lib/epafh/constants.rb
View file

@ -1,7 +0,0 @@
module Epafh
VERSION = "0.1.0"
EPAFI_CONFIG_DIR = File.join(ENV['HOME'], '.epafh')
EPAFI_CONFIG_FILE = File.join(EPAFI_CONFIG_DIR, 'config.yml')
EPAFI_IGNORE_FILE = File.join(ENV['HOME'], '.epafh', 'ignore.yml')
end

106
lib/epafh/contact_manager.rb
View file

@ -1,106 +0,0 @@
class Epafh::ContactManager
CRM_LOGIN_URL = '/login'
CRM_LEADS_URL = '/leads.json'
CRM_CONTACTS_URL = '/contacts.json'
def initialize config
@config = config
@browser = Mechanize.new { |agent|
agent.user_agent_alias = 'Mac Safari'
}
@ignore_list = Set.new
@keep_list = Set.new
## Load configuration file
#
unless File.exist? Epafh::EPAFI_CONFIG_FILE then
raise "Unable to find configuration file #{Epafh::EPAFI_CONFIG_FILE}"
end
@config = config
connect!
load_contacts
load_leads
load_ignore
#puts @keep_list.to_a
rescue RuntimeError => e
STDERR.puts e.message
end
def connect!
@browser.get(@config[:crm][:baseurl] + CRM_LOGIN_URL) do |page|
page.form_with(action: '/authentication') do |f|
f['authentication[username]'] = @config[:crm][:login]
f['authentication[password]'] = @config[:crm][:password]
end.click_button
end
rescue Mechanize::ResponseCodeError
raise "Authentication error. Verify your credentials."
end
def load_ignore
if File.exist? Epafh::EPAFI_IGNORE_FILE
ignore_list = YAML.load_file(Epafh::EPAFI_IGNORE_FILE)
ignore_list.each do |email|
@ignore_list << email.strip.downcase
end
end
end
def load_leads page=1
crm_leads_page = @browser.get(@config[:crm][:baseurl] + CRM_LEADS_URL + "?page=#{page}")
crm_leads = JSON.parse crm_leads_page.body
crm_leads.each do |lead_obj|
keep_contact lead_obj['lead']['email'].split(',')
keep_contact lead_obj['lead']['alt_email'].split(',')
end
if crm_leads.size > 0 then
load_leads (page + 1)
end
end
def load_contacts page=1
crm_contacts_page = @browser.get(@config[:crm][:baseurl] + CRM_CONTACTS_URL + "?page=#{page}")
crm_contacts = JSON.parse crm_contacts_page.body
crm_contacts.each do |contact_obj|
keep_contact contact_obj['contact']['email'].split(',')
keep_contact contact_obj['contact']['alt_email'].split(',')
end
if crm_contacts.size > 0 then
load_contacts (page + 1)
end
#contacts.to_a.sort.join(', ')
end
def keep_contact emails
emails = emails.to_a if emails.is_a? Set
[emails].flatten.each do |mail|
@keep_list << mail.strip.downcase
end
end
def ignore_contact emails
emails = emails.to_a if emails.is_a? Set
[emails].flatten.each do |mail|
@ignore_list << mail.strip.downcase
end
File.open(Epafh::EPAFI_IGNORE_FILE, 'w') do |f|
f.write @ignore_list.to_a.to_yaml
end
end
def include? mail
return (
(@ignore_list.include? mail.strip.downcase) or
(@keep_list.include? mail.strip.downcase)
)
end
end

160
lib/epafh/crawler.rb
View file

@ -1,160 +0,0 @@
require 'ruby-progressbar'
class Epafh::Crawler
attr_reader :imap
attr_reader :contacts
TMPMAIL_FILE = '.tmpmail'
MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/
def initialize config
@saved_key = 'RFC822'
@filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase
@config = config
@imap = nil
@contact_manager = Epafh::ContactManager.new config
end
def connect!
@imap = Net::IMAP.new(
@config[:imap][:server],
ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE},
port: 993
)
@imap.login(@config[:imap][:login], @config[:imap][:password])
#@imap.select(SOURCE_MAILBOX)
end
def disconnect!
imap.logout
imap.disconnect
end
def examine_message message
m = Mail.read_from_string message.attr[@saved_key]
return if m.from.nil?
return if m.to.nil?
body_emails = extract_body_mail m.body.parts
## Create association between extracted addreses and email part
mail_struct = {
from: [m.from || []].flatten.reject{|e| e.nil?},
to: [m.to || []].flatten.reject{|e| e.nil?},
cc: [m.cc || []].flatten.reject{|e| e.nil?},
body: (body_emails.to_a || []).reject{|e| e.nil?}
}
#pp m
#pp mail_struct
emails = Set.new
mail_struct.each {|key, val| emails.merge val }
remaining_emails = emails.reject{|e| @contact_manager.include?(e) }
# Skip examination of no addresses are remaining
if remaining_emails.empty? then
return
end
display_header mail_struct, remaining_emails
while true
begin
puts "\n### #{m.subject}"
print "#{mail_struct[:from].join(',')} --> #{mail_struct[:to].join(',')} "
puts "[Ignore/Add/Skip/Detail] ?"
i = STDIN.gets
case i.strip
when /^[iI]$/ then # ignore
@contact_manager.ignore_contact remaining_emails
break
when /^[aA]$/ then # add
@contact_manager.keep_contact remaining_emails
break
when /^[sS]$/ then #skip
break
when /^[dD]$/ then # decode
# puts m.body.decoded
File.open(TMPMAIL_FILE + ".2", 'w') do |f|
f.write message.attr[@saved_key]
end
system "formail < #{TMPMAIL_FILE}.2 > #{TMPMAIL_FILE}"
system "mutt -R -f #{TMPMAIL_FILE}"
end
rescue Encoding::ConverterNotFoundError
STDERR.puts "ERROR: encoding problem in email. Unable to convert."
end
end
return
end
def examine_all
@imap.list('', '*').each do |mailbox|
puts "\nMAILBOX #{mailbox.name}".yellow
next unless mailbox.name =~ /#{@config[:imap][:pattern]}/
@imap.examine mailbox.name
puts "Searching #{mailbox.name}"
messages_in_mailbox = @imap.responses['EXISTS'][0]
if not messages_in_mailbox then
say "#{mailbox.name} does not have any messages"
next
end
@imap.select mailbox.name #GYR: TEST
ids = @imap.search('SINCE 1-Jan-2001')
# NOT OR TO "@agilefant.org" CC "@agilefant.org"')
if ids.empty?
puts "\tFound no messages"
else
examine_message_list mailbox.name, ids
end
end
end
def examine_message_list mailbox_name, ids
progressbar = ProgressBar.create(:total => ids.size)
ids.each do |id|
@imap.select mailbox_name #GYR: TEST
message = imap.fetch(id, [@saved_key])[0]
examine_message message
progressbar.increment
end
rescue IOError
# re-connect and try again
connect!
retry
end
def extract_body_mail body_parts
body_emails = Set.new
body_parts.each do |part|
next if part.content_type != 'text/plain'
part_emails = part.decoded.scan MAIL_REGEXP
if not part_emails.empty? then
body_emails.merge part_emails
end
end
body_emails
end
def display_header header_struct, remaining_emails
puts ""
header_struct.each do |key, list|
#pp list
list.each do |addr|
addr_str = if remaining_emails.include? addr then
addr.yellow.on_black
else addr
end
str = "%4s: %s" % [key.to_s.upcase, addr_str]
puts str
end
end
puts ""
end
end

11
lib/epafh/hash.rb
View file

@ -1,11 +0,0 @@
class Hash
#take keys of hash and transform those to a symbols
def self.transform_keys_to_symbols(value)
return value if not value.is_a?(Hash)
hash = value.inject({}) do |memo,(k,v)|
memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo
end
return hash
end
end

3
lib/epafh/version.rb Normal file
View file

@ -0,0 +1,3 @@
module Epafh
VERSION = "0.1.0"
end