Added comment to TODO.md

Add a progressbar !
Hide debug.
2015-11-30 21:23:34 +01:00 · 2015-11-30 01:00:52 +01:00 · 2015-11-29 21:14:02 +01:00 · 2015-11-29 11:35:46 +01:00 · 2015-11-28 22:26:55 +01:00 · 2015-11-28 12:29:41 +01:00
11 changed files with 436 additions and 364 deletions
--- a/Gemfile.lock
+++ b/Gemfile.lock
@ -4,11 +4,13 @@ PATH
    epafh (0.1.0)
      colorize
      hash_validator
      highline
      json
      mail (~> 2.6.3)
      mechanize
      pry
      pry-rescue
      ruby-progressbar
      thor
 GEM
@ -19,6 +21,7 @@ GEM
    domain_name (0.5.25)
      unf (>= 0.0.5, < 1.0.0)
    hash_validator (0.4.0)
    highline (1.7.8)
    http-cookie (1.0.2)
      domain_name (~> 0.5)
    interception (0.5)
@ -50,6 +53,7 @@ GEM
      interception (>= 0.5)
      pry
    rake (10.4.2)
    ruby-progressbar (1.7.5)
    slop (3.6.0)
    thor (0.19.1)
    unf (0.1.4)
--- a/TODO.md
+++ b/TODO.md
@ -3,6 +3,12 @@ TODO
 ## Features
 * Add support for mailbox exclude filter
 * Add support for email exclude filter 
    support@.*
    no-reply.*@.*
 * Add POP3 server support
--- a/bin/epafh
+++ b/bin/epafh
@ -15,352 +15,13 @@ require 'thor'
 require 'json'
 require 'mail'
 require 'colorize'
 require 'epafh'
 #Net::IMAP.debug = true
-class Hash
+begin
-  #take keys of hash and transform those to a symbols
+  Epafh::App.start(ARGV)
-  def self.transform_keys_to_symbols(value)
+rescue Interrupt
-    return value if not value.is_a?(Hash)
+  puts " Interrupt! Exiting."
-    hash = value.inject({}) do |memo,(k,v)| 
+  exit 1
 			memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo
 		end
    return hash
  end
 end
 module Epafh
 	EPAFI_CONFIG_FILE = File.join(ENV['HOME'],'.epafh','config.yml')
 	EPAFI_IGNORE_FILE = File.join(ENV['HOME'],'.epafh','ignore.yml')
 	class ContactManager
 		CRM_LOGIN_URL = '/login'
 		CRM_LEADS_URL = '/leads.json'
 		CRM_CONTACTS_URL = '/contacts.json'
 		def initialize config
 			@config = config
 			@browser = Mechanize.new { |agent|
 				agent.user_agent_alias = 'Mac Safari'
 			}
 			@ignore_list = Set.new
 			@keep_list = Set.new
 			## Load configuration file
 			#
 			unless File.exist? EPAFI_CONFIG_FILE then
 				raise "Unable to find configuration file #{EPAFI_CONFIG_FILE}" 
 			end
 			@config = config
 			connect!
 			load_contacts
 			load_leads
 			load_ignore
 			#puts @keep_list.to_a
 		rescue RuntimeError => e
 			STDERR.puts e.message
 		end
 		def connect!
 			@browser.get(@config[:crm][:baseurl] + CRM_LOGIN_URL) do |page|
 				page.form_with(action: '/authentication') do |f|
 					f['authentication[username]'] = @config[:crm][:login]
 					f['authentication[password]'] = @config[:crm][:password]
 				end.click_button
 			end
 		rescue Mechanize::ResponseCodeError
 			raise "Authentication error. Verify your credentials." 
 		end
 		def load_ignore
 			if File.exist? EPAFI_IGNORE_FILE
 				ignore_list = YAML.load_file(EPAFI_IGNORE_FILE)
 				ignore_list.each do |email|
 					@ignore_list << email.strip.downcase
 				end
 			end
 		end
 		def load_leads page=1
 			crm_leads_page = @browser.get(@config[:crm][:baseurl] + CRM_LEADS_URL + "?page=#{page}")
 			crm_leads = JSON.parse crm_leads_page.body
 			crm_leads.each do |lead_obj|
 				keep_contact lead_obj['lead']['email'].split(',')
 				keep_contact lead_obj['lead']['alt_email'].split(',')
 			end
 			if crm_leads.size > 0 then
 				load_leads (page + 1)
 			end
 		end
 		def load_contacts page=1
 			crm_contacts_page = @browser.get(@config[:crm][:baseurl] + CRM_CONTACTS_URL + "?page=#{page}")
 			crm_contacts = JSON.parse crm_contacts_page.body
 			crm_contacts.each do |contact_obj|
 				keep_contact contact_obj['contact']['email'].split(',')
 	 			keep_contact contact_obj['contact']['alt_email'].split(',')
 			end
 			if crm_contacts.size > 0 then
 				load_contacts (page + 1)
 			end
 			#contacts.to_a.sort.join(', ')
 		end
 		def keep_contact emails
 			emails = emails.to_a if emails.is_a? Set
 	 		[emails].flatten.each do |mail|
 				@keep_list << mail.strip.downcase
 			end
 		end
 		def ignore_contact emails
 			emails = emails.to_a if emails.is_a? Set
 	 		[emails].flatten.each do |mail|
 				@ignore_list << mail.strip.downcase
 			end
 			File.open(EPAFI_IGNORE_FILE, 'w') do |f| 
 				f.write @ignore_list.to_a.to_yaml 
 			end
 		end
 		def include? mail
 			return (
 				(@ignore_list.include? mail.strip.downcase) or 
 				(@keep_list.include? mail.strip.downcase)
 			)
 		end
 	end
 	class CrawlerApp
 		attr_reader :imap
 		attr_reader :contacts
 		TMPMAIL_FILE = '.tmpmail'
 		def initialize config
    	@saved_key = 'RFC822'
    	@filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase
 			@config = config
 			@imap = nil
 			@contact_manager = ContactManager.new config
 		end
 		def connect!
    	@imap = Net::IMAP.new(
 				@config[:imap][:server], 
 				ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE},
 				port: 993
 			)
    	@imap.login(@config[:imap][:login], @config[:imap][:password])
      #@imap.select(SOURCE_MAILBOX)
 		end
 		def disconnect!
    	imap.logout
    	imap.disconnect
 		end
 		MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/
 		def examine_message message
      m = Mail.read_from_string message.attr[@saved_key]
 			return if m.from.nil?
 			return if m.to.nil?
 			emails = Set.new
      begin
 			  emails.merge m.from
 			  emails.merge [m.to].flatten if m.to
 			  emails.merge [m.cc].flatten if m.cc
      rescue => e
        binding.pry
      end
 			body_emails = Set.new
 			m.body.parts.each do |part|
 				next if part.content_type != 'text/plain'
 				#body_emails = m.body.decoded.scan MAIL_REGEXP
 				part_emails = part.decoded.scan MAIL_REGEXP
 				#pp body_emails
 				if not part_emails.empty? then
 					body_emails.merge part_emails
 				end
 			end
 			emails.merge body_emails
 			# puts emails.to_a.join(' , ')
 			remaining_emails = (
 				emails
 				.map{ |e| [e, (@contact_manager.include? e)] }
 				.select{ |e,t| !t }
 			)
 			seen_emails = (
 				remaining_emails
 				.empty? 
 			)
 			# puts @contacts.to_a.join(', ')
 			if seen_emails then
 				print "."
 				return
 			else
 				puts ""
 				all_addr = { 
 					from: (m.from || []),
 					to: (m.to || []),
 					cc: (m.cc || []),
 					body: (body_emails || [])
 				}
 				all_addr.each do |key, list|
 					list.each do |addr|
 						addr_str = if remaining_emails.map{|e,t| e}.include? addr then
 								   	     addr.yellow.on_black
 							   	     else addr
 							   	     end
 						str = "%4s: %s" % [key.to_s.upcase, addr_str]
 						puts str
 					end
 				end
 				puts ""
 				#puts " ORIGINAL EMAILS: #{emails.to_a.join(', ')}"
 				#puts "REMAINING EMAILS: #{remaining_emails.map{|e,t| e}.join(', ')}".yellow.on_black
 				#puts "     SEEN EMAILS: #{seen_emails}"
 			end
 			while true
 				begin
 					puts "\n### #{m.subject}"
 					print "#{m.from.join(',')} --> #{m.to.join(',')} "
 					puts "[Ignore/Add/Skip/Detail] ?"
 					i = STDIN.gets 
 					case i.strip
 					when /^[iI]$/ then # ignore
 						@contact_manager.ignore_contact remaining_emails.map{|e,t| e}
 						break
 					when /^[aA]$/ then # add
 						@contact_manager.keep_contact remaining_emails.map{|e,t| e}
 						break
 					when /^[sS]$/ then #skip
 						break
 					when /^[dD]$/ then # decode
 						# puts m.body.decoded
 						File.open(TMPMAIL_FILE + ".2", 'w') do |f| 
 							f.write message.attr[@saved_key]
 						end
 						system "formail < #{TMPMAIL_FILE}.2 > #{TMPMAIL_FILE}"
 						system "mutt -R -f #{TMPMAIL_FILE}"
 					end
 				rescue Encoding::ConverterNotFoundError
 					STDERR.puts "ERROR: encoding problem in email. Unable to convert."
 				end
 			end
 			return
 		end
 		def examine_all
    	@imap.list('', '*').each do |mailbox|
 				puts "\nMAILBOX #{mailbox.name}".yellow
 				next unless mailbox.name =~ /#{@config[:imap][:pattern]}/
      	@imap.examine mailbox.name
        puts "Searching #{mailbox.name}"
      	messages_in_mailbox = @imap.responses['EXISTS'][0]
      	if not messages_in_mailbox then
        	say "#{mailbox.name} does not have any messages"
 					next
 				end
        @imap.select mailbox.name #GYR: TEST
        ids = @imap.search('SINCE 1-Jan-2001')
 				# NOT OR TO "@agilefant.org" CC "@agilefant.org"')
        if ids.empty?
          puts "\tFound no messages"
 				else
 					examine_message_list mailbox.name, ids
 				end
    	end
 		end
 		def examine_message_list mailbox_name, ids
      ids.each do |id|
        @imap.select mailbox_name #GYR: TEST
 				message = imap.fetch(id, [@saved_key])[0]
 				examine_message message
      end 
 		rescue IOError
 			# re-connect and try again
 			connect!
 			retry
 		end
 	end
 	class Crawler < Thor
 		CONFIG_FILE = 'config/secrey.yml'
  	include Thor::Actions
  	default_task :crawl
  	desc 'crawl', 'Crawls email to save mails'
  	def crawl
    	#saved_info = []
 			parse_configuration
 			## Run application
 			app = CrawlerApp.new @config
 			app.connect!
 			app.examine_all
 			#pp saved_info
 			app.disconnect!
  	end
 		def initialize *args
 			@config = {}
 			super
 		end
 		private
 		def parse_configuration
 			## Load configuration
 			@config.merge! Hash.transform_keys_to_symbols(
 				YAML::load( File.open( EPAFI_CONFIG_FILE ) )
 			)
 			## Validate configuration structure 
 			validations = {
 				crm: {
 					baseurl: lambda { |url| url =~ URI::regexp },
 					login: 'string',
 					password: 'string'
 				},
 				imap: {
 					server: 'string',
 					login: 'string',
 					password: 'string'
 				}
 			}
 			validator = HashValidator.validate(@config, validations)
 			raise "Configuration is not valid: #{validator.errors.inspect}" unless validator.valid?
 		end
 	end
 end
 Epafh::Crawler.start
--- a/epafh.gemspec
+++ b/epafh.gemspec
@ -1,27 +1,19 @@
 # coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'epafh/version'
+require 'epafh/constants'
 Gem::Specification.new do |spec|
  spec.name          = "epafh"
  spec.version       = Epafh::VERSION
-  spec.authors       = ["@@@No user configured@@@"]
+  spec.authors       = ["Glenn Y. Rolland"]
-  spec.email         = ["@@@No user configured@@@"]
+  spec.email         = ["glenux@glenux.net"]
-  spec.summary       = %q{TODO: Write a short summary, because Rubygems requires one.}
+  spec.summary       = %q{A handy tool to extract emails and URLs from an IMAP account.}
-  spec.description   = %q{TODO: Write a longer description or delete this line.}
+  spec.description   = %q{A handy tool to extract emails and URLs from an IMAP account.}
-  spec.homepage      = "TODO: Put your gem's website or public repo URL here."
+  spec.homepage      = "https://github.com/glenux/epafh"
  spec.license       = "LGPL-3"
  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
  # delete this section to allow pushing this gem to any host.
  if spec.respond_to?(:metadata)
    spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
  else
    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
  end
  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
  spec.bindir        = "bin"
  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@ -37,4 +29,7 @@ Gem::Specification.new do |spec|
  spec.add_runtime_dependency "hash_validator"
  spec.add_runtime_dependency "pry"
  spec.add_runtime_dependency "pry-rescue"
  spec.add_runtime_dependency "highline"
  spec.add_runtime_dependency "ruby-progressbar"
 end
--- a/lib/epafh.rb
+++ b/lib/epafh.rb
@ -1,5 +1,10 @@
 require "epafh/version"
 module Epafh
  # Your code goes here...
 end
 require "epafh/hash"
 require "epafh/constants"
 require "epafh/contact_manager"
 require "epafh/crawler"
 require "epafh/app"
--- a/lib/epafh/app.rb
+++ b/lib/epafh/app.rb
@ -0,0 +1,120 @@
 require 'highline'
 class Epafh::App < Thor
  class InvalidConfiguration < RuntimeError ; end
 	CONFIG_FILE = 'config/secrey.yml'
 	CONFIG_DEFAULT = { 
    'imap' => {
      'server'   => '',
      'login'    => '',
      'password' => ''
    },
    'crm' => {
 			'baseurl'  => '',
 			'login'    => '',
 			'password' => ''
    } 
  }
  include Thor::Actions
  default_task :crawl
  desc 'config', 'Initialize configuration'
  def config
    puts "Welcome to Epafh !".green
    config = CONFIG_DEFAULT
 	  if File.exist? Epafh::EPAFI_CONFIG_FILE then
 	    config =  config.merge(YAML::load( File.open( Epafh::EPAFI_CONFIG_FILE ) ) || {})
 	  end
 	  imap_params = {
 	    server:   {desc: 'IMAP hostname ? ' },
 	    login:    {desc: 'IMAP username ? ' },
 	    password: {desc: 'IMAP password ? ', hidden: true}
 	  }
 	  crm_params = {
 	    baseurl:  {desc: 'CRM base url ? ' },
 	    login:    {desc: 'CRM username ? ' },
 	    password: {desc: 'CRM password ? ', hidden: true}
 	  }
 	  config['imap'] = config_with_rules config['imap'], imap_params
 	  config['crm'] = config_with_rules config['crm'], crm_params
    FileUtils.mkdir_p(Epafh::EPAFI_CONFIG_DIR)
    File.open(Epafh::EPAFI_CONFIG_FILE, 'w'){|f| f.write(config.to_yaml)}
  end
  desc 'crawl', 'Crawls email to save mails'
  def crawl
 		parse_configuration
 		## Run application
 		app = Epafh::Crawler.new @config
 		app.connect!
 		app.examine_all
 		app.disconnect!
  end
 	def initialize *args
 		@config = {}
 		super
 	end
 	private
 	# Ask parameters, with rule constraints
 	#
 	def config_with_rules cfg_in, rules
    cli = ::HighLine.new
 	  cfg_out = cfg_in.clone
 	  # Loop parameter rules
    rules.each.map {|param,values| [param.to_s,values] }
    .each do |param, values| 
      backup = cfg_out[param]
      backup_hidden = cfg_out[param].gsub(/./,'*')
      cfg_out[param] = cli.ask(values[:desc]) do |q| 
        # Disable echo if hidden enabled
        q.echo = '*' if values[:hidden]
        # Replace default value by stars if hidden
        if not cfg_out[param].empty? then
          q.default = 
            if (values[:hidden]) then backup_hidden
            else cfg_out[param]
            end
        end
      end
      # When RETURN is pressed, Highline uses default (starred)
      # We have to replace it with the real value
      if values[:hidden] and cfg_out[param] == backup_hidden then
        cfg_out[param] = backup
      end
    end
    cfg_out
 	end
 	def parse_configuration
 		## Load configuration
 		@config.merge! Hash.transform_keys_to_symbols(
 			YAML::load( File.open( Epafh::EPAFI_CONFIG_FILE ) )
 		)
 		## Validate configuration structure 
 		validations = {
 			crm: {
 				baseurl: lambda { |url| url =~ URI::regexp },
 				login: 'string',
 				password: 'string'
 			},
 			imap: {
 				server: 'string',
 				login: 'string',
 				password: 'string'
 			}
 		}
 		validator = HashValidator.validate(@config, validations)
 		raise InvalidConfiguration, "Configuration is not valid: #{validator.errors.inspect}" unless validator.valid?
 	end
 end
--- a/lib/epafh/constants.rb
+++ b/lib/epafh/constants.rb
@ -0,0 +1,7 @@
 module Epafh
  VERSION = "0.1.0"
 	EPAFI_CONFIG_DIR = File.join(ENV['HOME'], '.epafh')
 	EPAFI_CONFIG_FILE = File.join(EPAFI_CONFIG_DIR, 'config.yml')
 	EPAFI_IGNORE_FILE = File.join(ENV['HOME'], '.epafh', 'ignore.yml')
 end
--- a/lib/epafh/contact_manager.rb
+++ b/lib/epafh/contact_manager.rb
@ -0,0 +1,106 @@
 class Epafh::ContactManager
 	CRM_LOGIN_URL = '/login'
 	CRM_LEADS_URL = '/leads.json'
 	CRM_CONTACTS_URL = '/contacts.json'
 	def initialize config
 		@config = config
 		@browser = Mechanize.new { |agent|
 			agent.user_agent_alias = 'Mac Safari'
 		}
 		@ignore_list = Set.new
 		@keep_list = Set.new
 		## Load configuration file
 		#
 		unless File.exist? Epafh::EPAFI_CONFIG_FILE then
 			raise "Unable to find configuration file #{Epafh::EPAFI_CONFIG_FILE}" 
 		end
 		@config = config
 		connect!
 		load_contacts
 		load_leads
 		load_ignore
 		#puts @keep_list.to_a
 	rescue RuntimeError => e
 		STDERR.puts e.message
 	end
 	def connect!
 		@browser.get(@config[:crm][:baseurl] + CRM_LOGIN_URL) do |page|
 			page.form_with(action: '/authentication') do |f|
 				f['authentication[username]'] = @config[:crm][:login]
 				f['authentication[password]'] = @config[:crm][:password]
 			end.click_button
 		end
 	rescue Mechanize::ResponseCodeError
 		raise "Authentication error. Verify your credentials." 
 	end
 	def load_ignore
 		if File.exist? Epafh::EPAFI_IGNORE_FILE
 			ignore_list = YAML.load_file(Epafh::EPAFI_IGNORE_FILE)
 			ignore_list.each do |email|
 				@ignore_list << email.strip.downcase
 			end
 		end
 	end
 	def load_leads page=1
 		crm_leads_page = @browser.get(@config[:crm][:baseurl] + CRM_LEADS_URL + "?page=#{page}")
 		crm_leads = JSON.parse crm_leads_page.body
 		crm_leads.each do |lead_obj|
 			keep_contact lead_obj['lead']['email'].split(',')
 			keep_contact lead_obj['lead']['alt_email'].split(',')
 		end
 		if crm_leads.size > 0 then
 			load_leads (page + 1)
 		end
 	end
 	def load_contacts page=1
 		crm_contacts_page = @browser.get(@config[:crm][:baseurl] + CRM_CONTACTS_URL + "?page=#{page}")
 		crm_contacts = JSON.parse crm_contacts_page.body
 		crm_contacts.each do |contact_obj|
 			keep_contact contact_obj['contact']['email'].split(',')
 	 		keep_contact contact_obj['contact']['alt_email'].split(',')
 		end
 		if crm_contacts.size > 0 then
 			load_contacts (page + 1)
 		end
 		#contacts.to_a.sort.join(', ')
 	end
 	def keep_contact emails
 		emails = emails.to_a if emails.is_a? Set
 	 	[emails].flatten.each do |mail|
 			@keep_list << mail.strip.downcase
 		end
 	end
 	def ignore_contact emails
 		emails = emails.to_a if emails.is_a? Set
 	 	[emails].flatten.each do |mail|
 			@ignore_list << mail.strip.downcase
 		end
 		File.open(Epafh::EPAFI_IGNORE_FILE, 'w') do |f| 
 			f.write @ignore_list.to_a.to_yaml 
 		end
 	end
 	def include? mail
 		return (
 			(@ignore_list.include? mail.strip.downcase) or 
 			(@keep_list.include? mail.strip.downcase)
 		)
 	end
 end
--- a/lib/epafh/crawler.rb
+++ b/lib/epafh/crawler.rb
@ -0,0 +1,160 @@
 require 'ruby-progressbar'
 class Epafh::Crawler
 	attr_reader :imap
 	attr_reader :contacts
 	TMPMAIL_FILE = '.tmpmail'
 	MAIL_REGEXP = /\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,4}\b/
 	def initialize config
    @saved_key = 'RFC822'
    @filter_headers = 'BODY[HEADER.FIELDS (FROM TO Subject)]'.upcase
 		@config = config
 		@imap = nil
 		@contact_manager = Epafh::ContactManager.new config
 	end
 	def connect!
    @imap = Net::IMAP.new(
 			@config[:imap][:server], 
 			ssl: {verify_mode: OpenSSL::SSL::VERIFY_NONE},
 			port: 993
 		)
    @imap.login(@config[:imap][:login], @config[:imap][:password])
    #@imap.select(SOURCE_MAILBOX)
 	end
 	def disconnect!
    imap.logout
    imap.disconnect
 	end
 	def examine_message message
    m = Mail.read_from_string message.attr[@saved_key]
 		return if m.from.nil?
 		return if m.to.nil?
 		body_emails = extract_body_mail m.body.parts
    ## Create association between extracted addreses and email part
 		mail_struct = { 
 			from: [m.from || []].flatten.reject{|e| e.nil?},
 			to:   [m.to || []].flatten.reject{|e| e.nil?},
 			cc:   [m.cc || []].flatten.reject{|e| e.nil?},
 			body: (body_emails.to_a || []).reject{|e| e.nil?}
 		}
 		#pp m
 		#pp mail_struct
 		emails = Set.new
 		mail_struct.each {|key, val|  emails.merge val }
 		remaining_emails = emails.reject{|e| @contact_manager.include?(e) }
 		# Skip examination of no addresses are remaining
 		if remaining_emails.empty? then
 			return
 		end
 		display_header mail_struct, remaining_emails
 		while true
 			begin
 				puts "\n### #{m.subject}"
 				print "#{mail_struct[:from].join(',')} --> #{mail_struct[:to].join(',')} "
 				puts "[Ignore/Add/Skip/Detail] ?"
 				i = STDIN.gets 
 				case i.strip
 				when /^[iI]$/ then # ignore
 					@contact_manager.ignore_contact remaining_emails
 					break
 				when /^[aA]$/ then # add
 					@contact_manager.keep_contact remaining_emails
 					break
 				when /^[sS]$/ then #skip
 					break
 				when /^[dD]$/ then # decode
 					# puts m.body.decoded
 					File.open(TMPMAIL_FILE + ".2", 'w') do |f| 
 						f.write message.attr[@saved_key]
 					end
 					system "formail < #{TMPMAIL_FILE}.2 > #{TMPMAIL_FILE}"
 					system "mutt -R -f #{TMPMAIL_FILE}"
 				end
 			rescue Encoding::ConverterNotFoundError
 				STDERR.puts "ERROR: encoding problem in email. Unable to convert."
 			end
 		end
 		return
 	end
 	def examine_all
    @imap.list('', '*').each do |mailbox|
 			puts "\nMAILBOX #{mailbox.name}".yellow
 			next unless mailbox.name =~ /#{@config[:imap][:pattern]}/
      @imap.examine mailbox.name
      puts "Searching #{mailbox.name}"
      messages_in_mailbox = @imap.responses['EXISTS'][0]
      if not messages_in_mailbox then
        say "#{mailbox.name} does not have any messages"
 				next
      end
      @imap.select mailbox.name #GYR: TEST
      ids = @imap.search('SINCE 1-Jan-2001')
 			# NOT OR TO "@agilefant.org" CC "@agilefant.org"')
      if ids.empty?
        puts "\tFound no messages"
 			else
 				examine_message_list mailbox.name, ids
      end
    end
 	end
 	def examine_message_list mailbox_name, ids
 	  progressbar = ProgressBar.create(:total => ids.size)
    ids.each do |id|
      @imap.select mailbox_name #GYR: TEST
 			message = imap.fetch(id, [@saved_key])[0]
 			examine_message message
 			progressbar.increment
    end 
 	rescue IOError
 		# re-connect and try again
 		connect!
 		retry
 	end
  def extract_body_mail body_parts
 		body_emails = Set.new
 		body_parts.each do |part|
 			next if part.content_type != 'text/plain'
 			part_emails = part.decoded.scan MAIL_REGEXP
 			if not part_emails.empty? then
 				body_emails.merge part_emails
 			end
 		end
 		body_emails
  end
  def display_header header_struct, remaining_emails
 		puts ""
 		header_struct.each do |key, list|
 			#pp list
 			list.each do |addr|
 				addr_str = if remaining_emails.include? addr then
 								   	 addr.yellow.on_black
 							   	 else addr
 							   	 end
 				str = "%4s: %s" % [key.to_s.upcase, addr_str]
 				puts str
 			end
 		end
 		puts ""
  end
 end
--- a/lib/epafh/hash.rb
+++ b/lib/epafh/hash.rb
@ -0,0 +1,11 @@
 class Hash
  #take keys of hash and transform those to a symbols
  def self.transform_keys_to_symbols(value)
    return value if not value.is_a?(Hash)
    hash = value.inject({}) do |memo,(k,v)| 
 			memo[k.to_sym] = Hash.transform_keys_to_symbols(v); memo
 		end
    return hash
  end
 end
--- a/lib/epafh/version.rb
+++ b/lib/epafh/version.rb
@ -1,3 +0,0 @@
 module Epafh
  VERSION = "0.1.0"
 end
Author	SHA1	Message	Date
Glenn Y. Rolland	ee6e689157	Added comment to TODO.md	2015-11-30 21:23:34 +01:00
Glenn Y. Rolland	b53cb336f7	Add a progressbar !	2015-11-30 01:00:52 +01:00
Glenn Y. Rolland	1460435676	Hide debug.	2015-11-29 21:14:02 +01:00
Glenn Y. Rolland	6a7ad7733d	Make sure bad values are filtered out.	2015-11-29 11:35:46 +01:00
Glenn Y. Rolland	f07a23833f	Cleanup & restructure Crawler class.	2015-11-28 22:26:55 +01:00
Glenn Y. Rolland	7bfc3beb2a	Remove default value for baseurl.	2015-11-28 12:29:41 +01:00
Glenn Y. Rolland	b2f55c89f0	Move ask procedure out of config function.	2015-11-28 12:28:10 +01:00
Glenn Y. Rolland	8669fdaa83	Improve configuration step for password.	2015-11-28 11:55:19 +01:00
Glenn Y. Rolland	8654b94cae	Handle interrupts.	2015-11-28 09:48:10 +01:00
Glenn Y. Rolland	5bb262dc92	Added config option to cli command.	2015-11-27 09:09:05 +01:00
Glenn Y. Rolland	08c63bb7fd	Split into multiple lib/ files.	2015-11-26 22:35:20 +01:00