diff options
author | 2015-02-23 15:21:27 +0100 | |
---|---|---|
committer | 2015-02-23 15:21:27 +0100 | |
commit | 2a13f18aa0a7ac3fe7d19eeea45842de818a615c (patch) | |
tree | c67b09ac642018550f2ca203851c87dc3ccadc33 /lib | |
parent | Implement --delete (diff) | |
download | backend-2a13f18aa0a7ac3fe7d19eeea45842de818a615c.tar.gz backend-2a13f18aa0a7ac3fe7d19eeea45842de818a615c.tar.bz2 backend-2a13f18aa0a7ac3fe7d19eeea45842de818a615c.zip |
use more threads!
Diffstat (limited to 'lib')
-rw-r--r-- | lib/storage.rb | 10 | ||||
-rw-r--r-- | lib/threading.rb | 4 |
2 files changed, 8 insertions, 6 deletions
diff --git a/lib/storage.rb b/lib/storage.rb index f255633..d32ba2b 100644 --- a/lib/storage.rb +++ b/lib/storage.rb @@ -7,7 +7,7 @@ module Ag::Storage begin $es.indices.delete index: 'ml-' + list rescue Elasticsearch::Transport::Transport::Errors::NotFound => e - $stderr.puts "Index did not exist yet. Creating." + $stderr.puts "Index did not exist yet. Creating." if $options.debug end $es.indices.create( @@ -84,10 +84,10 @@ module Ag::Storage content = Ag::Utils.fix_encoding(raw_content || '', true).strip if content == '' - $stderr.puts "#{message.message_id}: Content empty?" + $stderr.puts "#{message.message_id}: Content empty?" if $options.debug end rescue => e - $stderr.puts "Cannot render message #{message.message_id} (file: #{filename}): #{e}" + $stderr.puts "Cannot render message #{message.message_id} (file: #{filename}): #{e}" if $options.debug end content @@ -174,7 +174,7 @@ module Ag::Storage ) end - def fix_threading(list) + def fix_threading(list, pass) result = $es.search( index: 'ml-' + list, size: 100000, @@ -201,7 +201,7 @@ module Ag::Storage } ) - result['hits']['hits'].each do |hit| + Parallel.each(result['hits']['hits'], progress: "Calculating Threading (Pass #{pass})") do |hit| msg = resolve_message_id(list, hit['_source']['raw_parent']) unless msg == nil diff --git a/lib/threading.rb b/lib/threading.rb index 8988f23..212bb98 100644 --- a/lib/threading.rb +++ b/lib/threading.rb @@ -57,11 +57,13 @@ module Ag def calc(list) number_of_root_threads = -1 + pass = 1 loop do - new_num = Ag::Storage.fix_threading(list) + new_num = Ag::Storage.fix_threading(list, pass) break if new_num == number_of_root_threads number_of_root_threads = new_num + pass += 1 end end end |