From 7e41d12fbeaaa6d62b926e9a53e0f720eceaa5e6 Mon Sep 17 00:00:00 2001 From: Peter Wilmott Date: Sun, 12 Apr 2015 15:34:51 +0100 Subject: When building a package also build it with the next target if possible --- bin/tinder.sh | 24 +++++++++++ web/Rakefile | 27 ++++++++---- web/app.rb | 1 - web/lib/ci.rb | 119 +++++++++++++++++++++++++++++++++++++++++---------- web/lib/packages.rb | 10 ++++- web/lib/repoman.rb | 92 --------------------------------------- web/views/layout.erb | 6 +-- 7 files changed, 151 insertions(+), 128 deletions(-) delete mode 100644 web/lib/repoman.rb diff --git a/bin/tinder.sh b/bin/tinder.sh index 4116fa8..6d246f4 100755 --- a/bin/tinder.sh +++ b/bin/tinder.sh @@ -26,6 +26,20 @@ function SETUP () { rm "/usr/portage/packages/$PACKAGE.tbz2" fi + if [[ $TYPE == "next_target" ]]; then + mkdir -p "$SCRIPT_DIR/overlay" + mkdir -p "$SCRIPT_DIR/overlay/$CATEGORY/$NAME" + cp "/usr/portage/$CATEGORY/$NAME/$NAME-$VERSION.ebuild" "$SCRIPT_DIR/overlay/$CATEGORY/$NAME" + cp "/usr/portage/$CATEGORY/$NAME/metadata.xml" "$SCRIPT_DIR/overlay/$CATEGORY/$NAME" + cp -r "/usr/portage/$CATEGORY/$NAME/files" "$SCRIPT_DIR/overlay/$CATEGORY/$NAME" || true + + cd "$SCRIPT_DIR/overlay/$CATEGORY/$NAME" + sed -i -e "/^USE_RUBY/s/$CURR_TARGET/$CURR_TARGET $NEXT_TARGET/" "$NAME-$VERSION.ebuild" + repoman manifest + repoman full + cd - + fi + set +e emerge --pretend --quiet "=$PACKAGE" if [[ $? == 1 ]]; then @@ -75,6 +89,10 @@ function CLEANUP() { rm -r /var/tmp/portage/* || true emerge --depclean --quiet echo -e "$PACKAGE : $RESULT" + + if [[ $TYPE == "next_target" ]]; then + rm -r "$SCRIPT_DIR/overlay" + fi } ENV_SETUP @@ -95,6 +113,12 @@ if [[ $# -eq 1 ]]; then EMERGE CLEANUP elif [[ $# -eq 3 ]]; then + TYPE="current_target" + PACKAGE=$1 + SETUP + EMERGE + CLEANUP + TYPE="next_target" PACKAGE=$1 CURR_TARGET=$2 diff --git a/web/Rakefile b/web/Rakefile index 27525e0..e2d5b44 100644 --- a/web/Rakefile +++ b/web/Rakefile @@ -20,14 +20,19 @@ namespace :db do update_packages end + desc 'Clear the packages database' + task :clear_packages do + clear_build + end + desc 'Update the build database with logfiles from ci-logs/' - task :update_ci do - update_ci + task :update_build do + update_build end desc 'Clear the build database' - task :clear_ci do - clear_ci + task :clear_build do + clear_build end desc 'Update the repoman database with logfiles from repo-logs/' @@ -65,24 +70,28 @@ namespace :docker do @volume_container.start @volume_container.wait(36_000) - docker_path = File.dirname(File.expand_path(File.dirname(__FILE__))) - @ci_image = Docker::Image.build_from_dir(docker_path) + if ENV['CI_IMAGE'].nil? + docker_path = File.dirname(File.expand_path(File.dirname(__FILE__))) + @ci_image = Docker::Image.build_from_dir(docker_path) + else + @ci_image = Docker::Image.get(ENV['CI_IMAGE']) + end end desc 'Remove a previously built docker image' task :teardown do - @ci_image.delete + @ci_image.delete if ENV['CI_IMAGE'].nil? @volume_container.delete(v: 1) @volume_image.delete end desc 'Build test packages ( NUM_OF_PACKAGES={5,all,untested} )' task :run_build do - run_ci(@volume_container, @ci_image, num_of_packages) + run_ci(@volume_container, @ci_image, 'build', num_of_packages) end desc 'QA test packages ( NUM_OF_PACKAGES={5,all} )' task :run_repoman do - run_repoman(@ci_image, num_of_packages) + run_ci(@volume_container, @ci_image, 'repoman', num_of_packages) end end diff --git a/web/app.rb b/web/app.rb index 661d0dd..1aa3ce8 100644 --- a/web/app.rb +++ b/web/app.rb @@ -12,7 +12,6 @@ require_relative 'lib/ci' require_relative 'lib/helpers' require_relative 'lib/models' require_relative 'lib/packages' -require_relative 'lib/repoman' class RubyTinderbox < Sinatra::Base get '/' do diff --git a/web/lib/ci.rb b/web/lib/ci.rb index 13e7ac0..bbe0b23 100644 --- a/web/lib/ci.rb +++ b/web/lib/ci.rb @@ -1,4 +1,39 @@ -def run_ci(volume_container, ci_image, num_of_packages) +def run_ci(volume_container, ci_image, ci_type, num_of_packages) + packages = generate_package_list(ci_type, num_of_packages) + + packages.peach(8) do |package| + package = package.split(' ') + identifier = package[0] + current_target = package[1] + next_target = package[2] + + if ci_type == 'build' + cmd = %W[/ruby-tinderbox/tinder.sh #{identifier} #{current_target} #{next_target}] + elsif ci_type == 'repoman' + cmd = %W[/ruby-tinderbox/repoman.sh #{identifier} #{current_target} #{next_target}] + end + ci_container = Docker::Container.create( + Cmd: cmd, + Image: ci_image.id + ) + ci_container.start(VolumesFrom: volume_container.id) + ci_container.wait(36_000) + + tar = Tempfile.new('tar') + File.open(tar, 'w') do |file| + ci_container.copy('/ruby-tinderbox/ci-logs') do |chunk| + file.write(chunk) + end + end + Archive::Tar::Minitar.unpack(tar, File.dirname(File.expand_path(File.dirname(__FILE__)))) + tar.close + tar.unlink + + ci_container.delete + end +end + +def generate_package_list(ci_type, num_of_packages) packages = [] Package.each do |package| packages << package[:identifier] @@ -6,7 +41,9 @@ def run_ci(volume_container, ci_image, num_of_packages) if num_of_packages == 'all' packages = packages - elsif num_of_packages == 'untested' + elsif num_of_packages == 'untested' and ci_type == 'repoman' + packages = packages + elsif num_of_packages == 'untested' and ci_type == 'build' packages = [] Package.each do |package| next if package.build.count > 0 @@ -30,33 +67,33 @@ def run_ci(volume_container, ci_image, num_of_packages) packages = packages.sample(num_of_packages) else puts 'ERROR: Invalid value for NUM_OF_PACKAGES' + puts ci_type + puts num_of_packages exit end - packages = packages.uniq - packages.each do |package| - ci_container = Docker::Container.create( - Cmd: %W[/ruby-tinderbox/tinder.sh #{package}], - Image: ci_image.id - ) - ci_container.start(VolumesFrom: volume_container.id) - ci_container.wait(36_000) + packages_with_targets = [] + packages.uniq.each do |package| + package = Package.where(identifier: package).first - tar = Tempfile.new('tar') - File.open(tar, 'w') do |file| - ci_container.copy('/ruby-tinderbox/ci-logs') do |chunk| - file.write(chunk) - end - end - Archive::Tar::Minitar.unpack(tar, File.dirname(File.expand_path(File.dirname(__FILE__)))) - tar.close - tar.unlink + target = 'unknown' + target = package[:r19_target] unless package[:r19_target] == 'nil' + target = package[:r20_target] unless package[:r20_target] == 'nil' + target = package[:r21_target] unless package[:r21_target] == 'nil' + target = package[:r22_target] unless package[:r22_target] == 'nil' - ci_container.delete + next_target = 'unknown' + next_target = 'ruby20' if target == 'ruby19' + next_target = 'ruby21' if target == 'ruby20' + next_target = 'ruby22' if target == 'ruby21' + + packages_with_targets << "#{package[:identifier]} #{target} #{next_target}" end + + packages_with_targets end -def update_ci +def update_build Dir.glob('ci-logs/*/*/builds/*') do |build| begin build_array = build.split('/') @@ -91,6 +128,44 @@ def update_ci end end -def clear_ci +def update_repoman + Dir.glob('ci-logs/*/*/repomans/*') do |repoman| + begin + repoman_array = repoman.split('/') + sha1 = repoman_array[1] + timestamp = repoman_array[4] + target = repoman_array[2].sub('_target', '') + + log = File.read("#{repoman}/repoman_log") + + result = 'unknown' + if log.include?('If everyone were like you, I\'d be out of business!') + result = 'passed' + elsif log.include?('You\'re only giving me a partial QA payment?') + result = 'partial' + elsif log.include?('Make your QA payment on time and you\'ll never see the likes of me.') + result = 'failed' + end + + Package.where(sha1: sha1).first.add_repoman( + Repoman.find_or_create( + timestamp: timestamp, + target: target, + result: result, + log: log + ) + ) + rescue => e + puts "ERROR: #{e}" + next + end + end +end + +def clear_build Build.map(&:delete) end + +def clear_repoman + Repoman.map(&:delete) +end diff --git a/web/lib/packages.rb b/web/lib/packages.rb index eb62883..1d4b7ea 100644 --- a/web/lib/packages.rb +++ b/web/lib/packages.rb @@ -1,5 +1,5 @@ def update_packages - packages_txt = `python3 lib/packages.py` + packages_txt = `python3 lib/packages.py`.uniq packages_txt.lines.peach do |line| category, name, version, revision, slot, amd64_keyword, r19_target, r20_target, r21_target, r22_target = line.split(' ') identifier = category + '/' + name + '-' + version + (revision == 'r0' ? '' : "-#{revision}") @@ -39,3 +39,11 @@ def update_packages Package.dataset.update(update_timestamp: update_timestamp) Package.dataset.update(portage_timestamp: portage_timestamp) end + +def clear_packages + Package.each do |package| + package.build.map(&:delete) + package.repoman.map(&:delete) + package.delete + end +end diff --git a/web/lib/repoman.rb b/web/lib/repoman.rb deleted file mode 100644 index 1a44177..0000000 --- a/web/lib/repoman.rb +++ /dev/null @@ -1,92 +0,0 @@ -def run_repoman(ci_image, num_of_packages) - packages = [] - Package.each do |package| - target = '' - target = package[:r19_target] unless package[:r19_target] == 'nil' - target = package[:r20_target] unless package[:r20_target] == 'nil' - target = package[:r21_target] unless package[:r21_target] == 'nil' - target = package[:r22_target] unless package[:r22_target] == 'nil' - next if target.empty? - - next_target = '' - next_target = 'ruby20' if target == 'ruby19' - next_target = 'ruby21' if target == 'ruby20' - next_target = 'ruby22' if target == 'ruby21' - next if next_target.empty? - - packages << "#{package[:identifier]} #{target} #{next_target}" - end - - if num_of_packages == 'all' - packages = packages - elsif num_of_packages.is_a?(Integer) - packages = packages.sample(num_of_packages) - else - puts 'ERROR: Invalid value for NUM_OF_PACKAGES' - exit - end - - packages = packages.uniq - packages.peach(8) do |package| - package = package.split(' ') - identifier = package[0] - current_target = package[1] - next_target = package[2] - ci_container = Docker::Container.create( - Cmd: %W[/ruby-tinderbox/repoman.sh #{identifier} #{current_target} #{next_target}], - Image: ci_image.id - ) - ci_container.start - ci_container.wait(36_000) - - tar = Tempfile.new('tar') - File.open(tar, 'w') do |file| - ci_container.copy('/ruby-tinderbox/ci-logs') do |chunk| - file.write(chunk) - end - end - Archive::Tar::Minitar.unpack(tar, File.dirname(File.expand_path(File.dirname(__FILE__)))) - tar.close - tar.unlink - - ci_container.delete - end -end - -def update_repoman - Dir.glob('ci-logs/*/*/repomans/*') do |repoman| - begin - repoman_array = repoman.split('/') - sha1 = repoman_array[1] - timestamp = repoman_array[4] - target = repoman_array[2].sub('_target', '') - - log = File.read("#{repoman}/repoman_log") - - result = 'unknown' - if log.include?('If everyone were like you, I\'d be out of business!') - result = 'passed' - elsif log.include?('You\'re only giving me a partial QA payment?') - result = 'partial' - elsif log.include?('Make your QA payment on time and you\'ll never see the likes of me.') - result = 'failed' - end - - Package.where(sha1: sha1).first.add_repoman( - Repoman.find_or_create( - timestamp: timestamp, - target: target, - result: result, - log: log - ) - ) - rescue => e - puts "ERROR: #{e}" - next - end - end -end - -def clear_repoman - Repoman.map(&:delete) -end diff --git a/web/views/layout.erb b/web/views/layout.erb index 9d564b3..0e961a8 100644 --- a/web/views/layout.erb +++ b/web/views/layout.erb @@ -5,9 +5,9 @@ - - - + + + -- cgit v1.2.3-65-gdbad