import sys import protocol import socket import pickle from time import sleep import os import subprocess from subprocess import CalledProcessError import re import string from string import atoi import portage from traceback import print_exc, format_exc from util import WritableObject, flatten_deps from common.exceptions import ChrootPreparationException import config from logger import log, init_logging class Tinderbox(object): def __init__(self): self.hostname = config.MATCHBOX_HOST self.port = config.MATCHBOX_PORT self.sock = None self.settings = portage.config(clone=portage.settings) self.trees = portage.create_trees() self.settings["PORTAGE_VERBOSE"]="1" self.settings.backup_changes("PORTAGE_VERBOSE") def start_tinderbox(self): self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # TODO load settings for contacting matchbox self.sock.connect((self.hostname,self.port)) while 1: # TODO error/exception checking msg = protocol.GetNextPackage() msg_pickled = pickle.dumps(msg) self.sock.sendall(msg_pickled) reply = self.sock.recv(1024) reply_unpickled = pickle.loads(reply) if type(reply_unpickled) is protocol.GetNextPackageReply: gnp = reply_unpickled print "going to compile: %s\nuse flags: %s" %\ (gnp.package_name,gnp.use_flags) package = Package(gnp.package_name, gnp.version, gnp.use_flags) sleep(5) self.emerge_package(package) else: print "Unknown reply: %s" % reply_unpickled def emerge_package(self, package): settings = self.settings porttree = self.trees[portage.root]['porttree'] portdb = porttree.dbapi if not package.version: # we are compiling ALL versions of package allversions = portdb.xmatch('match-all', package.name) else: # we were told exact version to compile allversions = ["%s-%s" % (package.name, package.version)] for pkg in allversions: ebuild = portdb.findname(pkg) deps = portdb.aux_get(pkg,["DEPEND"]) deps = portage.dep.paren_reduce(deps[0]) settings.setcpv(pkg, mydb=portdb) use_enabled = set(settings["PORTAGE_USE"].split()) iuse = set(settings["IUSE"].split()) # only count deps enabled by USE flags use_deps = portage.dep.use_reduce(deps, list(use_enabled & iuse)) use_deps = flatten_deps(use_deps) dep_groups = self.create_dep_groups(use_deps) # prepare chroot & fork & do work try: subprocess.check_call([config.MK_CHROOT_SCRIPT,"-s",config.STAGE_TARBALL, config.BASE_CHROOT, config.WORK_CHROOT]) except CalledProcessError, cpe: raise ChrootPreparationException("Chroot preparation for %s failed with error code: %d" % (pkg, cpe.returncode)) except OSError, ose: raise ChrootPreparationException("Chroot preparation for %s failed with error(%d): %s\n\ Check your settings" % (pkg, ose.errno, ose.strerror)) self.child_pid = os.fork() childpid = self.child_pid if 0 == childpid: # we are the child! try: # setup logging! os.chroot(config.WORK_CHROOT) init_logging() package.version = "-".join(portage.pkgsplit(pkg)[1:]) self._emerge_package_subprocess(pkg, ebuild, dep_groups, package) sys.exit(0) except Exception, e: print_exc() print "Something went really bad and we need logging, stat!" log.error("Unrecoverable error in tinderbox slave, see backtrace for possible solutions:") log.error(format_exc()) sys.exit(1) (retpid, status) = os.waitpid(childpid, 0) if 0 != status: # something went really wrong, grab all the info we can print "Something went really bad and we need logging, stat!" log.error("Emerge of package %s failed with error code: %d" % (pkg, status)) package_infos = self._load_info('package_infos') msg = protocol.AddPackageInfo(package_infos) self.sock.sendall(pickle.dumps(msg)) #TODO make binpkg def _emerge_package_subprocess(self, pkg, ebuild, dep_groups, package): # We are chrooted inside WORK_CHROOT remember! porttree = self.trees[portage.root]['porttree'] portdb = porttree.dbapi vartree = self.trees[portage.root]["vartree"] package_infos = [] settings = self.settings for group in dep_groups: dep_failed = False deps_processed = [] for dep in group: try: # this will need to change since it's only a quick hack so that # we don't have to do dep resolution ourselves import _emerge as emerge dep_emergepid = os.fork() # we need to run emerge_main() in child process since a lot of stuff in there # likes to call sys.exit() and we don't want that do we? if 0 == dep_emergepid: try: extra_use = dep[0] if extra_use: os.environ["USE"]=" ".join(extra_use) sys.argv = ["emerge","--verbose","=%s" % dep[1]] exit_code = emerge.emerge_main() sys.exit(exit_code) except Exception, e: print_exc() log.error(format_exc()) sys.exit(1) ret = os.waitpid(dep_emergepid, 0) if 0 != ret[1]: raise Exception("emerge_main() failed with error code %d" % ret) except Exception, e: log.error(format_exc()) log.error("Unable to merge dependency %s for package %s" % (dep, pkg)) dep_failed = True break deps_processed.append(dep) settings.setcpv(dep[1], mydb=portdb) dep_use_enabled = set(settings["PORTAGE_USE"].split()) dep_iuse = set(settings["IUSE"].split()) dep_name, dep_ver, dep_rev = portage.pkgsplit(dep[1]) real_use_enabled = list(dep_use_enabled & dep_iuse) if dep[0]: for useflag in dep[0]: if useflag.startswith('-'): real_use_enabled.remove(useflag[1:]) elif 0 == real_use_enabled.count(useflag): real_use_enabled.append(useflag) dep_pkg = Package(dep_name,"%s-%s" % (dep_ver, dep_rev), real_use_enabled) package_infos.append(dep_pkg.get_info()) if dep_failed: log.error("Unable to emerge package %s with deps %s" % (pkg, group)) # TODO unmerge succeeded deps continue settings.setcpv(pkg, mydb=portdb) ret = portage.doebuild(ebuild, "merge", portage.root, settings, debug = False, tree="porttree") if 0 != ret: # error installing, grab logs try: logfile = open("%s/build.log" % settings["T"], "r") except IOError, (errno, strerror): print "Unable to read build logs: %d: %s" % (errno, strerror) log.warning("Unable to read build logs %d: %s" % (errno, strerror)) else: package.attachments["build_log"] = logfile.read() logfile.close() del logfile try: envfile = open("%s/environment" % settings["T"],"r") except IOError, (errno, strerror): print "Unable to read build environment: %d: %s" % (errno, strerror) log.warning("Unable to read build environment %d: %s" % (errno, strerror)) else: package.attachments["build_env"] = envfile.read() envfile.close() del envfile package.depends = [x[1] for x in deps_processed] package_infos.append(package.get_info()) for dep in group: dep_cat, dep_pv = portage.catsplit(dep[1]) ret = portage.unmerge(dep_cat, dep_pv, portage.root, settings, True, vartree=vartree) if 0 != ret: log.error("Unable to unmerge dep %s" % dep) pkg_cat, pkg_pv = portage.catsplit(pkg) ret = portage.unmerge(pkg_cat, pkg_pv, portage.root, settings, True, vartree=vartree) if ret != 0: log.error("Unable to unmerge package %s" % dep) self._save_info('package_infos', package_infos) def _add_attachment(self, pkg, path): try: attfile = open(path,"r") except IOError, (errno, strerror): print "Unable to read file %s: %d: %s" % (path, errno, strerror) log.warning("Unable to read file (%s) %d: %s" % (path, errno, strerror)) pkg.attachments[os.path.basename(path)] = "Unable to read file %s" % path else: pkg.attachments[os.path.basename(path)] = attfile.read() attfile.close() del attfile def _save_info(self, key, data): """ Save data inside CHROOT_LOGS directory, under name 'key'. This function is called from within _emerge_package_subprocess to save data for parent process @param key: key used to identify data inside CHROOT_LOGS directory @type key: string @param data: data to be saved @rtype: None """ outfile = open("%s/%s" % (config.CHROOT_LOGS, key), "w") pickle.dump(data, outfile) outfile.close() def _load_info(self, key): """ Load data from CHROOT_LOGS directory (under WORK_CHROOT) with filename 'key' @param key: key used to identify data inside directory @type key: string @rtype: depends on key @returns: data loaded from filename, usually blob """ infile = open("%s/%s/%s" % (config.WORK_CHROOT, config.CHROOT_LOGS, key),"r") return pickle.load(infile) def get_emerge_info(self): """ @rtype: string @returns: emerge --info output """ infoout = WritableObject() sys.stdout = infoout # emerge.action_info(self.settings, self.trees, [], []) # REDO withouth emerge.action_info ret = sys.stdout.content sys.stdout = sys.__stdout__ return string.join(infoout.content,sep='') def create_dep_groups(self, deps): """ Create dependency groups from package dependencies Every valid version of dep (according to input spec) has to be at least in one output dependency group @param deps: dependencies of package (atoms) @type deps: List @rtype: List of Lists @returns: List of dependency groups (list of dependency versions) together with special use flags needed Example: Input: ['=dev-libs/glib-2*', '>=net-fs/samba-3.0.0', 'x11-libs/libX11','dev-util/subversion[-dso]'] Output: [[(None,'net-fs/samba-3.2.11'),(None, 'dev-libs/glib-2.18.4-r1'),(None, 'x11-libs/libX11-1.1.2-r1'), (['-dso'],'dev-util/subversion-1.5.5'], [(None,'net-fs/samba-3.0.32'),(None, 'dev-libs/glib-2.18.4-r1'),(None, 'x11-libs/libX11-1.1.3-r1'), (['-dso'],'dev-util/subversion-1.5.5'], [(None,'net-fs/samba-3.0.32'),(None, 'dev-libs/glib-2.20.3'),(None, 'x11-libs/libX11-1.1.3'), (['-dso'],'dev-util/subversion-1.5.5'], """ result = None porttree = self.trees[portage.root]['porttree'] portdb = porttree.dbapi deps_expanded = [] max_dep_versions = 0 for dep in deps: if dep[0].startswith('!'): continue dep_useflag = list(portage.dep.dep_getusedeps(dep)) if 0 == len(dep_useflag): dep_useflag = None depversions = portdb.xmatch('match-all',dep) depversions = [(dep_useflag,x) for x in depversions] deps_expanded.append(depversions) if len(depversions) > max_dep_versions: max_dep_versions = len(depversions) for dep in deps_expanded: while len(dep) < max_dep_versions: print dep dep.append(dep[0]) result = [] for i in range(max_dep_versions): group = [] for dep in deps_expanded: group.append(dep.pop()) result.append(group) return result class Package(object): def __init__(self, name, version, use_flags): """ @param name: category/name of given package (excluding version/release) @type name: string @param version: version part of CPV e.g. 1.2.0-r4 @type version: string @param use_flags: list of enabled use flags for given package @type use_flags: list """ self.name = name self.version = version self.use_flags = use_flags self.content = None self.attachments = {} self.depends = None def get_info(self): """Returns protocol.PackageInfo with information about this package @returns: PackageInfo for sending across network @rtype: protocol.PackageInfo """ pi = protocol.PackageInfo() pi.name = self.name pi.version = self.version pi.use_flags = self.use_flags if self.content is None: self.content = self.get_package_contents() pi.content = self.content pi.attachments = self.attachments pi.depends = self.depends return pi def get_package_contents(self): """Returns package contents as dict with paths as keys data values are tuples of information (such as hash and size) @returns: package contents (or {} if it's not installed) @rtype: dict """ vartree = portage.db[portage.root]["vartree"] cpv = "%s-%s" % (self.name, self.version) if not vartree.dbapi.cpv_exists(cpv): # maybe raise exception? instead return {} cat, pkg = portage.catsplit(cpv) dblink = portage.dblink(cat, pkg, portage.root, vartree.settings, treetype="vartree", vartree=vartree) return dblink.getcontents()