diff options
author | 2014-02-17 17:55:51 +0600 | |
---|---|---|
committer | 2014-02-17 17:55:51 +0600 | |
commit | 5a3f506c9ef1cfd78940b0509f10ef94b4434e29 (patch) | |
tree | 147c35a17a8bcd8ff467bb3063adab623da51fac | |
parent | fixed a deadlock (diff) | |
download | autodep-5a3f506c9ef1cfd78940b0509f10ef94b4434e29.tar.gz autodep-5a3f506c9ef1cfd78940b0509f10ef94b4434e29.tar.bz2 autodep-5a3f506c9ef1cfd78940b0509f10ef94b4434e29.zip |
updated portage to 2.2.8-r1
622 files changed, 21297 insertions, 16543 deletions
diff --git a/portage_with_autodep/bin/archive-conf b/portage_with_autodep/bin/archive-conf index 5a03b85..7978668 100755 --- a/portage_with_autodep/bin/archive-conf +++ b/portage_with_autodep/bin/archive-conf @@ -20,7 +20,7 @@ except ImportError: import portage from portage import os -import dispatch_conf +from portage import dispatch_conf FIND_EXTANT_CONTENTS = "find %s -name CONTENTS" diff --git a/portage_with_autodep/bin/bashrc-functions.sh b/portage_with_autodep/bin/bashrc-functions.sh new file mode 100755 index 0000000..4da5585 --- /dev/null +++ b/portage_with_autodep/bin/bashrc-functions.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# Copyright 1999-2011 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +portageq() { + PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ + "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" "$@" +} + +register_die_hook() { + local x + for x in $* ; do + has $x $EBUILD_DEATH_HOOKS || \ + export EBUILD_DEATH_HOOKS="$EBUILD_DEATH_HOOKS $x" + done +} + +register_success_hook() { + local x + for x in $* ; do + has $x $EBUILD_SUCCESS_HOOKS || \ + export EBUILD_SUCCESS_HOOKS="$EBUILD_SUCCESS_HOOKS $x" + done +} + +strip_duplicate_slashes() { + if [[ -n $1 ]] ; then + local removed=$1 + while [[ ${removed} == *//* ]] ; do + removed=${removed//\/\///} + done + echo ${removed} + fi +} + +# this is a function for removing any directory matching a passed in pattern from +# PATH +remove_path_entry() { + save_IFS + IFS=":" + stripped_path="${PATH}" + while [ -n "$1" ]; do + cur_path="" + for p in ${stripped_path}; do + if [ "${p/${1}}" == "${p}" ]; then + cur_path="${cur_path}:${p}" + fi + done + stripped_path="${cur_path#:*}" + shift + done + restore_IFS + PATH="${stripped_path}" +} + +# Set given variables unless these variable have been already set (e.g. during emerge +# invocation) to values different than values set in make.conf. +set_unless_changed() { + if [[ $# -lt 1 ]]; then + die "${FUNCNAME}() requires at least 1 argument: VARIABLE=VALUE" + fi + + local argument value variable + for argument in "$@"; do + if [[ ${argument} != *=* ]]; then + die "${FUNCNAME}(): Argument '${argument}' has incorrect syntax" + fi + variable="${argument%%=*}" + value="${argument#*=}" + if eval "[[ \${${variable}} == \$(env -u ${variable} portageq envvar ${variable}) ]]"; then + eval "${variable}=\"\${value}\"" + fi + done +} + +# Unset given variables unless these variable have been set (e.g. during emerge +# invocation) to values different than values set in make.conf. +unset_unless_changed() { + if [[ $# -lt 1 ]]; then + die "${FUNCNAME}() requires at least 1 argument: VARIABLE" + fi + + local variable + for variable in "$@"; do + if eval "[[ \${${variable}} == \$(env -u ${variable} portageq envvar ${variable}) ]]"; then + unset ${variable} + fi + done +} + +KV_major() { + [[ -z $1 ]] && return 1 + + local KV=$@ + echo "${KV%%.*}" +} + +KV_minor() { + [[ -z $1 ]] && return 1 + + local KV=$@ + KV=${KV#*.} + echo "${KV%%.*}" +} + +KV_micro() { + [[ -z $1 ]] && return 1 + + local KV=$@ + KV=${KV#*.*.} + echo "${KV%%[^[:digit:]]*}" +} + +KV_to_int() { + [[ -z $1 ]] && return 1 + + local KV_MAJOR=$(KV_major "$1") + local KV_MINOR=$(KV_minor "$1") + local KV_MICRO=$(KV_micro "$1") + local KV_int=$(( KV_MAJOR * 65536 + KV_MINOR * 256 + KV_MICRO )) + + # We make version 2.2.0 the minimum version we will handle as + # a sanity check ... if its less, we fail ... + if [[ ${KV_int} -ge 131584 ]] ; then + echo "${KV_int}" + return 0 + fi + + return 1 +} + +_RC_GET_KV_CACHE="" +get_KV() { + [[ -z ${_RC_GET_KV_CACHE} ]] \ + && _RC_GET_KV_CACHE=$(uname -r) + + echo $(KV_to_int "${_RC_GET_KV_CACHE}") + + return $? +} diff --git a/portage_with_autodep/bin/chpathtool.py b/portage_with_autodep/bin/chpathtool.py new file mode 100755 index 0000000..d0d49cb --- /dev/null +++ b/portage_with_autodep/bin/chpathtool.py @@ -0,0 +1,182 @@ +#!/usr/bin/python +# Copyright 2011 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import io +import optparse +import os +import stat +import sys + +CONTENT_ENCODING = "utf_8" +FS_ENCODING = "utf_8" + +try: + import magic +except ImportError: + magic = None +else: + try: + magic.MIME_TYPE + except AttributeError: + # magic module seems to be broken + magic = None + +class IsTextFile(object): + + def __init__(self): + if magic is not None: + self._call = self._is_text_magic + self._m = magic.open(magic.MIME_TYPE) + self._m.load() + else: + self._call = self._is_text_encoding + self._encoding = CONTENT_ENCODING + + def __call__(self, filename): + """ + Returns True if the given file is a text file, and False otherwise. + """ + return self._call(filename) + + def _is_text_magic(self, filename): + mime_type = self._m.file(filename) + return mime_type.startswith("text/") + + def _is_text_encoding(self, filename): + try: + for line in io.open(filename, mode='r', encoding=self._encoding): + pass + except UnicodeDecodeError: + return False + return True + +def chpath_inplace(filename, is_text_file, old, new): + """ + Returns True if any modifications were made, and False otherwise. + """ + + modified = False + orig_stat = os.lstat(filename) + try: + f = io.open(filename, buffering=0, mode='r+b') + except IOError: + try: + orig_mode = stat.S_IMODE(os.lstat(filename).st_mode) + except OSError as e: + sys.stderr.write("%s: %s\n" % (e, filename)) + return + temp_mode = 0o200 | orig_mode + os.chmod(filename, temp_mode) + try: + f = io.open(filename, buffering=0, mode='r+b') + finally: + os.chmod(filename, orig_mode) + + len_old = len(old) + len_new = len(new) + matched_byte_count = 0 + while True: + in_byte = f.read(1) + + if not in_byte: + break + + if in_byte == old[matched_byte_count]: + matched_byte_count += 1 + if matched_byte_count == len_old: + modified = True + matched_byte_count = 0 + end_position = f.tell() + start_position = end_position - len_old + if not is_text_file: + # search backwards for leading slashes written by + # a previous invocation of this tool + num_to_write = len_old + f.seek(start_position - 1) + while True: + if f.read(1) != b'/': + break + num_to_write += 1 + f.seek(f.tell() - 2) + + # pad with as many leading slashes as necessary + while num_to_write > len_new: + f.write(b'/') + num_to_write -= 1 + f.write(new) + else: + remainder = f.read() + f.seek(start_position) + f.write(new) + if remainder: + f.write(remainder) + f.truncate() + f.seek(start_position + len_new) + elif matched_byte_count > 0: + # back up an try to start a new match after + # the first byte of the previous partial match + f.seek(f.tell() - matched_byte_count) + matched_byte_count = 0 + + f.close() + if modified: + orig_mtime = orig_stat[stat.ST_MTIME] + os.utime(filename, (orig_mtime, orig_mtime)) + return modified + +def chpath_inplace_symlink(filename, st, old, new): + target = os.readlink(filename) + if target.startswith(old): + new_target = new + target[len(old):] + os.unlink(filename) + os.symlink(new_target, filename) + os.lchown(filename, st.st_uid, st.st_gid) + +def main(argv): + + usage = "%s [options] <location> <old> <new>" % (os.path.basename(argv[0],)) + parser = optparse.OptionParser(usage=usage) + options, args = parser.parse_args(argv[1:]) + + if len(args) != 3: + parser.error("3 args required, got %s" % (len(args),)) + + location, old, new = args + + is_text_file = IsTextFile() + + if not isinstance(location, bytes): + location = location.encode(FS_ENCODING) + if not isinstance(old, bytes): + old = old.encode(FS_ENCODING) + if not isinstance(new, bytes): + new = new.encode(FS_ENCODING) + + st = os.lstat(location) + + if stat.S_ISDIR(st.st_mode): + for parent, dirs, files in os.walk(location): + for filename in files: + filename = os.path.join(parent, filename) + try: + st = os.lstat(filename) + except OSError: + pass + else: + if stat.S_ISREG(st.st_mode): + chpath_inplace(filename, + is_text_file(filename), old, new) + elif stat.S_ISLNK(st.st_mode): + chpath_inplace_symlink(filename, st, old, new) + + elif stat.S_ISREG(st.st_mode): + chpath_inplace(location, + is_text_file(location), old, new) + elif stat.S_ISLNK(st.st_mode): + chpath_inplace_symlink(location, st, old, new) + + return os.EX_OK + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/portage_with_autodep/bin/dispatch-conf b/portage_with_autodep/bin/dispatch-conf index 1e21a52..139a001 100755 --- a/portage_with_autodep/bin/dispatch-conf +++ b/portage_with_autodep/bin/dispatch-conf @@ -1,5 +1,5 @@ #!/usr/bin/python -O -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # @@ -27,13 +27,11 @@ except ImportError: from portage import os from portage import dispatch_conf from portage import _unicode_decode -from portage.dispatch_conf import diffstatusoutput_len +from portage.dispatch_conf import diffstatusoutput from portage.process import find_binary FIND_EXTANT_CONFIGS = "find '%s' %s -name '._cfg????_%s' ! -name '.*~' ! -iname '.*.bak' -print" DIFF_CONTENTS = "diff -Nu '%s' '%s'" -DIFF_CVS_INTERP = "diff -Nu '%s' '%s' | grep '^[+-][^+-]' | grep -v '# .Header:.*'" -DIFF_WSCOMMENTS = "diff -Nu '%s' '%s' | grep '^[+-][^+-]' | grep -v '^[-+]#' | grep -v '^[-+][:space:]*$'" # We need a secure scratch dir and python does silly verbose errors on the use of tempnam oldmask = os.umask(0o077) @@ -62,7 +60,21 @@ def cleanup(mydir=SCRATCH_DIR): shutil.rmtree(mydir) atexit.register(cleanup) -MANDATORY_OPTS = [ 'archive-dir', 'diff', 'replace-cvs', 'replace-wscomments', 'merge' ] +MANDATORY_OPTS = [ 'archive-dir', 'diff', 'replace-cvs', 'replace-wscomments', 'merge' ] + +def cmd_var_is_valid(cmd): + """ + Return true if the first whitespace-separated token contained + in cmd is an executable file, false otherwise. + """ + cmd = portage.util.shlex_split(cmd) + if not cmd: + return False + + if os.path.isabs(cmd[0]): + return os.access(cmd[0], os.EX_OK) + + return find_binary(cmd[0]) is not None class dispatch: options = {} @@ -71,7 +83,7 @@ class dispatch: confs = [] count = 0 - config_root = '/' + config_root = portage.const.EPREFIX or os.sep self.options = portage.dispatch_conf.read_config(MANDATORY_OPTS) if "log-file" in self.options: @@ -84,12 +96,30 @@ class dispatch: else: self.options["log-file"] = "/dev/null" + pager = self.options.get("pager") + if pager is None or not cmd_var_is_valid(pager): + pager = os.environ.get("PAGER") + if pager is None or not cmd_var_is_valid(pager): + pager = "cat" + + pager_basename = os.path.basename(portage.util.shlex_split(pager)[0]) + if pager_basename == "less": + less_opts = self.options.get("less-opts") + if less_opts is not None and less_opts.strip(): + pager += " " + less_opts + + if pager_basename == "cat": + pager = "" + else: + pager = " | " + pager + # # Build list of extant configs # for path in config_paths: - path = portage.normalize_path(path) + path = portage.normalize_path( + os.path.join(config_root, path.lstrip(os.sep))) try: mymode = os.stat(path).st_mode except OSError: @@ -100,7 +130,9 @@ class dispatch: path, basename = os.path.split(path) find_opts = "-maxdepth 1" - confs += self.massage(os.popen(FIND_EXTANT_CONFIGS % (path, find_opts, basename)).readlines()) + with os.popen(FIND_EXTANT_CONFIGS % + (path, find_opts, basename)) as proc: + confs += self.massage(proc.readlines()) if self.options['use-rcs'] == 'yes': for rcs_util in ("rcs", "ci", "co", "rcsmerge"): @@ -118,6 +150,9 @@ class dispatch: portage.util.shlex_split( portage.settings.get('CONFIG_PROTECT_MASK', ''))) + def diff(file1, file2): + return diffstatusoutput(DIFF_CONTENTS, file1, file2) + # # Remove new configs identical to current # and @@ -134,11 +169,11 @@ class dispatch: else: mrgfail = portage.dispatch_conf.file_archive(archive, conf['current'], conf['new'], mrgconf) if os.path.exists(archive + '.dist'): - unmodified = diffstatusoutput_len(DIFF_CONTENTS % (conf['current'], archive + '.dist'))[1] == 0 + unmodified = len(diff(conf['current'], archive + '.dist')[1]) == 0 else: unmodified = 0 if os.path.exists(mrgconf): - if mrgfail or diffstatusoutput_len(DIFF_CONTENTS % (conf['new'], mrgconf))[1] == 0: + if mrgfail or len(diff(conf['new'], mrgconf)[1]) == 0: os.unlink(mrgconf) newconf = conf['new'] else: @@ -149,24 +184,34 @@ class dispatch: if newconf == mrgconf and \ self.options.get('ignore-previously-merged') != 'yes' and \ os.path.exists(archive+'.dist') and \ - diffstatusoutput_len(DIFF_CONTENTS % (archive+'.dist', conf['new']))[1] == 0: + len(diff(archive+'.dist', conf['new'])[1]) == 0: # The current update is identical to the archived .dist # version that has previously been merged. os.unlink(mrgconf) newconf = conf['new'] - mystatus, myoutput_len = diffstatusoutput_len( - DIFF_CONTENTS % (conf ['current'], newconf)) + mystatus, myoutput = diff(conf['current'], newconf) + myoutput_len = len(myoutput) same_file = 0 == myoutput_len if mystatus >> 8 == 2: # Binary files differ same_cvs = False same_wsc = False else: - same_cvs = 0 == diffstatusoutput_len( - DIFF_CVS_INTERP % (conf ['current'], newconf))[1] - same_wsc = 0 == diffstatusoutput_len( - DIFF_WSCOMMENTS % (conf ['current'], newconf))[1] + # Extract all the normal diff lines (ignore the headers). + mylines = re.findall('^[+-][^\n+-].*$', myoutput, re.MULTILINE) + + # Filter out all the cvs headers + cvs_header = re.compile('# [$]Header:') + cvs_lines = list(filter(cvs_header.search, mylines)) + same_cvs = len(mylines) == len(cvs_lines) + + # Filter out comments and whitespace-only changes. + # Note: be nice to also ignore lines that only differ in whitespace... + wsc_lines = [] + for x in ['^[-+]\s*#', '^[-+]\s*$']: + wsc_lines += list(filter(re.compile(x).match, mylines)) + same_wsc = len(mylines) == len(wsc_lines) # Do options permit? same_cvs = same_cvs and self.options['replace-cvs'] == 'yes' @@ -224,10 +269,12 @@ class dispatch: clear_screen() if show_new_diff: cmd = self.options['diff'] % (conf['new'], mrgconf) + cmd += pager spawn_shell(cmd) show_new_diff = 0 else: cmd = self.options['diff'] % (conf['current'], newconf) + cmd += pager spawn_shell(cmd) print() @@ -423,6 +470,19 @@ def spawn_shell(cmd): else: os.system(cmd) +def usage(argv): + print('dispatch-conf: sane configuration file update\n') + print('Usage: dispatch-conf [config dirs]\n') + print('See the dispatch-conf(1) man page for more details') + sys.exit(os.EX_OK) + +for x in sys.argv: + if x in ('-h', '--help'): + usage(sys.argv) + elif x in ('--version'): + print("Portage", portage.VERSION) + sys.exit(os.EX_OK) + # run d = dispatch () diff --git a/portage_with_autodep/bin/dohtml.py b/portage_with_autodep/bin/dohtml.py index 00258ec..f0a7f2c 100755 --- a/portage_with_autodep/bin/dohtml.py +++ b/portage_with_autodep/bin/dohtml.py @@ -56,9 +56,9 @@ def install(basename, dirname, options, prefix=""): fullpath = dirname + "/" + fullpath if options.DOCDESTTREE: - destdir = options.D + "usr/share/doc/" + options.PF + "/" + options.DOCDESTTREE + "/" + options.doc_prefix + "/" + prefix + destdir = options.ED + "usr/share/doc/" + options.PF + "/" + options.DOCDESTTREE + "/" + options.doc_prefix + "/" + prefix else: - destdir = options.D + "usr/share/doc/" + options.PF + "/html/" + options.doc_prefix + "/" + prefix + destdir = options.ED + "usr/share/doc/" + options.PF + "/html/" + options.doc_prefix + "/" + prefix if not os.path.exists(fullpath): sys.stderr.write("!!! dohtml: %s does not exist\n" % fullpath) @@ -86,13 +86,16 @@ def install(basename, dirname, options, prefix=""): class OptionsClass: def __init__(self): self.PF = "" - self.D = "" + self.ED = "" self.DOCDESTTREE = "" if "PF" in os.environ: self.PF = os.environ["PF"] - if "D" in os.environ: - self.D = os.environ["D"] + if "force-prefix" not in os.environ.get("FEATURES", "").split() and \ + os.environ.get("EAPI", "0") in ("0", "1", "2"): + self.ED = os.environ.get("D", "") + else: + self.ED = os.environ.get("ED", "") if "_E_DOCDESTTREE_" in os.environ: self.DOCDESTTREE = os.environ["_E_DOCDESTTREE_"] diff --git a/portage_with_autodep/bin/eapi.sh b/portage_with_autodep/bin/eapi.sh new file mode 100755 index 0000000..623b89f --- /dev/null +++ b/portage_with_autodep/bin/eapi.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# PHASES + +___eapi_has_pkg_pretend() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_has_src_prepare() { + [[ ! ${1-${EAPI}} =~ ^(0|1)$ ]] +} + +___eapi_has_src_configure() { + [[ ! ${1-${EAPI}} =~ ^(0|1)$ ]] +} + +___eapi_default_src_test_disables_parallel_jobs() { + [[ ${1-${EAPI}} =~ ^(0|1|2|3|4|4-python|4-slot-abi)$ ]] +} + +___eapi_has_S_WORKDIR_fallback() { + [[ ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +# VARIABLES + +___eapi_has_prefix_variables() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2)$ || " ${FEATURES} " == *" force-prefix "* ]] +} + +___eapi_has_HDEPEND() { + [[ ${1-${EAPI}} =~ ^(5-hdepend)$ ]] +} + +___eapi_has_RDEPEND_DEPEND_fallback() { + [[ ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +# HELPERS PRESENCE + +___eapi_has_dohard() { + [[ ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_has_dosed() { + [[ ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_has_docompress() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_has_nonfatal() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_has_doheader() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3|4|4-python|4-slot-abi)$ ]] +} + +___eapi_has_usex() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3|4|4-python|4-slot-abi)$ ]] +} + +___eapi_has_master_repositories() { + [[ ${1-${EAPI}} =~ ^(5-progress)$ ]] +} + +___eapi_has_repository_path() { + [[ ${1-${EAPI}} =~ ^(5-progress)$ ]] +} + +___eapi_has_available_eclasses() { + [[ ${1-${EAPI}} =~ ^(5-progress)$ ]] +} + +___eapi_has_eclass_path() { + [[ ${1-${EAPI}} =~ ^(5-progress)$ ]] +} + +___eapi_has_license_path() { + [[ ${1-${EAPI}} =~ ^(5-progress)$ ]] +} + +___eapi_has_package_manager_build_user() { + [[ ${1-${EAPI}} =~ ^(5-progress)$ ]] +} + +___eapi_has_package_manager_build_group() { + [[ ${1-${EAPI}} =~ ^(5-progress)$ ]] +} + +# HELPERS BEHAVIOR + +___eapi_best_version_and_has_version_support_--host-root() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3|4|4-python|4-slot-abi)$ ]] +} + +___eapi_unpack_supports_xz() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2)$ ]] +} + +___eapi_econf_passes_--disable-dependency-tracking() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_econf_passes_--disable-silent-rules() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3|4|4-python|4-slot-abi)$ ]] +} + +___eapi_use_enable_and_use_with_support_empty_third_argument() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_dodoc_supports_-r() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_doins_and_newins_preserve_symlinks() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_newins_supports_reading_from_standard_input() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3|4|4-python|4-slot-abi)$ ]] +} + +___eapi_helpers_can_die() { + [[ ! ${1-${EAPI}} =~ ^(0|1|2|3)$ ]] +} + +___eapi_disallows_helpers_in_global_scope() { + [[ ${1-${EAPI}} =~ ^(4-python|5-progress)$ ]] +} + +___eapi_unpack_is_case_sensitive() { + [[ ${1-${EAPI}} =~ ^(0|1|2|3|4|4-python|4-slot-abi|5|5-hdepend)$ ]] +} + +# OTHERS + +___eapi_enables_globstar() { + [[ ${1-${EAPI}} =~ ^(4-python|5-progress)$ ]] +} diff --git a/portage_with_autodep/bin/ebuild b/portage_with_autodep/bin/ebuild index f8b6d79..35cdc14 100755 --- a/portage_with_autodep/bin/ebuild +++ b/portage_with_autodep/bin/ebuild @@ -1,5 +1,5 @@ #!/usr/bin/python -O -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -29,6 +29,7 @@ def debug_signal(signum, frame): signal.signal(signal.SIGUSR1, debug_signal) import imp +import io import optparse import os @@ -46,6 +47,8 @@ parser.add_option("--color", help="enable or disable color output", type="choice", choices=("y", "n")) parser.add_option("--debug", help="show debug output", action="store_true", dest="debug") +parser.add_option("--version", help="show version and exit", + action="store_true", dest="version") parser.add_option("--ignore-default-opts", action="store_true", help="do not use the EBUILD_DEFAULT_OPTS environment variable") @@ -54,14 +57,6 @@ parser.add_option("--skip-manifest", help="skip all manifest checks", opts, pargs = parser.parse_args(args=sys.argv[1:]) -if len(pargs) < 2: - parser.error("missing required args") - -if "merge" in pargs: - print("Disabling noauto in features... merge disables it. (qmerge doesn't)") - os.environ["FEATURES"] = os.environ.get("FEATURES", "") + " -noauto" - -os.environ["PORTAGE_CALLER"]="ebuild" try: import portage except ImportError: @@ -79,6 +74,13 @@ from portage.const import VDB_PATH from _emerge.Package import Package from _emerge.RootConfig import RootConfig +if opts.version: + print("Portage", portage.VERSION) + sys.exit(os.EX_OK) + +if len(pargs) < 2: + parser.error("missing required args") + if not opts.ignore_default_opts: default_opts = portage.settings.get("EBUILD_DEFAULT_OPTS", "").split() opts, pargs = parser.parse_args(default_opts + sys.argv[1:]) @@ -138,27 +140,24 @@ vdb_path = os.path.realpath(os.path.join(portage.settings['EROOT'], VDB_PATH)) # Make sure that portdb.findname() returns the correct ebuild. if ebuild_portdir != vdb_path and \ ebuild_portdir not in portage.portdb.porttrees: + portdir_overlay = portage.settings.get("PORTDIR_OVERLAY", "") if sys.hexversion >= 0x3000000: os.environ["PORTDIR_OVERLAY"] = \ - os.environ.get("PORTDIR_OVERLAY","") + \ + portdir_overlay + \ " " + _shell_quote(ebuild_portdir) else: os.environ["PORTDIR_OVERLAY"] = \ - os.environ.get("PORTDIR_OVERLAY","") + \ + _unicode_encode(portdir_overlay, + encoding=_encodings['content'], errors='strict') + \ " " + _unicode_encode(_shell_quote(ebuild_portdir), encoding=_encodings['content'], errors='strict') print("Appending %s to PORTDIR_OVERLAY..." % ebuild_portdir) imp.reload(portage) -# Constrain eclass resolution to the master(s) -# that are specified in layout.conf (using an -# approach similar to repoman's). myrepo = None if ebuild_portdir != vdb_path: myrepo = portage.portdb.getRepositoryName(ebuild_portdir) - repo_info = portage.portdb._repo_info[ebuild_portdir] - portage.portdb.porttrees = list(repo_info.eclass_db.porttrees) if not os.path.exists(ebuild): print("'%s' does not exist." % ebuild) @@ -167,7 +166,12 @@ if not os.path.exists(ebuild): ebuild_split = ebuild.split("/") cpv = "%s/%s" % (ebuild_split[-3], pf) -if not portage.catpkgsplit(cpv): +with io.open(_unicode_encode(ebuild, encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace') as f: + eapi = portage._parse_eapi_ebuild_head(f)[0] +if eapi is None: + eapi = "0" +if not portage.catpkgsplit(cpv, eapi=eapi): print("!!! %s does not follow correct package syntax." % (cpv)) sys.exit(1) @@ -204,9 +208,10 @@ def discard_digests(myebuild, mysettings, mydbapi): portage._doebuild_manifest_exempt_depend += 1 pkgdir = os.path.dirname(myebuild) fetchlist_dict = portage.FetchlistDict(pkgdir, mysettings, mydbapi) - from portage.manifest import Manifest - mf = Manifest(pkgdir, mysettings["DISTDIR"], - fetchlist_dict=fetchlist_dict, manifest1_compat=False) + mf = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"], + fetchlist_dict=fetchlist_dict) mf.create(requiredDistfiles=None, assumeDistHashesSometimes=True, assumeDistHashesAlways=True) distfiles = fetchlist_dict[cpv] @@ -228,10 +233,8 @@ build_dir_phases = set(["setup", "unpack", "prepare", "configure", "compile", # sourced again even if $T/environment already exists. ebuild_changed = False if mytree == "porttree" and build_dir_phases.intersection(pargs): - metadata, st, emtime = \ - portage.portdb._pull_valid_cache(cpv, ebuild, ebuild_portdir) - if metadata is None: - ebuild_changed = True + ebuild_changed = \ + portage.portdb._pull_valid_cache(cpv, ebuild, ebuild_portdir)[0] is None tmpsettings = portage.config(clone=portage.settings) tmpsettings["PORTAGE_VERBOSE"] = "1" @@ -257,16 +260,20 @@ if "test" in pargs: tmpsettings.features.discard("fail-clean") +if "merge" in pargs and "noauto" in tmpsettings.features: + print("Disabling noauto in features... merge disables it. (qmerge doesn't)") + tmpsettings.features.discard("noauto") + try: metadata = dict(zip(Package.metadata_keys, - portage.db[portage.settings["ROOT"]][mytree].dbapi.aux_get( + portage.db[portage.settings['EROOT']][mytree].dbapi.aux_get( cpv, Package.metadata_keys, myrepo=myrepo))) except KeyError: # aux_get failure, message should have been shown on stderr. sys.exit(1) root_config = RootConfig(portage.settings, - portage.db[portage.settings["ROOT"]], None) + portage.db[portage.settings['EROOT']], None) pkg = Package(built=(pkg_type != "ebuild"), cpv=cpv, installed=(pkg_type=="installed"), @@ -275,7 +282,10 @@ pkg = Package(built=(pkg_type != "ebuild"), cpv=cpv, # Apply package.env and repo-level settings. This allows per-package # FEATURES and other variables (possibly PORTAGE_TMPDIR) to be -# available as soon as possible. +# available as soon as possible. Also, note that the only way to ensure +# that setcpv gets metadata from the correct repository is to pass in +# a Package instance, as we do here (previously we had to modify +# portdb.porttrees in order to accomplish this). tmpsettings.setcpv(pkg) def stale_env_warning(): diff --git a/portage_with_autodep/bin/ebuild-helpers/sed b/portage_with_autodep/bin/ebuild-helpers/bsd/sed index b21e856..01b8847 100755 --- a/portage_with_autodep/bin/ebuild-helpers/sed +++ b/portage_with_autodep/bin/ebuild-helpers/bsd/sed @@ -1,27 +1,27 @@ #!/bin/bash -# Copyright 2007 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 scriptpath=${BASH_SOURCE[0]} scriptname=${scriptpath##*/} -if [[ sed == ${scriptname} ]] && [[ -n ${ESED} ]]; then +if [[ sed == ${scriptname} && -n ${ESED} ]]; then exec ${ESED} "$@" elif type -P g${scriptname} > /dev/null ; then exec g${scriptname} "$@" else old_IFS="${IFS}" IFS=":" - + for path in $PATH; do - [[ ${path}/${scriptname} == ${scriptpath} ]] && continue if [[ -x ${path}/${scriptname} ]]; then - exec ${path}/${scriptname} "$@" + [[ ${path}/${scriptname} -ef ${scriptpath} ]] && continue + exec "${path}/${scriptname}" "$@" exit 0 fi done - + IFS="${old_IFS}" fi - + exit 1 diff --git a/portage_with_autodep/bin/ebuild-helpers/dobin b/portage_with_autodep/bin/ebuild-helpers/dobin index e385455..f90d893 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dobin +++ b/portage_with_autodep/bin/ebuild-helpers/dobin @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -9,15 +9,18 @@ if [[ $# -lt 1 ]] ; then exit 1 fi -if [[ ! -d ${D}${DESTTREE}/bin ]] ; then - install -d "${D}${DESTTREE}/bin" || { helpers_die "${0##*/}: failed to install ${D}${DESTTREE}/bin"; exit 2; } +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +if [[ ! -d ${ED}${DESTTREE}/bin ]] ; then + install -d "${ED}${DESTTREE}/bin" || { helpers_die "${0##*/}: failed to install ${ED}${DESTTREE}/bin"; exit 2; } fi ret=0 for x in "$@" ; do if [[ -e ${x} ]] ; then - install -m0755 -o ${PORTAGE_INST_UID:-0} -g ${PORTAGE_INST_GID:-0} "${x}" "${D}${DESTTREE}/bin" + install -m0755 -o ${PORTAGE_INST_UID:-0} -g ${PORTAGE_INST_GID:-0} "${x}" "${ED}${DESTTREE}/bin" else echo "!!! ${0##*/}: $x does not exist" 1>&2 false diff --git a/portage_with_autodep/bin/ebuild-helpers/dodir b/portage_with_autodep/bin/ebuild-helpers/dodir index f40bee7..90a3efe 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dodir +++ b/portage_with_autodep/bin/ebuild-helpers/dodir @@ -1,10 +1,13 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -install -d ${DIROPTIONS} "${@/#/${D}/}" +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +install -d ${DIROPTIONS} "${@/#/${ED}/}" ret=$? [[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/dodoc b/portage_with_autodep/bin/ebuild-helpers/dodoc index 65713db..1f333a6 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dodoc +++ b/portage_with_autodep/bin/ebuild-helpers/dodoc @@ -9,7 +9,10 @@ if [ $# -lt 1 ] ; then exit 1 fi -dir="${D}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +dir="${ED}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" if [ ! -d "${dir}" ] ; then install -d "${dir}" fi diff --git a/portage_with_autodep/bin/ebuild-helpers/doexe b/portage_with_autodep/bin/ebuild-helpers/doexe index 360800e..fb228f9 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doexe +++ b/portage_with_autodep/bin/ebuild-helpers/doexe @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -9,8 +9,11 @@ if [[ $# -lt 1 ]] ; then exit 1 fi -if [[ ! -d ${D}${_E_EXEDESTTREE_} ]] ; then - install -d "${D}${_E_EXEDESTTREE_}" +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +if [[ ! -d ${ED}${_E_EXEDESTTREE_} ]] ; then + install -d "${ED}${_E_EXEDESTTREE_}" fi TMP=$T/.doexe_tmp @@ -29,7 +32,7 @@ for x in "$@" ; do mysrc="${x}" fi if [ -e "$mysrc" ] ; then - install $EXEOPTIONS "$mysrc" "$D$_E_EXEDESTTREE_" + install $EXEOPTIONS "$mysrc" "$ED$_E_EXEDESTTREE_" else echo "!!! ${0##*/}: $mysrc does not exist" 1>&2 false diff --git a/portage_with_autodep/bin/ebuild-helpers/dohard b/portage_with_autodep/bin/ebuild-helpers/dohard index 2270487..b52fd7c 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dohard +++ b/portage_with_autodep/bin/ebuild-helpers/dohard @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2007 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 if [[ $# -ne 2 ]] ; then @@ -7,7 +7,10 @@ if [[ $# -ne 2 ]] ; then exit 1 fi +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + destdir=${2%/*} -[[ ! -d ${D}${destdir} ]] && dodir "${destdir}" +[[ ! -d ${ED}${destdir} ]] && dodir "${destdir}" -exec ln -f "${D}$1" "${D}$2" +exec ln -f "${ED}$1" "${ED}$2" diff --git a/portage_with_autodep/bin/ebuild-helpers/doheader b/portage_with_autodep/bin/ebuild-helpers/doheader new file mode 100755 index 0000000..3795365 --- /dev/null +++ b/portage_with_autodep/bin/ebuild-helpers/doheader @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 1999-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh + +if ! ___eapi_has_doheader; then + die "${0##*/} is not supported in EAPI ${EAPI}" +fi + +if [[ $# -lt 1 ]] || [[ $1 == -r && $# -lt 2 ]] ; then + __helpers_die "${0##*/}: at least one argument needed" + exit 1 +fi + +exec \ +env \ +INSDESTTREE="/usr/include/" \ +doins "$@" diff --git a/portage_with_autodep/bin/ebuild-helpers/doinfo b/portage_with_autodep/bin/ebuild-helpers/doinfo index 54fb8da..8fd7d45 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doinfo +++ b/portage_with_autodep/bin/ebuild-helpers/doinfo @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -9,11 +9,14 @@ if [[ -z $1 ]] ; then exit 1 fi -if [[ ! -d ${D}usr/share/info ]] ; then - install -d "${D}usr/share/info" || { helpers_die "${0##*/}: failed to install ${D}usr/share/info"; exit 1; } +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +if [[ ! -d ${ED}usr/share/info ]] ; then + install -d "${ED}usr/share/info" || { helpers_die "${0##*/}: failed to install ${ED}usr/share/info"; exit 1; } fi -install -m0644 "$@" "${D}usr/share/info" +install -m0644 "$@" "${ED}usr/share/info" rval=$? if [ $rval -ne 0 ] ; then for x in "$@" ; do diff --git a/portage_with_autodep/bin/ebuild-helpers/doins b/portage_with_autodep/bin/ebuild-helpers/doins index 7dec146..443bfdb 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doins +++ b/portage_with_autodep/bin/ebuild-helpers/doins @@ -27,12 +27,15 @@ else DOINSRECUR=n fi -if [[ ${INSDESTTREE#${D}} != "${INSDESTTREE}" ]]; then +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) export ED="${D}" ;; esac + +if [[ ${INSDESTTREE#${ED}} != "${INSDESTTREE}" ]]; then vecho "-------------------------------------------------------" 1>&2 - vecho "You should not use \${D} with helpers." 1>&2 + vecho "You should not use \${D} or \${ED} with helpers." 1>&2 vecho " --> ${INSDESTTREE}" 1>&2 vecho "-------------------------------------------------------" 1>&2 - helpers_die "${0##*/} used with \${D}" + helpers_die "${0##*/} used with \${D} or \${ED}" exit 1 fi @@ -49,7 +52,7 @@ export TMP=$T/.doins_tmp # Use separate directories to avoid potential name collisions. mkdir -p "$TMP"/{1,2} -[[ ! -d ${D}${INSDESTTREE} ]] && dodir "${INSDESTTREE}" +[[ ! -d ${ED}${INSDESTTREE} ]] && dodir "${INSDESTTREE}" _doins() { local mysrc="$1" mydir="$2" cleanup="" rval @@ -63,8 +66,8 @@ _doins() { # $PORTAGE_ACTUAL_DISTDIR/. if [ $PRESERVE_SYMLINKS = y ] && \ ! [[ $(readlink "$mysrc") == "$PORTAGE_ACTUAL_DISTDIR"/* ]] ; then - rm -rf "$D$INSDESTTREE/$mydir/${mysrc##*/}" || return $? - cp -P "$mysrc" "$D$INSDESTTREE/$mydir/${mysrc##*/}" + rm -rf "${ED}$INSDESTTREE/$mydir/${mysrc##*/}" || return $? + cp -P "$mysrc" "${ED}$INSDESTTREE/$mydir/${mysrc##*/}" return $? else cp "$mysrc" "$TMP/2/${mysrc##*/}" || return $? @@ -73,7 +76,7 @@ _doins() { fi fi - install ${INSOPTIONS} "${mysrc}" "${D}${INSDESTTREE}/${mydir}" + install ${INSOPTIONS} "${mysrc}" "${ED}${INSDESTTREE}/${mydir}" rval=$? [[ -n ${cleanup} ]] && rm -f "${cleanup}" [ $rval -ne 0 ] && echo "!!! ${0##*/}: $mysrc does not exist" 1>&2 diff --git a/portage_with_autodep/bin/ebuild-helpers/dolib b/portage_with_autodep/bin/ebuild-helpers/dolib index 87ade42..9af5418 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dolib +++ b/portage_with_autodep/bin/ebuild-helpers/dolib @@ -1,9 +1,12 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + # Setup ABI cruft LIBDIR_VAR="LIBDIR_${ABI}" if [[ -n ${ABI} && -n ${!LIBDIR_VAR} ]] ; then @@ -12,7 +15,7 @@ fi unset LIBDIR_VAR # we need this to default to lib so that things dont break CONF_LIBDIR=${CONF_LIBDIR:-lib} -libdir="${D}${DESTTREE}/${CONF_LIBDIR}" +libdir="${ED}${DESTTREE}/${CONF_LIBDIR}" if [[ $# -lt 1 ]] ; then diff --git a/portage_with_autodep/bin/ebuild-helpers/doman b/portage_with_autodep/bin/ebuild-helpers/doman index 4561bef..b4047ce 100755 --- a/portage_with_autodep/bin/ebuild-helpers/doman +++ b/portage_with_autodep/bin/ebuild-helpers/doman @@ -9,6 +9,9 @@ if [[ $# -lt 1 ]] ; then exit 1 fi +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + i18n="" ret=0 @@ -44,11 +47,11 @@ for x in "$@" ; do if [[ ${mandir} == *man[0-9n] ]] ; then if [[ -s ${x} ]] ; then - if [[ ! -d ${D}/usr/share/man/${mandir} ]] ; then - install -d "${D}/usr/share/man/${mandir}" + if [[ ! -d ${ED}/usr/share/man/${mandir} ]] ; then + install -d "${ED}/usr/share/man/${mandir}" fi - install -m0644 "${x}" "${D}/usr/share/man/${mandir}/${name}" + install -m0644 "${x}" "${ED}/usr/share/man/${mandir}/${name}" ((ret|=$?)) elif [[ ! -e ${x} ]] ; then echo "!!! ${0##*/}: $x does not exist" 1>&2 diff --git a/portage_with_autodep/bin/ebuild-helpers/domo b/portage_with_autodep/bin/ebuild-helpers/domo index 4737f44..d994343 100755 --- a/portage_with_autodep/bin/ebuild-helpers/domo +++ b/portage_with_autodep/bin/ebuild-helpers/domo @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -9,8 +9,12 @@ if [ ${mynum} -lt 1 ] ; then helpers_die "${0}: at least one argument needed" exit 1 fi -if [ ! -d "${D}${DESTTREE}/share/locale" ] ; then - install -d "${D}${DESTTREE}/share/locale/" + +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +if [ ! -d "${ED}${DESTTREE}/share/locale" ] ; then + install -d "${ED}${DESTTREE}/share/locale/" fi ret=0 @@ -18,7 +22,7 @@ ret=0 for x in "$@" ; do if [ -e "${x}" ] ; then mytiny="${x##*/}" - mydir="${D}${DESTTREE}/share/locale/${mytiny%.*}/LC_MESSAGES" + mydir="${ED}${DESTTREE}/share/locale/${mytiny%.*}/LC_MESSAGES" if [ ! -d "${mydir}" ] ; then install -d "${mydir}" fi diff --git a/portage_with_autodep/bin/ebuild-helpers/dosbin b/portage_with_autodep/bin/ebuild-helpers/dosbin index 87a3091..d101c8a 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dosbin +++ b/portage_with_autodep/bin/ebuild-helpers/dosbin @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -9,15 +9,18 @@ if [[ $# -lt 1 ]] ; then exit 1 fi -if [[ ! -d ${D}${DESTTREE}/sbin ]] ; then - install -d "${D}${DESTTREE}/sbin" || { helpers_die "${0##*/}: failed to install ${D}${DESTTREE}/sbin"; exit 2; } +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +if [[ ! -d ${ED}${DESTTREE}/sbin ]] ; then + install -d "${ED}${DESTTREE}/sbin" || { helpers_die "${0##*/}: failed to install ${ED}${DESTTREE}/sbin"; exit 2; } fi ret=0 for x in "$@" ; do if [[ -e ${x} ]] ; then - install -m0755 -o ${PORTAGE_INST_UID:-0} -g ${PORTAGE_INST_GID:-0} "${x}" "${D}${DESTTREE}/sbin" + install -m0755 -o ${PORTAGE_INST_UID:-0} -g ${PORTAGE_INST_GID:-0} "${x}" "${ED}${DESTTREE}/sbin" else echo "!!! ${0##*/}: ${x} does not exist" 1>&2 false diff --git a/portage_with_autodep/bin/ebuild-helpers/dosed b/portage_with_autodep/bin/ebuild-helpers/dosed index afc949b..f202df7 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dosed +++ b/portage_with_autodep/bin/ebuild-helpers/dosed @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 if [[ $# -lt 1 ]] ; then @@ -7,12 +7,15 @@ if [[ $# -lt 1 ]] ; then exit 1 fi +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + ret=0 file_found=0 -mysed="s:${D}::g" +mysed="s:${ED}::g" for x in "$@" ; do - y=$D${x#/} + y=$ED${x#/} if [ -e "${y}" ] ; then if [ -f "${y}" ] ; then file_found=1 diff --git a/portage_with_autodep/bin/ebuild-helpers/dosym b/portage_with_autodep/bin/ebuild-helpers/dosym index 500dad0..2489e22 100755 --- a/portage_with_autodep/bin/ebuild-helpers/dosym +++ b/portage_with_autodep/bin/ebuild-helpers/dosym @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -9,10 +9,22 @@ if [[ $# -ne 2 ]] ; then exit 1 fi +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +if [[ ${2} == */ ]] || \ + [[ -d ${ED}${2} && ! -L ${ED}${2} ]] ; then + # implicit basename not allowed by PMS (bug #379899) + eqawarn "QA Notice: dosym target omits basename: '${2}'" +fi + destdir=${2%/*} -[[ ! -d ${D}${destdir} ]] && dodir "${destdir}" +[[ ! -d ${ED}${destdir} ]] && dodir "${destdir}" +# when absolute, prefix with offset for Gentoo Prefix +target="${1}" +[[ ${target:0:1} == "/" ]] && target="${EPREFIX}${target}" +ln -snf "${target}" "${ED}${2}" -ln -snf "$1" "${D}$2" ret=$? [[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/ecompressdir b/portage_with_autodep/bin/ebuild-helpers/ecompressdir index 7a95120..a2c9e52 100755 --- a/portage_with_autodep/bin/ebuild-helpers/ecompressdir +++ b/portage_with_autodep/bin/ebuild-helpers/ecompressdir @@ -1,27 +1,30 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/helper-functions.sh if [[ -z $1 ]] ; then helpers_die "${0##*/}: at least one argument needed" exit 1 fi +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} EPREFIX= ;; esac + case $1 in --ignore) shift for skip in "$@" ; do - [[ -d ${D}${skip} || -f ${D}${skip} ]] \ - && >> "${D}${skip}.ecompress.skip" + [[ -d ${ED}${skip} || -f ${ED}${skip} ]] \ + && >> "${ED}${skip}.ecompress.skip" done exit 0 ;; --queue) shift set -- "${@/%/.ecompress.dir}" - set -- "${@/#/${D}}" + set -- "${@/#/${ED}}" ret=0 for x in "$@" ; do >> "$x" @@ -32,10 +35,10 @@ case $1 in ;; --dequeue) [[ -n $2 ]] && vecho "${0##*/}: --dequeue takes no additional arguments" 1>&2 - find "${D}" -name '*.ecompress.dir' -print0 \ - | sed -e 's:\.ecompress\.dir::g' -e "s:${D}:/:g" \ + find "${ED}" -name '*.ecompress.dir' -print0 \ + | sed -e 's:\.ecompress\.dir::g' -e "s:${ED}:/:g" \ | ${XARGS} -0 ecompressdir - find "${D}" -name '*.ecompress.skip' -print0 | ${XARGS} -0 rm -f + find "${ED}" -name '*.ecompress.skip' -print0 | ${XARGS} -0 rm -f exit 0 ;; --*) @@ -66,6 +69,17 @@ funk_up_dir() { while read -r -d $'\0' brokenlink ; do [[ -e ${brokenlink} ]] && continue olddest=$(readlink "${brokenlink}") + # Ignore temporarily broken symlinks due to + # _relocate_skip_dirs (bug #399595), and handle + # absolute symlinks to files that aren't merged + # yet (bug #405327). + if [[ ${olddest} == /* ]] ; then + [ -e "${D}${olddest}" ] && continue + skip_dir_dest=${T}/ecompress-skip/${olddest#${EPREFIX}} + else + skip_dir_dest=${T}/ecompress-skip/${actual_dir#${ED}}/${brokenlink%/*}/${olddest} + fi + [[ -e ${skip_dir_dest} ]] && continue [[ ${act} == "compress" ]] \ && newdest="${olddest}${suffix}" \ || newdest="${olddest%${suffix}}" @@ -95,18 +109,28 @@ _relocate_skip_dirs() { mv "${src}.ecompress.skip" "${dst}.ecompress.skip" done } -hide_skip_dirs() { _relocate_skip_dirs "${D}" "${T}"/ecompress-skip/ ; } -restore_skip_dirs() { _relocate_skip_dirs "${T}"/ecompress-skip/ "${D}" ; } +hide_skip_dirs() { _relocate_skip_dirs "${ED}" "${T}"/ecompress-skip/ ; } +restore_skip_dirs() { _relocate_skip_dirs "${T}"/ecompress-skip/ "${ED}" ; } ret=0 rm -rf "${T}"/ecompress-skip +decompressors=( + ".Z" "gunzip -f" + ".gz" "gunzip -f" + ".bz2" "bunzip2 -f" + ".xz" "unxz -f" + ".lzma" "unxz -f" +) + +multijob_init + for dir in "$@" ; do dir=${dir#/} - dir="${D}${dir}" + dir="${ED}${dir}" if [[ ! -d ${dir} ]] ; then - vecho "${0##*/}: /${dir#${D}} does not exist!" + vecho "${0##*/}: /${dir#${ED}} does not exist!" continue fi cd "${dir}" @@ -122,18 +146,31 @@ for dir in "$@" ; do find "${dir}" -type f -name '*.ecompress.file' -print0 | ${XARGS} -0 rm -f # not uncommon for packages to compress doc files themselves - funk_up_dir "decompress" ".Z" "gunzip -f" - funk_up_dir "decompress" ".gz" "gunzip -f" - funk_up_dir "decompress" ".bz2" "bunzip2 -f" + for (( d = 0; d < ${#decompressors[@]}; d += 2 )) ; do + # It's faster to parallelize at this stage than to try to + # parallelize the compressors. This is because the find|xargs + # ends up launching less compressors overall, so the overhead + # of forking children ends up dominating. + ( + multijob_child_init + funk_up_dir "decompress" "${decompressors[i]}" "${decompressors[i+1]}" + ) & + multijob_post_fork + : $(( ret |= $? )) + done # forcibly break all hard links as some compressors whine about it find "${dir}" -type f -links +1 -exec env file="{}" sh -c \ 'cp -p "${file}" "${file}.ecompress.break" ; mv -f "${file}.ecompress.break" "${file}"' \; + multijob_finish + : $(( ret |= $? )) + # now lets do our work - [[ -z ${suffix} ]] && continue - vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${D}}" - funk_up_dir "compress" "${suffix}" "ecompress" + if [[ -n ${suffix} ]] ; then + vecho "${0##*/}: $(ecompress --bin) /${actual_dir#${ED}}" + funk_up_dir "compress" "${suffix}" "ecompress" + fi # finally, restore the skipped stuff restore_skip_dirs diff --git a/portage_with_autodep/bin/ebuild-helpers/fowners b/portage_with_autodep/bin/ebuild-helpers/fowners index 4cc6bfa..a213c9e 100755 --- a/portage_with_autodep/bin/ebuild-helpers/fowners +++ b/portage_with_autodep/bin/ebuild-helpers/fowners @@ -1,13 +1,22 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) EPREFIX= ED=${D} ;; esac + # we can't prefix all arguments because # chown takes random options slash="/" -chown "${@/#${slash}/${D}${slash}}" +chown "${@/#${slash}/${ED}${slash}}" ret=$? + +if [[ ${ret} != 0 && -n ${EPREFIX} && ${EUID} != 0 ]] ; then + ewarn "fowners failure ignored in Prefix with non-privileged user" + exit 0 +fi + [[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/fperms b/portage_with_autodep/bin/ebuild-helpers/fperms index 0260bdc..a2f77ea 100755 --- a/portage_with_autodep/bin/ebuild-helpers/fperms +++ b/portage_with_autodep/bin/ebuild-helpers/fperms @@ -1,13 +1,16 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + # we can't prefix all arguments because # chmod takes random options slash="/" -chmod "${@/#${slash}/${D}${slash}}" +chmod "${@/#${slash}/${ED}${slash}}" ret=$? [[ $ret -ne 0 ]] && helpers_die "${0##*/} failed" exit $ret diff --git a/portage_with_autodep/bin/ebuild-helpers/keepdir b/portage_with_autodep/bin/ebuild-helpers/keepdir new file mode 100755 index 0000000..bec2feb --- /dev/null +++ b/portage_with_autodep/bin/ebuild-helpers/keepdir @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright 1999-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh + +if ! ___eapi_has_prefix_variables; then + ED=${D} +fi + +dodir "$@" +ret=$? + +for x in "$@"; do + >> "${ED}${x}/.keep_${CATEGORY}_${PN}-${SLOT%/*}" || \ + { echo "!!! ${0##*/}: cannot write .keep in ${ED}${x}" 1>&2; ret=1; } +done + +[[ ${ret} -ne 0 ]] && __helpers_die "${0##*/} failed" +exit ${ret} diff --git a/portage_with_autodep/bin/ebuild-helpers/newbin b/portage_with_autodep/bin/ebuild-helpers/newbin index 30f19b0..bf98744 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newbin +++ b/portage_with_autodep/bin/ebuild-helpers/newbin @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec dobin "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newconfd b/portage_with_autodep/bin/ebuild-helpers/newconfd index 5752cfa..fa3710d 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newconfd +++ b/portage_with_autodep/bin/ebuild-helpers/newconfd @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec doconfd "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newdoc b/portage_with_autodep/bin/ebuild-helpers/newdoc index f97ce0d..df6fb1d 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newdoc +++ b/portage_with_autodep/bin/ebuild-helpers/newdoc @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec dodoc "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newenvd b/portage_with_autodep/bin/ebuild-helpers/newenvd index 83c556e..c54af05 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newenvd +++ b/portage_with_autodep/bin/ebuild-helpers/newenvd @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec doenvd "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newexe b/portage_with_autodep/bin/ebuild-helpers/newexe index 92dbe9f..9bcf64b 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newexe +++ b/portage_with_autodep/bin/ebuild-helpers/newexe @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec doexe "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newheader b/portage_with_autodep/bin/ebuild-helpers/newheader new file mode 120000 index 0000000..59a0db2 --- /dev/null +++ b/portage_with_autodep/bin/ebuild-helpers/newheader @@ -0,0 +1 @@ +newins
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/newinitd b/portage_with_autodep/bin/ebuild-helpers/newinitd index fc6003a..03bbe68 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newinitd +++ b/portage_with_autodep/bin/ebuild-helpers/newinitd @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec doinitd "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newins b/portage_with_autodep/bin/ebuild-helpers/newins index 065477f..adf2d80 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newins +++ b/portage_with_autodep/bin/ebuild-helpers/newins @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" || exit $? case "$EAPI" in 0|1|2|3|3_pre2) diff --git a/portage_with_autodep/bin/ebuild-helpers/newlib.a b/portage_with_autodep/bin/ebuild-helpers/newlib.a index eef4104..7ff8195 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newlib.a +++ b/portage_with_autodep/bin/ebuild-helpers/newlib.a @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec dolib.a "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newlib.so b/portage_with_autodep/bin/ebuild-helpers/newlib.so index c8696f3..fd4c097 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newlib.so +++ b/portage_with_autodep/bin/ebuild-helpers/newlib.so @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec dolib.so "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newman b/portage_with_autodep/bin/ebuild-helpers/newman index ffb8a2d..889e0f9 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newman +++ b/portage_with_autodep/bin/ebuild-helpers/newman @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec doman "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/newsbin b/portage_with_autodep/bin/ebuild-helpers/newsbin index 82242aa..9df0af2 100755 --- a/portage_with_autodep/bin/ebuild-helpers/newsbin +++ b/portage_with_autodep/bin/ebuild-helpers/newsbin @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -14,6 +14,9 @@ if [ ! -e "$1" ] ; then exit 1 fi +(($#>2)) && \ + eqawarn "QA Notice: ${0##*/} called with more than 2 arguments: ${@:3}" + rm -rf "${T}/${2}" && \ cp -f "${1}" "${T}/${2}" && \ exec dosbin "${T}/${2}" diff --git a/portage_with_autodep/bin/ebuild-helpers/prepall b/portage_with_autodep/bin/ebuild-helpers/prepall index 701ecba..49e646c 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepall +++ b/portage_with_autodep/bin/ebuild-helpers/prepall @@ -4,12 +4,15 @@ source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + if has chflags $FEATURES ; then # Save all the file flags for restoration at the end of prepall. - mtree -c -p "${D}" -k flags > "${T}/bsdflags.mtree" + mtree -c -p "${ED}" -k flags > "${T}/bsdflags.mtree" # Remove all the file flags so that prepall can do anything necessary. - chflags -R noschg,nouchg,nosappnd,nouappnd "${D}" - chflags -R nosunlnk,nouunlnk "${D}" 2>/dev/null + chflags -R noschg,nouchg,nosappnd,nouappnd "${ED}" + chflags -R nosunlnk,nouunlnk "${ED}" 2>/dev/null fi prepallman @@ -19,5 +22,5 @@ prepallstrip if has chflags $FEATURES ; then # Restore all the file flags that were saved at the beginning of prepall. - mtree -U -e -p "${D}" -k flags < "${T}/bsdflags.mtree" &> /dev/null + mtree -U -e -p "${ED}" -k flags < "${T}/bsdflags.mtree" &> /dev/null fi diff --git a/portage_with_autodep/bin/ebuild-helpers/prepalldocs b/portage_with_autodep/bin/ebuild-helpers/prepalldocs index fdc735d..560a02b 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepalldocs +++ b/portage_with_autodep/bin/ebuild-helpers/prepalldocs @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh @@ -8,8 +8,10 @@ if [[ -n $1 ]] ; then vecho "${0##*/}: invalid usage; takes no arguments" 1>&2 fi -cd "${D}" -[[ -d usr/share/doc ]] || exit 0 +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +[[ -d ${ED}usr/share/doc ]] || exit 0 ecompressdir --ignore /usr/share/doc/${PF}/html ecompressdir --queue /usr/share/doc diff --git a/portage_with_autodep/bin/ebuild-helpers/prepallinfo b/portage_with_autodep/bin/ebuild-helpers/prepallinfo index 0d97803..db9bbfa 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepallinfo +++ b/portage_with_autodep/bin/ebuild-helpers/prepallinfo @@ -1,9 +1,12 @@ #!/bin/bash -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh -[[ ! -d ${D}usr/share/info ]] && exit 0 +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +[[ -d ${ED}usr/share/info ]] || exit 0 exec prepinfo diff --git a/portage_with_autodep/bin/ebuild-helpers/prepallman b/portage_with_autodep/bin/ebuild-helpers/prepallman index e50de6d..dee1c72 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepallman +++ b/portage_with_autodep/bin/ebuild-helpers/prepallman @@ -7,11 +7,14 @@ source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh # replaced by controllable compression in EAPI 4 has "${EAPI}" 0 1 2 3 || exit 0 +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + ret=0 -find "${D}" -type d -name man > "${T}"/prepallman.filelist +find "${ED}" -type d -name man > "${T}"/prepallman.filelist while read -r mandir ; do - mandir=${mandir#${D}} + mandir=${mandir#${ED}} prepman "${mandir%/man}" ((ret|=$?)) done < "${T}"/prepallman.filelist diff --git a/portage_with_autodep/bin/ebuild-helpers/prepallstrip b/portage_with_autodep/bin/ebuild-helpers/prepallstrip index ec12ce6..28320d9 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepallstrip +++ b/portage_with_autodep/bin/ebuild-helpers/prepallstrip @@ -1,5 +1,8 @@ #!/bin/bash -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -exec prepstrip "${D}" +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + +exec prepstrip "${ED}" diff --git a/portage_with_autodep/bin/ebuild-helpers/prepinfo b/portage_with_autodep/bin/ebuild-helpers/prepinfo index 691fd13..ffe2ece 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepinfo +++ b/portage_with_autodep/bin/ebuild-helpers/prepinfo @@ -4,17 +4,20 @@ source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + if [[ -z $1 ]] ; then infodir="/usr/share/info" else - if [[ -d ${D}$1/share/info ]] ; then + if [[ -d ${ED}$1/share/info ]] ; then infodir="$1/share/info" else infodir="$1/info" fi fi -if [[ ! -d ${D}${infodir} ]] ; then +if [[ ! -d ${ED}${infodir} ]] ; then if [[ -n $1 ]] ; then vecho "${0##*/}: '${infodir}' does not exist!" exit 1 @@ -23,7 +26,7 @@ if [[ ! -d ${D}${infodir} ]] ; then fi fi -find "${D}${infodir}" -type d -print0 | while read -r -d $'\0' x ; do +find "${ED}${infodir}" -type d -print0 | while read -r -d $'\0' x ; do for f in "${x}"/.keepinfodir*; do [[ -e ${f} ]] && continue 2 done diff --git a/portage_with_autodep/bin/ebuild-helpers/preplib b/portage_with_autodep/bin/ebuild-helpers/preplib index 76aabe6..6e91cf3 100755 --- a/portage_with_autodep/bin/ebuild-helpers/preplib +++ b/portage_with_autodep/bin/ebuild-helpers/preplib @@ -1,11 +1,14 @@ #!/bin/bash -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh eqawarn "QA Notice: Deprecated call to 'preplib'" +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + LIBDIR_VAR="LIBDIR_${ABI}" if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then CONF_LIBDIR="${!LIBDIR_VAR}" @@ -18,9 +21,9 @@ if [ -z "${CONF_LIBDIR}" ]; then fi if [ -z "$1" ] ; then - z="${D}usr/${CONF_LIBDIR}" + z="${ED}usr/${CONF_LIBDIR}" else - z="${D}$1/${CONF_LIBDIR}" + z="${ED}$1/${CONF_LIBDIR}" fi if [ -d "${z}" ] ; then diff --git a/portage_with_autodep/bin/ebuild-helpers/prepman b/portage_with_autodep/bin/ebuild-helpers/prepman index c9add8a..f96b641 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepman +++ b/portage_with_autodep/bin/ebuild-helpers/prepman @@ -4,14 +4,17 @@ source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) ED=${D} ;; esac + if [[ -z $1 ]] ; then - mandir="${D}usr/share/man" + mandir="${ED}usr/share/man" else - mandir="${D}$1/man" + mandir="${ED}$1/man" fi if [[ ! -d ${mandir} ]] ; then - eqawarn "QA Notice: prepman called with non-existent dir '${mandir#${D}}'" + eqawarn "QA Notice: prepman called with non-existent dir '${mandir#${ED}}'" exit 0 fi @@ -27,6 +30,6 @@ for subdir in "${mandir}"/man* "${mandir}"/*/man* ; do [[ -d ${subdir} ]] && really_is_mandir=1 && break done -[[ ${really_is_mandir} == 1 ]] && exec ecompressdir --queue "${mandir#${D}}" +[[ ${really_is_mandir} == 1 ]] && exec ecompressdir --queue "${mandir#${ED}}" exit 0 diff --git a/portage_with_autodep/bin/ebuild-helpers/prepstrip b/portage_with_autodep/bin/ebuild-helpers/prepstrip index d25259d..85d5d6a 100755 --- a/portage_with_autodep/bin/ebuild-helpers/prepstrip +++ b/portage_with_autodep/bin/ebuild-helpers/prepstrip @@ -1,85 +1,153 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/helper-functions.sh + +# avoid multiple calls to `has`. this creates things like: +# FEATURES_foo=false +# if "foo" is not in $FEATURES +tf() { "$@" && echo true || echo false ; } +exp_tf() { + local flag var=$1 + shift + for flag in "$@" ; do + eval ${var}_${flag}=$(tf has ${flag} ${!var}) + done +} +exp_tf FEATURES compressdebug installsources nostrip splitdebug +exp_tf RESTRICT binchecks installsources strip + +[[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "${EAPI}" in 0|1|2) EPREFIX= ED=${D} ;; esac banner=false SKIP_STRIP=false -if has nostrip ${FEATURES} || \ - has strip ${RESTRICT} -then +if ${RESTRICT_strip} || ${FEATURES_nostrip} ; then SKIP_STRIP=true banner=true - has installsources ${FEATURES} || exit 0 + ${FEATURES_installsources} || exit 0 fi -STRIP=${STRIP:-${CHOST}-strip} -type -P -- ${STRIP} > /dev/null || STRIP=strip -OBJCOPY=${OBJCOPY:-${CHOST}-objcopy} -type -P -- ${OBJCOPY} > /dev/null || OBJCOPY=objcopy +# look up the tools we might be using +for t in STRIP:strip OBJCOPY:objcopy READELF:readelf ; do + v=${t%:*} # STRIP + t=${t#*:} # strip + eval ${v}=\"${!v:-${CHOST}-${t}}\" + type -P -- ${!v} >/dev/null || eval ${v}=${t} +done -# We'll leave out -R .note for now until we can check out the relevance -# of the section when it has the ALLOC flag set on it ... -export SAFE_STRIP_FLAGS="--strip-unneeded" -export PORTAGE_STRIP_FLAGS=${PORTAGE_STRIP_FLAGS-${SAFE_STRIP_FLAGS} -R .comment} -prepstrip_sources_dir=/usr/src/debug/${CATEGORY}/${PF} +# Figure out what tool set we're using to strip stuff +unset SAFE_STRIP_FLAGS DEF_STRIP_FLAGS SPLIT_STRIP_FLAGS +case $(${STRIP} --version 2>/dev/null) in +*elfutils*) # dev-libs/elfutils + # elfutils default behavior is always safe, so don't need to specify + # any flags at all + SAFE_STRIP_FLAGS="" + DEF_STRIP_FLAGS="--remove-comment" + SPLIT_STRIP_FLAGS="-f" + ;; +*GNU*) # sys-devel/binutils + # We'll leave out -R .note for now until we can check out the relevance + # of the section when it has the ALLOC flag set on it ... + SAFE_STRIP_FLAGS="--strip-unneeded" + DEF_STRIP_FLAGS="-R .comment -R .GCC.command.line" + SPLIT_STRIP_FLAGS= + ;; +esac +: ${PORTAGE_STRIP_FLAGS=${SAFE_STRIP_FLAGS} ${DEF_STRIP_FLAGS}} -if has installsources ${FEATURES} && ! type -P debugedit >/dev/null ; then - ewarn "FEATURES=installsources is enabled but the debugedit binary could not" - ewarn "be found. This feature will not work unless debugedit is installed!" -fi +prepstrip_sources_dir=${EPREFIX}/usr/src/debug/${CATEGORY}/${PF} + +type -P debugedit >/dev/null && debugedit_found=true || debugedit_found=false +debugedit_warned=false -unset ${!INODE_*} +multijob_init -inode_var_name() { - if [[ $USERLAND = BSD ]] ; then - stat -f 'INODE_%d_%i' "$1" +# Setup $T filesystem layout that we care about. +tmpdir="${T}/prepstrip" +rm -rf "${tmpdir}" +mkdir -p "${tmpdir}"/{inodes,splitdebug,sources} + +# Usage: inode_var_name: <file> +inode_file_link() { + echo -n "${tmpdir}/inodes/" + if [[ ${USERLAND} == "BSD" ]] ; then + stat -f '%i' "$1" else - stat -c 'INODE_%d_%i' "$1" + stat -c '%i' "$1" fi } +# Usage: save_elf_sources <elf> save_elf_sources() { - has installsources ${FEATURES} || return 0 - has installsources ${RESTRICT} && return 0 - type -P debugedit >/dev/null || return 0 + ${FEATURES_installsources} || return 0 + ${RESTRICT_installsources} && return 0 + if ! ${debugedit_found} ; then + if ! ${debugedit_warned} ; then + debugedit_warned=true + ewarn "FEATURES=installsources is enabled but the debugedit binary could not" + ewarn "be found. This feature will not work unless debugedit is installed!" + fi + return 0 + fi local x=$1 - local inode=$(inode_var_name "$x") - [[ -n ${!inode} ]] && return 0 - debugedit -b "${WORKDIR}" -d "${prepstrip_sources_dir}" \ - -l "${T}"/debug.sources "${x}" + [[ -f $(inode_file_link "${x}") ]] && return 0 + + # since we're editing the ELF here, we should recompute the build-id + # (the -i flag below). save that output so we don't need to recompute + # it later on in the save_elf_debug step. + buildid=$(debugedit -i \ + -b "${WORKDIR}" \ + -d "${prepstrip_sources_dir}" \ + -l "${tmpdir}/sources/${x##*/}.${BASHPID}" \ + "${x}") } +# Usage: save_elf_debug <elf> [splitdebug file] save_elf_debug() { - has splitdebug ${FEATURES} || return 0 + ${FEATURES_splitdebug} || return 0 + # NOTE: Debug files must be installed in + # ${EPREFIX}/usr/lib/debug/${EPREFIX} (note that ${EPREFIX} occurs + # twice in this path) in order for gdb's debug-file-directory + # lookup to work correctly. local x=$1 - local y="${D}usr/lib/debug/${x:${#D}}.debug" + local splitdebug=$2 + local y=${ED}usr/lib/debug/${x:${#D}}.debug # dont save debug info twice [[ ${x} == *".debug" ]] && return 0 - # this will recompute the build-id, but for now that's ok - local buildid="$( type -P debugedit >/dev/null && debugedit -i "${x}" )" - - mkdir -p $(dirname "${y}") + mkdir -p "${y%/*}" - local inode=$(inode_var_name "$x") - if [[ -n ${!inode} ]] ; then - ln "${D}usr/lib/debug/${!inode:${#D}}.debug" "$y" + local inode=$(inode_file_link "${x}") + if [[ -f ${inode} ]] ; then + ln "${inode}" "${y}" else - eval $inode=\$x - ${OBJCOPY} --only-keep-debug "${x}" "${y}" - ${OBJCOPY} --add-gnu-debuglink="${y}" "${x}" - [[ -g ${x} ]] && chmod go-r "${y}" - [[ -u ${x} ]] && chmod go-r "${y}" - chmod a-x,o-w "${y}" + if [[ -n ${splitdebug} ]] ; then + mv "${splitdebug}" "${y}" + else + local objcopy_flags="--only-keep-debug" + ${FEATURES_compressdebug} && objcopy_flags+=" --compress-debug-sections" + ${OBJCOPY} ${objcopy_flags} "${x}" "${y}" + ${OBJCOPY} --add-gnu-debuglink="${y}" "${x}" + fi + local args="a-x,o-w" + [[ -g ${x} || -u ${x} ]] && args+=",go-r" + chmod ${args} "${y}" + ln "${y}" "${inode}" fi + # if we don't already have build-id from debugedit, look it up + if [[ -z ${buildid} ]] ; then + # convert the readelf output to something useful + buildid=$(${READELF} -x .note.gnu.build-id "${x}" 2>/dev/null \ + | awk '$NF ~ /GNU/ { getline; printf $2$3$4$5; getline; print $2 }') + fi if [[ -n ${buildid} ]] ; then - local buildid_dir="${D}usr/lib/debug/.build-id/${buildid:0:2}" + local buildid_dir="${ED}usr/lib/debug/.build-id/${buildid:0:2}" local buildid_file="${buildid_dir}/${buildid:2}" mkdir -p "${buildid_dir}" ln -s "../../${x:${#D}}.debug" "${buildid_file}.debug" @@ -87,33 +155,78 @@ save_elf_debug() { fi } +# Usage: process_elf <elf> +process_elf() { + local x=$1 strip_flags=${*:2} + + vecho " ${x:${#ED}}" + save_elf_sources "${x}" + + if ${strip_this} ; then + + # If two processes try to strip the same hardlink at the same + # time, it will cause one of them to lose the splitdebug info. + # So, use a lockfile to prevent interference (easily observed + # with dev-vcs/git which creates ~109 hardlinks to one file in + # /usr/libexec/git-core). + local lockfile=$(inode_file_link "${x}")_lockfile + if ! ln "${x}" "${lockfile}" ; then + while [[ -f ${lockfile} ]] ; do + sleep 1 + done + unset lockfile + fi + + # see if we can split & strip at the same time + if [[ -n ${SPLIT_STRIP_FLAGS} ]] ; then + local shortname="${x##*/}.debug" + local splitdebug="${tmpdir}/splitdebug/${shortname}.${BASHPID}" + ${STRIP} ${strip_flags} \ + -f "${splitdebug}" \ + -F "${shortname}" \ + "${x}" + save_elf_debug "${x}" "${splitdebug}" + else + save_elf_debug "${x}" + ${STRIP} ${strip_flags} "${x}" + fi + [[ -n ${lockfile} ]] && rm -f "${lockfile}" + fi +} + # The existance of the section .symtab tells us that a binary is stripped. # We want to log already stripped binaries, as this may be a QA violation. # They prevent us from getting the splitdebug data. -if ! has binchecks ${RESTRICT} && \ - ! has strip ${RESTRICT} ; then - log=$T/scanelf-already-stripped.log +if ! ${RESTRICT_binchecks} && ! ${RESTRICT_strip} ; then + # We need to do the non-stripped scan serially first before we turn around + # and start stripping the files ourselves. The log parsing can be done in + # parallel though. + log=${tmpdir}/scanelf-already-stripped.log + scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^${ED}##" > "${log}" + ( + multijob_child_init qa_var="QA_PRESTRIPPED_${ARCH/-/_}" [[ -n ${!qa_var} ]] && QA_PRESTRIPPED="${!qa_var}" - scanelf -yqRBF '#k%F' -k '!.symtab' "$@" | sed -e "s#^$D##" > "$log" - if [[ -n $QA_PRESTRIPPED && -s $log && \ + if [[ -n ${QA_PRESTRIPPED} && -s ${log} && \ ${QA_STRICT_PRESTRIPPED-unset} = unset ]] ; then shopts=$- set -o noglob - for x in $QA_PRESTRIPPED ; do - sed -e "s#^${x#/}\$##" -i "$log" + for x in ${QA_PRESTRIPPED} ; do + sed -e "s#^${x#/}\$##" -i "${log}" done set +o noglob - set -$shopts + set -${shopts} fi - sed -e "/^\$/d" -e "s#^#/#" -i "$log" - if [[ -s $log ]] ; then + sed -e "/^\$/d" -e "s#^#/#" -i "${log}" + if [[ -s ${log} ]] ; then vecho -e "\n" eqawarn "QA Notice: Pre-stripped files found:" - eqawarn "$(<"$log")" + eqawarn "$(<"${log}")" else - rm -f "$log" + rm -f "${log}" fi + ) & + multijob_post_fork fi # Now we look for unstripped binaries. @@ -126,8 +239,10 @@ do banner=true fi - f=$(file "${x}") || continue - [[ -z ${f} ]] && continue + ( + multijob_child_init + f=$(file "${x}") || exit 0 + [[ -z ${f} ]] && exit 0 if ! ${SKIP_STRIP} ; then # The noglob funk is to support STRIP_MASK="/*/booga" and to keep @@ -136,50 +251,61 @@ do set -o noglob strip_this=true for m in $(eval echo ${STRIP_MASK}) ; do - [[ /${x#${D}} == ${m} ]] && strip_this=false && break + [[ /${x#${ED}} == ${m} ]] && strip_this=false && break done set +o noglob else strip_this=false fi + # In Prefix we are usually an unprivileged user, so we can't strip + # unwritable objects. Make them temporarily writable for the + # stripping. + was_not_writable=false + if [[ ! -w ${x} ]] ; then + was_not_writable=true + chmod u+w "${x}" + fi + # only split debug info for final linked objects # or kernel modules as debuginfo for intermediatary # files (think crt*.o from gcc/glibc) is useless and # actually causes problems. install sources for all # elf types though cause that stuff is good. + buildid= if [[ ${f} == *"current ar archive"* ]] ; then - vecho " ${x:${#D}}" + vecho " ${x:${#ED}}" if ${strip_this} ; then # hmm, can we split debug/sources for .a ? ${STRIP} -g "${x}" fi elif [[ ${f} == *"SB executable"* || ${f} == *"SB shared object"* ]] ; then - vecho " ${x:${#D}}" - save_elf_sources "${x}" - if ${strip_this} ; then - save_elf_debug "${x}" - ${STRIP} ${PORTAGE_STRIP_FLAGS} "${x}" - fi + process_elf "${x}" ${PORTAGE_STRIP_FLAGS} elif [[ ${f} == *"SB relocatable"* ]] ; then - vecho " ${x:${#D}}" - save_elf_sources "${x}" - if ${strip_this} ; then - [[ ${x} == *.ko ]] && save_elf_debug "${x}" - ${STRIP} ${SAFE_STRIP_FLAGS} "${x}" - fi + process_elf "${x}" ${SAFE_STRIP_FLAGS} fi + + if ${was_not_writable} ; then + chmod u-w "${x}" + fi + ) & + multijob_post_fork done -if [[ -s ${T}/debug.sources ]] && \ - has installsources ${FEATURES} && \ - ! has installsources ${RESTRICT} && \ - type -P debugedit >/dev/null +# With a bit more work, we could run the rsync processes below in +# parallel, but not sure that'd be an overall improvement. +multijob_finish + +cd "${tmpdir}"/sources/ && cat * > "${tmpdir}/debug.sources" 2>/dev/null +if [[ -s ${tmpdir}/debug.sources ]] && \ + ${FEATURES_installsources} && \ + ! ${RESTRICT_installsources} && \ + ${debugedit_found} then vecho "installsources: rsyncing source files" [[ -d ${D}${prepstrip_sources_dir} ]] || mkdir -p "${D}${prepstrip_sources_dir}" - grep -zv '/<[^/>]*>$' "${T}"/debug.sources | \ + grep -zv '/<[^/>]*>$' "${tmpdir}"/debug.sources | \ (cd "${WORKDIR}"; LANG=C sort -z -u | \ rsync -tL0 --files-from=- "${WORKDIR}/" "${D}${prepstrip_sources_dir}/" ) @@ -188,6 +314,8 @@ then # https://bugzilla.redhat.com/show_bug.cgi?id=444310 while read -r -d $'\0' emptydir do - >> "$emptydir"/.keepdir + >> "${emptydir}"/.keepdir done < <(find "${D}${prepstrip_sources_dir}/" -type d -empty -print0) fi + +rm -rf "${tmpdir}" diff --git a/portage_with_autodep/bin/ebuild-helpers/unprivileged/chgrp b/portage_with_autodep/bin/ebuild-helpers/unprivileged/chgrp new file mode 120000 index 0000000..6fb0fcd --- /dev/null +++ b/portage_with_autodep/bin/ebuild-helpers/unprivileged/chgrp @@ -0,0 +1 @@ +chown
\ No newline at end of file diff --git a/portage_with_autodep/bin/ebuild-helpers/unprivileged/chown b/portage_with_autodep/bin/ebuild-helpers/unprivileged/chown new file mode 100755 index 0000000..08fa650 --- /dev/null +++ b/portage_with_autodep/bin/ebuild-helpers/unprivileged/chown @@ -0,0 +1,41 @@ +#!/bin/bash +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +scriptpath=${BASH_SOURCE[0]} +scriptname=${scriptpath##*/} + +IFS=':' + +for path in ${PATH}; do + [[ -x ${path}/${scriptname} ]] || continue + [[ ${path}/${scriptname} -ef ${scriptpath} ]] && continue + IFS=$' \t\n' + output=$("${path}/${scriptname}" "$@" 2>&1) + if [[ $? -ne 0 ]] ; then + + # Avoid an extreme performance problem when the + # output is very long (bug #470992). + if [[ $(wc -l <<< "${output}") -gt 100 ]]; then + output=$(head -n100 <<< "${output}") + output="${output}\n ... (further messages truncated)" + fi + + source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh + + if ! ___eapi_has_prefix_variables; then + EPREFIX= + fi + msg="${scriptname} failure ignored with unprivileged user:\n ${scriptname} $*\n ${output}" + # Reverse expansion of ${D} and ${EPREFIX}, for readability. + msg=${msg//${D}/'${D}'} + if [[ -n ${EPREFIX} ]] ; then + msg=${msg//${EPREFIX}/'${EPREFIX}'} + msg=${msg//${EPREFIX#/}/'${EPREFIX}'} + fi + ewarn "${msg}" + fi + exit 0 +done + +exit 1 diff --git a/portage_with_autodep/bin/ebuild-helpers/xattr/install b/portage_with_autodep/bin/ebuild-helpers/xattr/install new file mode 100755 index 0000000..f51f621 --- /dev/null +++ b/portage_with_autodep/bin/ebuild-helpers/xattr/install @@ -0,0 +1,12 @@ +#!/bin/bash +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +PORTAGE_BIN_PATH=${PORTAGE_BIN_PATH:-/usr/lib/portage/bin} +PORTAGE_PYM_PATH=${PORTAGE_PYM_PATH:-/usr/lib/portage/pym} +# Use safe cwd, avoiding unsafe import for bug #469338. +export __PORTAGE_HELPER_CWD=${PWD} +cd "${PORTAGE_PYM_PATH}" +export __PORTAGE_HELPER_PATH=${BASH_SOURCE[0]} +PYTHONPATH=${PORTAGE_PYTHONPATH:-${PORTAGE_PYM_PATH}} \ + exec "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/install.py" "$@" diff --git a/portage_with_autodep/bin/ebuild-ipc.py b/portage_with_autodep/bin/ebuild-ipc.py index 68ad985..29d4c23 100755 --- a/portage_with_autodep/bin/ebuild-ipc.py +++ b/portage_with_autodep/bin/ebuild-ipc.py @@ -13,6 +13,7 @@ import select import signal import sys import time +import traceback def debug_signal(signum, frame): import pdb @@ -228,14 +229,22 @@ class EbuildIpc(object): pid = os.fork() if pid == 0: - os.close(pr) - - # File streams are in unbuffered mode since we do atomic - # read and write of whole pickles. - output_file = open(self.ipc_in_fifo, 'wb', 0) - output_file.write(pickle.dumps(args)) - output_file.close() - os._exit(os.EX_OK) + retval = 2 + try: + os.close(pr) + + # File streams are in unbuffered mode since we do atomic + # read and write of whole pickles. + output_file = open(self.ipc_in_fifo, 'wb', 0) + output_file.write(pickle.dumps(args)) + output_file.close() + retval = os.EX_OK + except SystemExit: + raise + except: + traceback.print_exc() + finally: + os._exit(retval) os.close(pw) @@ -258,9 +267,16 @@ class EbuildIpc(object): pid = os.fork() if pid == 0: - os.close(pr) - retval = self._receive_reply(input_fd) - os._exit(retval) + retval = 2 + try: + os.close(pr) + retval = self._receive_reply(input_fd) + except SystemExit: + raise + except: + traceback.print_exc() + finally: + os._exit(retval) os.close(pw) retval = self._wait(pid, pr, portage.localization._('during read')) diff --git a/portage_with_autodep/bin/ebuild.sh b/portage_with_autodep/bin/ebuild.sh index d68e54b..2589113 100755 --- a/portage_with_autodep/bin/ebuild.sh +++ b/portage_with_autodep/bin/ebuild.sh @@ -1,17 +1,53 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 PORTAGE_BIN_PATH="${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}" PORTAGE_PYM_PATH="${PORTAGE_PYM_PATH:-/usr/lib/portage/pym}" -if [[ $PORTAGE_SANDBOX_COMPAT_LEVEL -lt 22 ]] ; then - # Ensure that /dev/std* streams have appropriate sandbox permission for - # bug #288863. This can be removed after sandbox is fixed and portage - # depends on the fixed version (sandbox-2.2 has the fix but it is - # currently unstable). - export SANDBOX_WRITE="${SANDBOX_WRITE:+${SANDBOX_WRITE}:}/dev/stdout:/dev/stderr" - export SANDBOX_READ="${SANDBOX_READ:+${SANDBOX_READ}:}/dev/stdin" +# Prevent aliases from causing portage to act inappropriately. +# Make sure it's before everything so we don't mess aliases that follow. +unalias -a + +source "${PORTAGE_BIN_PATH}/isolated-functions.sh" || exit 1 + +if [[ $EBUILD_PHASE != depend ]] ; then + source "${PORTAGE_BIN_PATH}/phase-functions.sh" || die + source "${PORTAGE_BIN_PATH}/save-ebuild-env.sh" || die + source "${PORTAGE_BIN_PATH}/phase-helpers.sh" || die + source "${PORTAGE_BIN_PATH}/bashrc-functions.sh" || die +else + # These dummy functions are for things that are likely to be called + # in global scope, even though they are completely useless during + # the "depend" phase. + for x in diropts docompress exeopts get_KV insopts \ + keepdir KV_major KV_micro KV_minor KV_to_int \ + libopts register_die_hook register_success_hook \ + remove_path_entry set_unless_changed strip_duplicate_slashes \ + unset_unless_changed use_with use_enable ; do + eval "${x}() { + if has \"\${EAPI:-0}\" 4-python; then + die \"\${FUNCNAME}() calls are not allowed in global scope\" + fi + }" + done + # These dummy functions return false in older EAPIs, in order to ensure that + # `use multislot` is false for the "depend" phase. + for x in use useq usev ; do + eval "${x}() { + if has \"\${EAPI:-0}\" 4-python; then + die \"\${FUNCNAME}() calls are not allowed in global scope\" + else + return 1 + fi + }" + done + # These functions die because calls to them during the "depend" phase + # are considered to be severe QA violations. + for x in best_version has_version portageq ; do + eval "${x}() { die \"\${FUNCNAME}() calls are not allowed in global scope\"; }" + done + unset x fi # Don't use sandbox's BASH_ENV for new shells because it does @@ -19,13 +55,6 @@ fi # environment by modifying our PATH. unset BASH_ENV -ROOTPATH=${ROOTPATH##:} -ROOTPATH=${ROOTPATH%%:} -PREROOTPATH=${PREROOTPATH##:} -PREROOTPATH=${PREROOTPATH%%:} -PATH=$PORTAGE_BIN_PATH/ebuild-helpers:$PREROOTPATH${PREROOTPATH:+:}/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin${ROOTPATH:+:}$ROOTPATH -export PATH - # This is just a temporary workaround for portage-9999 users since # earlier portage versions do not detect a version change in this case # (9999 to 9999) and therefore they try execute an incompatible version of @@ -67,15 +96,9 @@ EBUILD_SH_ARGS="$*" shift $# -# Prevent aliases from causing portage to act inappropriately. -# Make sure it's before everything so we don't mess aliases that follow. -unalias -a - # Unset some variables that break things. unset GZIP BZIP BZIP2 CDPATH GREP_OPTIONS GREP_COLOR GLOBIGNORE -source "${PORTAGE_BIN_PATH}/isolated-functions.sh" &>/dev/null - [[ $PORTAGE_QUIET != "" ]] && export PORTAGE_QUIET # sandbox support functions; defined prior to profile.bashrc srcing, since the profile might need to add a default exception (/usr/lib64/conftest fex) @@ -113,191 +136,12 @@ fi # the sandbox is disabled by default except when overridden in the relevant stages export SANDBOX_ON=0 -lchown() { - chown -h "$@" -} - -lchgrp() { - chgrp -h "$@" -} - esyslog() { # Custom version of esyslog() to take care of the "Red Star" bug. # MUST follow functions.sh to override the "" parameter problem. return 0 } -useq() { - has $EBUILD_PHASE prerm postrm || eqawarn \ - "QA Notice: The 'useq' function is deprecated (replaced by 'use')" - use ${1} -} - -usev() { - if use ${1}; then - echo "${1#!}" - return 0 - fi - return 1 -} - -use() { - local u=$1 - local found=0 - - # if we got something like '!flag', then invert the return value - if [[ ${u:0:1} == "!" ]] ; then - u=${u:1} - found=1 - fi - - if [[ $EBUILD_PHASE = depend ]] ; then - # TODO: Add a registration interface for eclasses to register - # any number of phase hooks, so that global scope eclass - # initialization can by migrated to phase hooks in new EAPIs. - # Example: add_phase_hook before pkg_setup $ECLASS_pre_pkg_setup - #if [[ -n $EAPI ]] && ! has "$EAPI" 0 1 2 3 ; then - # die "use() called during invalid phase: $EBUILD_PHASE" - #fi - true - - # Make sure we have this USE flag in IUSE - elif [[ -n $PORTAGE_IUSE && -n $EBUILD_PHASE ]] ; then - [[ $u =~ $PORTAGE_IUSE ]] || \ - eqawarn "QA Notice: USE Flag '${u}' not" \ - "in IUSE for ${CATEGORY}/${PF}" - fi - - if has ${u} ${USE} ; then - return ${found} - else - return $((!found)) - fi -} - -# Return true if given package is installed. Otherwise return false. -# Takes single depend-type atoms. -has_version() { - if [ "${EBUILD_PHASE}" == "depend" ]; then - die "portageq calls (has_version calls portageq) are not allowed in the global scope" - fi - - if [[ -n $PORTAGE_IPC_DAEMON ]] ; then - "$PORTAGE_BIN_PATH"/ebuild-ipc has_version "$ROOT" "$1" - else - PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ - "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" has_version "${ROOT}" "$1" - fi - local retval=$? - case "${retval}" in - 0|1) - return ${retval} - ;; - *) - die "unexpected portageq exit code: ${retval}" - ;; - esac -} - -portageq() { - if [ "${EBUILD_PHASE}" == "depend" ]; then - die "portageq calls are not allowed in the global scope" - fi - - PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ - "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" "$@" -} - - -# ---------------------------------------------------------------------------- -# ---------------------------------------------------------------------------- -# ---------------------------------------------------------------------------- - - -# Returns the best/most-current match. -# Takes single depend-type atoms. -best_version() { - if [ "${EBUILD_PHASE}" == "depend" ]; then - die "portageq calls (best_version calls portageq) are not allowed in the global scope" - fi - - if [[ -n $PORTAGE_IPC_DAEMON ]] ; then - "$PORTAGE_BIN_PATH"/ebuild-ipc best_version "$ROOT" "$1" - else - PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ - "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" 'best_version' "${ROOT}" "$1" - fi - local retval=$? - case "${retval}" in - 0|1) - return ${retval} - ;; - *) - die "unexpected portageq exit code: ${retval}" - ;; - esac -} - -use_with() { - if [ -z "$1" ]; then - echo "!!! use_with() called without a parameter." >&2 - echo "!!! use_with <USEFLAG> [<flagname> [value]]" >&2 - return 1 - fi - - if ! has "${EAPI:-0}" 0 1 2 3 ; then - local UW_SUFFIX=${3+=$3} - else - local UW_SUFFIX=${3:+=$3} - fi - local UWORD=${2:-$1} - - if use $1; then - echo "--with-${UWORD}${UW_SUFFIX}" - else - echo "--without-${UWORD}" - fi - return 0 -} - -use_enable() { - if [ -z "$1" ]; then - echo "!!! use_enable() called without a parameter." >&2 - echo "!!! use_enable <USEFLAG> [<flagname> [value]]" >&2 - return 1 - fi - - if ! has "${EAPI:-0}" 0 1 2 3 ; then - local UE_SUFFIX=${3+=$3} - else - local UE_SUFFIX=${3:+=$3} - fi - local UWORD=${2:-$1} - - if use $1; then - echo "--enable-${UWORD}${UE_SUFFIX}" - else - echo "--disable-${UWORD}" - fi - return 0 -} - -register_die_hook() { - local x - for x in $* ; do - has $x $EBUILD_DEATH_HOOKS || \ - export EBUILD_DEATH_HOOKS="$EBUILD_DEATH_HOOKS $x" - done -} - -register_success_hook() { - local x - for x in $* ; do - has $x $EBUILD_SUCCESS_HOOKS || \ - export EBUILD_SUCCESS_HOOKS="$EBUILD_SUCCESS_HOOKS $x" - done -} - # Ensure that $PWD is sane whenever possible, to protect against # exploitation of insecure search path for python -c in ebuilds. # See bug #239560. @@ -309,985 +153,6 @@ fi #if no perms are specified, dirs/files will have decent defaults #(not secretive, but not stupid) umask 022 -export DESTTREE=/usr -export INSDESTTREE="" -export _E_EXEDESTTREE_="" -export _E_DOCDESTTREE_="" -export INSOPTIONS="-m0644" -export EXEOPTIONS="-m0755" -export LIBOPTIONS="-m0644" -export DIROPTIONS="-m0755" -export MOPREFIX=${PN} -declare -a PORTAGE_DOCOMPRESS=( /usr/share/{doc,info,man} ) -declare -a PORTAGE_DOCOMPRESS_SKIP=( /usr/share/doc/${PF}/html ) - -# adds ".keep" files so that dirs aren't auto-cleaned -keepdir() { - dodir "$@" - local x - if [ "$1" == "-R" ] || [ "$1" == "-r" ]; then - shift - find "$@" -type d -printf "${D}%p/.keep_${CATEGORY}_${PN}-${SLOT}\n" \ - | tr "\n" "\0" | \ - while read -r -d $'\0' ; do - >> "$REPLY" || \ - die "Failed to recursively create .keep files" - done - else - for x in "$@"; do - >> "${D}${x}/.keep_${CATEGORY}_${PN}-${SLOT}" || \ - die "Failed to create .keep in ${D}${x}" - done - fi -} - -unpack() { - local srcdir - local x - local y - local myfail - local eapi=${EAPI:-0} - [ -z "$*" ] && die "Nothing passed to the 'unpack' command" - - for x in "$@"; do - vecho ">>> Unpacking ${x} to ${PWD}" - y=${x%.*} - y=${y##*.} - - if [[ ${x} == "./"* ]] ; then - srcdir="" - elif [[ ${x} == ${DISTDIR%/}/* ]] ; then - die "Arguments to unpack() cannot begin with \${DISTDIR}." - elif [[ ${x} == "/"* ]] ; then - die "Arguments to unpack() cannot be absolute" - else - srcdir="${DISTDIR}/" - fi - [[ ! -s ${srcdir}${x} ]] && die "${x} does not exist" - - _unpack_tar() { - if [ "${y}" == "tar" ]; then - $1 -c -- "$srcdir$x" | tar xof - - assert_sigpipe_ok "$myfail" - else - local cwd_dest=${x##*/} - cwd_dest=${cwd_dest%.*} - $1 -c -- "${srcdir}${x}" > "${cwd_dest}" || die "$myfail" - fi - } - - myfail="failure unpacking ${x}" - case "${x##*.}" in - tar) - tar xof "$srcdir$x" || die "$myfail" - ;; - tgz) - tar xozf "$srcdir$x" || die "$myfail" - ;; - tbz|tbz2) - ${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -c -- "$srcdir$x" | tar xof - - assert_sigpipe_ok "$myfail" - ;; - ZIP|zip|jar) - # unzip will interactively prompt under some error conditions, - # as reported in bug #336285 - ( while true ; do echo n || break ; done ) | \ - unzip -qo "${srcdir}${x}" || die "$myfail" - ;; - gz|Z|z) - _unpack_tar "gzip -d" - ;; - bz2|bz) - _unpack_tar "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}" - ;; - 7Z|7z) - local my_output - my_output="$(7z x -y "${srcdir}${x}")" - if [ $? -ne 0 ]; then - echo "${my_output}" >&2 - die "$myfail" - fi - ;; - RAR|rar) - unrar x -idq -o+ "${srcdir}${x}" || die "$myfail" - ;; - LHa|LHA|lha|lzh) - lha xfq "${srcdir}${x}" || die "$myfail" - ;; - a) - ar x "${srcdir}${x}" || die "$myfail" - ;; - deb) - # Unpacking .deb archives can not always be done with - # `ar`. For instance on AIX this doesn't work out. If - # we have `deb2targz` installed, prefer it over `ar` for - # that reason. We just make sure on AIX `deb2targz` is - # installed. - if type -P deb2targz > /dev/null; then - y=${x##*/} - local created_symlink=0 - if [ ! "$srcdir$x" -ef "$y" ] ; then - # deb2targz always extracts into the same directory as - # the source file, so create a symlink in the current - # working directory if necessary. - ln -sf "$srcdir$x" "$y" || die "$myfail" - created_symlink=1 - fi - deb2targz "$y" || die "$myfail" - if [ $created_symlink = 1 ] ; then - # Clean up the symlink so the ebuild - # doesn't inadvertently install it. - rm -f "$y" - fi - mv -f "${y%.deb}".tar.gz data.tar.gz || die "$myfail" - else - ar x "$srcdir$x" || die "$myfail" - fi - ;; - lzma) - _unpack_tar "lzma -d" - ;; - xz) - if has $eapi 0 1 2 ; then - vecho "unpack ${x}: file format not recognized. Ignoring." - else - _unpack_tar "xz -d" - fi - ;; - *) - vecho "unpack ${x}: file format not recognized. Ignoring." - ;; - esac - done - # Do not chmod '.' since it's probably ${WORKDIR} and PORTAGE_WORKDIR_MODE - # should be preserved. - find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ - ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w -} - -strip_duplicate_slashes() { - if [[ -n $1 ]] ; then - local removed=$1 - while [[ ${removed} == *//* ]] ; do - removed=${removed//\/\///} - done - echo ${removed} - fi -} - -hasg() { - local x s=$1 - shift - for x ; do [[ ${x} == ${s} ]] && echo "${x}" && return 0 ; done - return 1 -} -hasgq() { hasg "$@" >/dev/null ; } -econf() { - local x - - local phase_func=$(_ebuild_arg_to_phase "$EAPI" "$EBUILD_PHASE") - if [[ -n $phase_func ]] ; then - if has "$EAPI" 0 1 ; then - [[ $phase_func != src_compile ]] && \ - eqawarn "QA Notice: econf called in" \ - "$phase_func instead of src_compile" - else - [[ $phase_func != src_configure ]] && \ - eqawarn "QA Notice: econf called in" \ - "$phase_func instead of src_configure" - fi - fi - - : ${ECONF_SOURCE:=.} - if [ -x "${ECONF_SOURCE}/configure" ]; then - if [[ -n $CONFIG_SHELL && \ - "$(head -n1 "$ECONF_SOURCE/configure")" =~ ^'#!'[[:space:]]*/bin/sh([[:space:]]|$) ]] ; then - sed -e "1s:^#![[:space:]]*/bin/sh:#!$CONFIG_SHELL:" -i "$ECONF_SOURCE/configure" || \ - die "Substition of shebang in '$ECONF_SOURCE/configure' failed" - fi - if [ -e /usr/share/gnuconfig/ ]; then - find "${WORKDIR}" -type f '(' \ - -name config.guess -o -name config.sub ')' -print0 | \ - while read -r -d $'\0' x ; do - vecho " * econf: updating ${x/${WORKDIR}\/} with /usr/share/gnuconfig/${x##*/}" - cp -f /usr/share/gnuconfig/"${x##*/}" "${x}" - done - fi - - # EAPI=4 adds --disable-dependency-tracking to econf - if ! has "$EAPI" 0 1 2 3 3_pre2 && \ - "${ECONF_SOURCE}/configure" --help 2>/dev/null | \ - grep -q disable-dependency-tracking ; then - set -- --disable-dependency-tracking "$@" - fi - - # if the profile defines a location to install libs to aside from default, pass it on. - # if the ebuild passes in --libdir, they're responsible for the conf_libdir fun. - local CONF_LIBDIR LIBDIR_VAR="LIBDIR_${ABI}" - if [[ -n ${ABI} && -n ${!LIBDIR_VAR} ]] ; then - CONF_LIBDIR=${!LIBDIR_VAR} - fi - if [[ -n ${CONF_LIBDIR} ]] && ! hasgq --libdir=\* "$@" ; then - export CONF_PREFIX=$(hasg --exec-prefix=\* "$@") - [[ -z ${CONF_PREFIX} ]] && CONF_PREFIX=$(hasg --prefix=\* "$@") - : ${CONF_PREFIX:=/usr} - CONF_PREFIX=${CONF_PREFIX#*=} - [[ ${CONF_PREFIX} != /* ]] && CONF_PREFIX="/${CONF_PREFIX}" - [[ ${CONF_LIBDIR} != /* ]] && CONF_LIBDIR="/${CONF_LIBDIR}" - set -- --libdir="$(strip_duplicate_slashes ${CONF_PREFIX}${CONF_LIBDIR})" "$@" - fi - - set -- \ - --prefix=/usr \ - ${CBUILD:+--build=${CBUILD}} \ - --host=${CHOST} \ - ${CTARGET:+--target=${CTARGET}} \ - --mandir=/usr/share/man \ - --infodir=/usr/share/info \ - --datadir=/usr/share \ - --sysconfdir=/etc \ - --localstatedir=/var/lib \ - "$@" \ - ${EXTRA_ECONF} - vecho "${ECONF_SOURCE}/configure" "$@" - - if ! "${ECONF_SOURCE}/configure" "$@" ; then - - if [ -s config.log ]; then - echo - echo "!!! Please attach the following file when seeking support:" - echo "!!! ${PWD}/config.log" - fi - die "econf failed" - fi - elif [ -f "${ECONF_SOURCE}/configure" ]; then - die "configure is not executable" - else - die "no configure script found" - fi -} - -einstall() { - # CONF_PREFIX is only set if they didn't pass in libdir above. - local LOCAL_EXTRA_EINSTALL="${EXTRA_EINSTALL}" - LIBDIR_VAR="LIBDIR_${ABI}" - if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then - CONF_LIBDIR="${!LIBDIR_VAR}" - fi - unset LIBDIR_VAR - if [ -n "${CONF_LIBDIR}" ] && [ "${CONF_PREFIX:+set}" = set ]; then - EI_DESTLIBDIR="${D}/${CONF_PREFIX}/${CONF_LIBDIR}" - EI_DESTLIBDIR="$(strip_duplicate_slashes ${EI_DESTLIBDIR})" - LOCAL_EXTRA_EINSTALL="libdir=${EI_DESTLIBDIR} ${LOCAL_EXTRA_EINSTALL}" - unset EI_DESTLIBDIR - fi - - if [ -f ./[mM]akefile -o -f ./GNUmakefile ] ; then - if [ "${PORTAGE_DEBUG}" == "1" ]; then - ${MAKE:-make} -n prefix="${D}usr" \ - datadir="${D}usr/share" \ - infodir="${D}usr/share/info" \ - localstatedir="${D}var/lib" \ - mandir="${D}usr/share/man" \ - sysconfdir="${D}etc" \ - ${LOCAL_EXTRA_EINSTALL} \ - ${MAKEOPTS} ${EXTRA_EMAKE} -j1 \ - "$@" install - fi - ${MAKE:-make} prefix="${D}usr" \ - datadir="${D}usr/share" \ - infodir="${D}usr/share/info" \ - localstatedir="${D}var/lib" \ - mandir="${D}usr/share/man" \ - sysconfdir="${D}etc" \ - ${LOCAL_EXTRA_EINSTALL} \ - ${MAKEOPTS} ${EXTRA_EMAKE} -j1 \ - "$@" install || die "einstall failed" - else - die "no Makefile found" - fi -} - -_eapi0_pkg_nofetch() { - [ -z "${SRC_URI}" ] && return - - elog "The following are listed in SRC_URI for ${PN}:" - local x - for x in $(echo ${SRC_URI}); do - elog " ${x}" - done -} - -_eapi0_src_unpack() { - [[ -n ${A} ]] && unpack ${A} -} - -_eapi0_src_compile() { - if [ -x ./configure ] ; then - econf - fi - _eapi2_src_compile -} - -_eapi0_src_test() { - # Since we don't want emake's automatic die - # support (EAPI 4 and later), and we also don't - # want the warning messages that it produces if - # we call it in 'nonfatal' mode, we use emake_cmd - # to emulate the desired parts of emake behavior. - local emake_cmd="${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE}" - if $emake_cmd -j1 check -n &> /dev/null; then - vecho ">>> Test phase [check]: ${CATEGORY}/${PF}" - if ! $emake_cmd -j1 check; then - has test $FEATURES && die "Make check failed. See above for details." - has test $FEATURES || eerror "Make check failed. See above for details." - fi - elif $emake_cmd -j1 test -n &> /dev/null; then - vecho ">>> Test phase [test]: ${CATEGORY}/${PF}" - if ! $emake_cmd -j1 test; then - has test $FEATURES && die "Make test failed. See above for details." - has test $FEATURES || eerror "Make test failed. See above for details." - fi - else - vecho ">>> Test phase [none]: ${CATEGORY}/${PF}" - fi -} - -_eapi1_src_compile() { - _eapi2_src_configure - _eapi2_src_compile -} - -_eapi2_src_configure() { - if [[ -x ${ECONF_SOURCE:-.}/configure ]] ; then - econf - fi -} - -_eapi2_src_compile() { - if [ -f Makefile ] || [ -f GNUmakefile ] || [ -f makefile ]; then - emake || die "emake failed" - fi -} - -_eapi4_src_install() { - if [[ -f Makefile || -f GNUmakefile || -f makefile ]] ; then - emake DESTDIR="${D}" install - fi - - if ! declare -p DOCS &>/dev/null ; then - local d - for d in README* ChangeLog AUTHORS NEWS TODO CHANGES \ - THANKS BUGS FAQ CREDITS CHANGELOG ; do - [[ -s "${d}" ]] && dodoc "${d}" - done - elif [[ $(declare -p DOCS) == "declare -a "* ]] ; then - dodoc "${DOCS[@]}" - else - dodoc ${DOCS} - fi -} - -ebuild_phase() { - declare -F "$1" >/dev/null && qa_call $1 -} - -ebuild_phase_with_hooks() { - local x phase_name=${1} - for x in {pre_,,post_}${phase_name} ; do - ebuild_phase ${x} - done -} - -dyn_pretend() { - if [[ -e $PORTAGE_BUILDDIR/.pretended ]] ; then - vecho ">>> It appears that '$PF' is already pretended; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.pretended' to force pretend." - return 0 - fi - ebuild_phase pre_pkg_pretend - ebuild_phase pkg_pretend - >> "$PORTAGE_BUILDDIR/.pretended" || \ - die "Failed to create $PORTAGE_BUILDDIR/.pretended" - ebuild_phase post_pkg_pretend -} - -dyn_setup() { - if [[ -e $PORTAGE_BUILDDIR/.setuped ]] ; then - vecho ">>> It appears that '$PF' is already setup; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.setuped' to force setup." - return 0 - fi - ebuild_phase pre_pkg_setup - ebuild_phase pkg_setup - >> "$PORTAGE_BUILDDIR/.setuped" || \ - die "Failed to create $PORTAGE_BUILDDIR/.setuped" - ebuild_phase post_pkg_setup -} - -dyn_unpack() { - local newstuff="no" - if [ -e "${WORKDIR}" ]; then - local x - local checkme - for x in $A ; do - vecho ">>> Checking ${x}'s mtime..." - if [ "${PORTAGE_ACTUAL_DISTDIR:-${DISTDIR}}/${x}" -nt "${WORKDIR}" ]; then - vecho ">>> ${x} has been updated; recreating WORKDIR..." - newstuff="yes" - break - fi - done - if [ ! -f "${PORTAGE_BUILDDIR}/.unpacked" ] ; then - vecho ">>> Not marked as unpacked; recreating WORKDIR..." - newstuff="yes" - fi - fi - if [ "${newstuff}" == "yes" ]; then - # We don't necessarily have privileges to do a full dyn_clean here. - rm -rf "${PORTAGE_BUILDDIR}"/{.setuped,.unpacked,.prepared,.configured,.compiled,.tested,.installed,.packaged,build-info} - if ! has keepwork $FEATURES ; then - rm -rf "${WORKDIR}" - fi - if [ -d "${T}" ] && \ - ! has keeptemp $FEATURES ; then - rm -rf "${T}" && mkdir "${T}" - fi - fi - if [ -e "${WORKDIR}" ]; then - if [ "$newstuff" == "no" ]; then - vecho ">>> WORKDIR is up-to-date, keeping..." - return 0 - fi - fi - - if [ ! -d "${WORKDIR}" ]; then - install -m${PORTAGE_WORKDIR_MODE:-0700} -d "${WORKDIR}" || die "Failed to create dir '${WORKDIR}'" - fi - cd "${WORKDIR}" || die "Directory change failed: \`cd '${WORKDIR}'\`" - ebuild_phase pre_src_unpack - vecho ">>> Unpacking source..." - ebuild_phase src_unpack - >> "$PORTAGE_BUILDDIR/.unpacked" || \ - die "Failed to create $PORTAGE_BUILDDIR/.unpacked" - vecho ">>> Source unpacked in ${WORKDIR}" - ebuild_phase post_src_unpack -} - -dyn_clean() { - if [ -z "${PORTAGE_BUILDDIR}" ]; then - echo "Aborting clean phase because PORTAGE_BUILDDIR is unset!" - return 1 - elif [ ! -d "${PORTAGE_BUILDDIR}" ] ; then - return 0 - fi - if has chflags $FEATURES ; then - chflags -R noschg,nouchg,nosappnd,nouappnd "${PORTAGE_BUILDDIR}" - chflags -R nosunlnk,nouunlnk "${PORTAGE_BUILDDIR}" 2>/dev/null - fi - - rm -rf "${PORTAGE_BUILDDIR}/image" "${PORTAGE_BUILDDIR}/homedir" - rm -f "${PORTAGE_BUILDDIR}/.installed" - - if [[ $EMERGE_FROM = binary ]] || \ - ! has keeptemp $FEATURES && ! has keepwork $FEATURES ; then - rm -rf "${T}" - fi - - if [[ $EMERGE_FROM = binary ]] || ! has keepwork $FEATURES; then - rm -f "$PORTAGE_BUILDDIR"/.{ebuild_changed,logid,pretended,setuped,unpacked,prepared} \ - "$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged} \ - "$PORTAGE_BUILDDIR"/.die_hooks \ - "$PORTAGE_BUILDDIR"/.ipc_{in,out,lock} \ - "$PORTAGE_BUILDDIR"/.exit_status - - rm -rf "${PORTAGE_BUILDDIR}/build-info" - rm -rf "${WORKDIR}" - fi - - if [ -f "${PORTAGE_BUILDDIR}/.unpacked" ]; then - find "${PORTAGE_BUILDDIR}" -type d ! -regex "^${WORKDIR}" | sort -r | tr "\n" "\0" | $XARGS -0 rmdir &>/dev/null - fi - - # do not bind this to doebuild defined DISTDIR; don't trust doebuild, and if mistakes are made it'll - # result in it wiping the users distfiles directory (bad). - rm -rf "${PORTAGE_BUILDDIR}/distdir" - - # Some kernels, such as Solaris, return EINVAL when an attempt - # is made to remove the current working directory. - cd "$PORTAGE_BUILDDIR"/../.. - rmdir "$PORTAGE_BUILDDIR" 2>/dev/null - - true -} - -into() { - if [ "$1" == "/" ]; then - export DESTTREE="" - else - export DESTTREE=$1 - if [ ! -d "${D}${DESTTREE}" ]; then - install -d "${D}${DESTTREE}" - local ret=$? - if [[ $ret -ne 0 ]] ; then - helpers_die "${FUNCNAME[0]} failed" - return $ret - fi - fi - fi -} - -insinto() { - if [ "$1" == "/" ]; then - export INSDESTTREE="" - else - export INSDESTTREE=$1 - if [ ! -d "${D}${INSDESTTREE}" ]; then - install -d "${D}${INSDESTTREE}" - local ret=$? - if [[ $ret -ne 0 ]] ; then - helpers_die "${FUNCNAME[0]} failed" - return $ret - fi - fi - fi -} - -exeinto() { - if [ "$1" == "/" ]; then - export _E_EXEDESTTREE_="" - else - export _E_EXEDESTTREE_="$1" - if [ ! -d "${D}${_E_EXEDESTTREE_}" ]; then - install -d "${D}${_E_EXEDESTTREE_}" - local ret=$? - if [[ $ret -ne 0 ]] ; then - helpers_die "${FUNCNAME[0]} failed" - return $ret - fi - fi - fi -} - -docinto() { - if [ "$1" == "/" ]; then - export _E_DOCDESTTREE_="" - else - export _E_DOCDESTTREE_="$1" - if [ ! -d "${D}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" ]; then - install -d "${D}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" - local ret=$? - if [[ $ret -ne 0 ]] ; then - helpers_die "${FUNCNAME[0]} failed" - return $ret - fi - fi - fi -} - -insopts() { - export INSOPTIONS="$@" - - # `install` should never be called with '-s' ... - has -s ${INSOPTIONS} && die "Never call insopts() with -s" -} - -diropts() { - export DIROPTIONS="$@" -} - -exeopts() { - export EXEOPTIONS="$@" - - # `install` should never be called with '-s' ... - has -s ${EXEOPTIONS} && die "Never call exeopts() with -s" -} - -libopts() { - export LIBOPTIONS="$@" - - # `install` should never be called with '-s' ... - has -s ${LIBOPTIONS} && die "Never call libopts() with -s" -} - -docompress() { - has "${EAPI}" 0 1 2 3 && die "'docompress' not supported in this EAPI" - - local f g - if [[ $1 = "-x" ]]; then - shift - for f; do - f=$(strip_duplicate_slashes "${f}"); f=${f%/} - [[ ${f:0:1} = / ]] || f="/${f}" - for g in "${PORTAGE_DOCOMPRESS_SKIP[@]}"; do - [[ ${f} = "${g}" ]] && continue 2 - done - PORTAGE_DOCOMPRESS_SKIP[${#PORTAGE_DOCOMPRESS_SKIP[@]}]=${f} - done - else - for f; do - f=$(strip_duplicate_slashes "${f}"); f=${f%/} - [[ ${f:0:1} = / ]] || f="/${f}" - for g in "${PORTAGE_DOCOMPRESS[@]}"; do - [[ ${f} = "${g}" ]] && continue 2 - done - PORTAGE_DOCOMPRESS[${#PORTAGE_DOCOMPRESS[@]}]=${f} - done - fi -} - -abort_handler() { - local msg - if [ "$2" != "fail" ]; then - msg="${EBUILD}: ${1} aborted; exiting." - else - msg="${EBUILD}: ${1} failed; exiting." - fi - echo - echo "$msg" - echo - eval ${3} - #unset signal handler - trap - SIGINT SIGQUIT -} - -abort_prepare() { - abort_handler src_prepare $1 - rm -f "$PORTAGE_BUILDDIR/.prepared" - exit 1 -} - -abort_configure() { - abort_handler src_configure $1 - rm -f "$PORTAGE_BUILDDIR/.configured" - exit 1 -} - -abort_compile() { - abort_handler "src_compile" $1 - rm -f "${PORTAGE_BUILDDIR}/.compiled" - exit 1 -} - -abort_test() { - abort_handler "dyn_test" $1 - rm -f "${PORTAGE_BUILDDIR}/.tested" - exit 1 -} - -abort_install() { - abort_handler "src_install" $1 - rm -rf "${PORTAGE_BUILDDIR}/image" - exit 1 -} - -has_phase_defined_up_to() { - local phase - for phase in unpack prepare configure compile install; do - has ${phase} ${DEFINED_PHASES} && return 0 - [[ ${phase} == $1 ]] && return 1 - done - # We shouldn't actually get here - return 1 -} - -dyn_prepare() { - - if [[ -e $PORTAGE_BUILDDIR/.prepared ]] ; then - vecho ">>> It appears that '$PF' is already prepared; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.prepared' to force prepare." - return 0 - fi - - if [[ -d $S ]] ; then - cd "${S}" - elif has $EAPI 0 1 2 3 3_pre2 ; then - cd "${WORKDIR}" - elif [[ -z ${A} ]] && ! has_phase_defined_up_to prepare; then - cd "${WORKDIR}" - else - die "The source directory '${S}' doesn't exist" - fi - - trap abort_prepare SIGINT SIGQUIT - - ebuild_phase pre_src_prepare - vecho ">>> Preparing source in $PWD ..." - ebuild_phase src_prepare - >> "$PORTAGE_BUILDDIR/.prepared" || \ - die "Failed to create $PORTAGE_BUILDDIR/.prepared" - vecho ">>> Source prepared." - ebuild_phase post_src_prepare - - trap - SIGINT SIGQUIT -} - -dyn_configure() { - - if [[ -e $PORTAGE_BUILDDIR/.configured ]] ; then - vecho ">>> It appears that '$PF' is already configured; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.configured' to force configuration." - return 0 - fi - - if [[ -d $S ]] ; then - cd "${S}" - elif has $EAPI 0 1 2 3 3_pre2 ; then - cd "${WORKDIR}" - elif [[ -z ${A} ]] && ! has_phase_defined_up_to configure; then - cd "${WORKDIR}" - else - die "The source directory '${S}' doesn't exist" - fi - - trap abort_configure SIGINT SIGQUIT - - ebuild_phase pre_src_configure - - vecho ">>> Configuring source in $PWD ..." - ebuild_phase src_configure - >> "$PORTAGE_BUILDDIR/.configured" || \ - die "Failed to create $PORTAGE_BUILDDIR/.configured" - vecho ">>> Source configured." - - ebuild_phase post_src_configure - - trap - SIGINT SIGQUIT -} - -dyn_compile() { - - if [[ -e $PORTAGE_BUILDDIR/.compiled ]] ; then - vecho ">>> It appears that '${PF}' is already compiled; skipping." - vecho ">>> Remove '$PORTAGE_BUILDDIR/.compiled' to force compilation." - return 0 - fi - - if [[ -d $S ]] ; then - cd "${S}" - elif has $EAPI 0 1 2 3 3_pre2 ; then - cd "${WORKDIR}" - elif [[ -z ${A} ]] && ! has_phase_defined_up_to compile; then - cd "${WORKDIR}" - else - die "The source directory '${S}' doesn't exist" - fi - - trap abort_compile SIGINT SIGQUIT - - if has distcc $FEATURES && has distcc-pump $FEATURES ; then - if [[ -z $INCLUDE_SERVER_PORT ]] || [[ ! -w $INCLUDE_SERVER_PORT ]] ; then - eval $(pump --startup) - trap "pump --shutdown" EXIT - fi - fi - - ebuild_phase pre_src_compile - - vecho ">>> Compiling source in $PWD ..." - ebuild_phase src_compile - >> "$PORTAGE_BUILDDIR/.compiled" || \ - die "Failed to create $PORTAGE_BUILDDIR/.compiled" - vecho ">>> Source compiled." - - ebuild_phase post_src_compile - - trap - SIGINT SIGQUIT -} - -dyn_test() { - - if [[ -e $PORTAGE_BUILDDIR/.tested ]] ; then - vecho ">>> It appears that ${PN} has already been tested; skipping." - vecho ">>> Remove '${PORTAGE_BUILDDIR}/.tested' to force test." - return - fi - - if [ "${EBUILD_FORCE_TEST}" == "1" ] ; then - # If USE came from ${T}/environment then it might not have USE=test - # like it's supposed to here. - ! has test ${USE} && export USE="${USE} test" - fi - - trap "abort_test" SIGINT SIGQUIT - if [ -d "${S}" ]; then - cd "${S}" - else - cd "${WORKDIR}" - fi - - if ! has test $FEATURES && [ "${EBUILD_FORCE_TEST}" != "1" ]; then - vecho ">>> Test phase [not enabled]: ${CATEGORY}/${PF}" - elif has test $RESTRICT; then - einfo "Skipping make test/check due to ebuild restriction." - vecho ">>> Test phase [explicitly disabled]: ${CATEGORY}/${PF}" - else - local save_sp=${SANDBOX_PREDICT} - addpredict / - ebuild_phase pre_src_test - ebuild_phase src_test - >> "$PORTAGE_BUILDDIR/.tested" || \ - die "Failed to create $PORTAGE_BUILDDIR/.tested" - ebuild_phase post_src_test - SANDBOX_PREDICT=${save_sp} - fi - - trap - SIGINT SIGQUIT -} - -dyn_install() { - [ -z "$PORTAGE_BUILDDIR" ] && die "${FUNCNAME}: PORTAGE_BUILDDIR is unset" - if has noauto $FEATURES ; then - rm -f "${PORTAGE_BUILDDIR}/.installed" - elif [[ -e $PORTAGE_BUILDDIR/.installed ]] ; then - vecho ">>> It appears that '${PF}' is already installed; skipping." - vecho ">>> Remove '${PORTAGE_BUILDDIR}/.installed' to force install." - return 0 - fi - trap "abort_install" SIGINT SIGQUIT - ebuild_phase pre_src_install - rm -rf "${PORTAGE_BUILDDIR}/image" - mkdir "${PORTAGE_BUILDDIR}/image" - if [[ -d $S ]] ; then - cd "${S}" - elif has $EAPI 0 1 2 3 3_pre2 ; then - cd "${WORKDIR}" - elif [[ -z ${A} ]] && ! has_phase_defined_up_to install; then - cd "${WORKDIR}" - else - die "The source directory '${S}' doesn't exist" - fi - - vecho - vecho ">>> Install ${PF} into ${D} category ${CATEGORY}" - #our custom version of libtool uses $S and $D to fix - #invalid paths in .la files - export S D - - # Reset exeinto(), docinto(), insinto(), and into() state variables - # in case the user is running the install phase multiple times - # consecutively via the ebuild command. - export DESTTREE=/usr - export INSDESTTREE="" - export _E_EXEDESTTREE_="" - export _E_DOCDESTTREE_="" - - ebuild_phase src_install - >> "$PORTAGE_BUILDDIR/.installed" || \ - die "Failed to create $PORTAGE_BUILDDIR/.installed" - vecho ">>> Completed installing ${PF} into ${D}" - vecho - ebuild_phase post_src_install - - cd "${PORTAGE_BUILDDIR}"/build-info - set -f - local f x - IFS=$' \t\n\r' - for f in CATEGORY DEFINED_PHASES FEATURES INHERITED IUSE REQUIRED_USE \ - PF PKGUSE SLOT KEYWORDS HOMEPAGE DESCRIPTION ; do - x=$(echo -n ${!f}) - [[ -n $x ]] && echo "$x" > $f - done - if [[ $CATEGORY != virtual ]] ; then - for f in ASFLAGS CBUILD CC CFLAGS CHOST CTARGET CXX \ - CXXFLAGS EXTRA_ECONF EXTRA_EINSTALL EXTRA_MAKE \ - LDFLAGS LIBCFLAGS LIBCXXFLAGS ; do - x=$(echo -n ${!f}) - [[ -n $x ]] && echo "$x" > $f - done - fi - echo "${USE}" > USE - echo "${EAPI:-0}" > EAPI - set +f - - # local variables can leak into the saved environment. - unset f - - save_ebuild_env --exclude-init-phases | filter_readonly_variables \ - --filter-path --filter-sandbox --allow-extra-vars > environment - assert "save_ebuild_env failed" - - ${PORTAGE_BZIP2_COMMAND} -f9 environment - - cp "${EBUILD}" "${PF}.ebuild" - [ -n "${PORTAGE_REPO_NAME}" ] && echo "${PORTAGE_REPO_NAME}" > repository - if has nostrip ${FEATURES} ${RESTRICT} || has strip ${RESTRICT} - then - >> DEBUGBUILD - fi - trap - SIGINT SIGQUIT -} - -dyn_preinst() { - if [ -z "${D}" ]; then - eerror "${FUNCNAME}: D is unset" - return 1 - fi - ebuild_phase_with_hooks pkg_preinst -} - -dyn_help() { - echo - echo "Portage" - echo "Copyright 1999-2010 Gentoo Foundation" - echo - echo "How to use the ebuild command:" - echo - echo "The first argument to ebuild should be an existing .ebuild file." - echo - echo "One or more of the following options can then be specified. If more" - echo "than one option is specified, each will be executed in order." - echo - echo " help : show this help screen" - echo " pretend : execute package specific pretend actions" - echo " setup : execute package specific setup actions" - echo " fetch : download source archive(s) and patches" - echo " digest : create a manifest file for the package" - echo " manifest : create a manifest file for the package" - echo " unpack : unpack sources (auto-dependencies if needed)" - echo " prepare : prepare sources (auto-dependencies if needed)" - echo " configure : configure sources (auto-fetch/unpack if needed)" - echo " compile : compile sources (auto-fetch/unpack/configure if needed)" - echo " test : test package (auto-fetch/unpack/configure/compile if needed)" - echo " preinst : execute pre-install instructions" - echo " postinst : execute post-install instructions" - echo " install : install the package to the temporary install directory" - echo " qmerge : merge image into live filesystem, recording files in db" - echo " merge : do fetch, unpack, compile, install and qmerge" - echo " prerm : execute pre-removal instructions" - echo " postrm : execute post-removal instructions" - echo " unmerge : remove package from live filesystem" - echo " config : execute package specific configuration actions" - echo " package : create a tarball package in ${PKGDIR}/All" - echo " rpm : build a RedHat RPM package" - echo " clean : clean up all source and temporary files" - echo - echo "The following settings will be used for the ebuild process:" - echo - echo " package : ${PF}" - echo " slot : ${SLOT}" - echo " category : ${CATEGORY}" - echo " description : ${DESCRIPTION}" - echo " system : ${CHOST}" - echo " c flags : ${CFLAGS}" - echo " c++ flags : ${CXXFLAGS}" - echo " make flags : ${MAKEOPTS}" - echo -n " build mode : " - if has nostrip ${FEATURES} ${RESTRICT} || has strip ${RESTRICT} ; - then - echo "debug (large)" - else - echo "production (stripped)" - fi - echo " merge to : ${ROOT}" - echo - if [ -n "$USE" ]; then - echo "Additionally, support for the following optional features will be enabled:" - echo - echo " ${USE}" - fi - echo -} # debug-print() gets called from many places with verbose status information useful # for tracking down problems. The output is in $T/eclass-debug.log. @@ -1365,11 +230,15 @@ inherit() { unset $__export_funcs_var if [ "${EBUILD_PHASE}" != "depend" ] && \ + [ "${EBUILD_PHASE}" != "nofetch" ] && \ [[ ${EBUILD_PHASE} != *rm ]] && \ [[ ${EMERGE_FROM} != "binary" ]] ; then # This is disabled in the *rm phases because they frequently give # false alarms due to INHERITED in /var/db/pkg being outdated - # in comparison the the eclasses from the portage tree. + # in comparison the the eclasses from the portage tree. It's + # disabled for nofetch, since that can be called by repoman and + # that triggers bug #407449 due to repoman not exporting + # non-essential variables such as INHERITED. if ! has $ECLASS $INHERITED $__INHERITED_QA_CACHE ; then eqawarn "QA Notice: ECLASS '$ECLASS' inherited illegally in $CATEGORY/$PF $EBUILD_PHASE" fi @@ -1418,11 +287,11 @@ inherit() { # If each var has a value, append it to the global variable E_* to # be applied after everything is finished. New incremental behavior. - [ "${IUSE+set}" = set ] && export E_IUSE="${E_IUSE} ${IUSE}" - [ "${REQUIRED_USE+set}" = set ] && export E_REQUIRED_USE="${E_REQUIRED_USE} ${REQUIRED_USE}" - [ "${DEPEND+set}" = set ] && export E_DEPEND="${E_DEPEND} ${DEPEND}" - [ "${RDEPEND+set}" = set ] && export E_RDEPEND="${E_RDEPEND} ${RDEPEND}" - [ "${PDEPEND+set}" = set ] && export E_PDEPEND="${E_PDEPEND} ${PDEPEND}" + [ "${IUSE+set}" = set ] && E_IUSE+="${E_IUSE:+ }${IUSE}" + [ "${REQUIRED_USE+set}" = set ] && E_REQUIRED_USE+="${E_REQUIRED_USE:+ }${REQUIRED_USE}" + [ "${DEPEND+set}" = set ] && E_DEPEND+="${E_DEPEND:+ }${DEPEND}" + [ "${RDEPEND+set}" = set ] && E_RDEPEND+="${E_RDEPEND:+ }${RDEPEND}" + [ "${PDEPEND+set}" = set ] && E_PDEPEND+="${E_PDEPEND:+ }${PDEPEND}" [ "${B_IUSE+set}" = set ] && IUSE="${B_IUSE}" [ "${B_IUSE+set}" = set ] || unset IUSE @@ -1477,214 +346,6 @@ EXPORT_FUNCTIONS() { eval $__export_funcs_var+=\" $*\" } -# this is a function for removing any directory matching a passed in pattern from -# PATH -remove_path_entry() { - save_IFS - IFS=":" - stripped_path="${PATH}" - while [ -n "$1" ]; do - cur_path="" - for p in ${stripped_path}; do - if [ "${p/${1}}" == "${p}" ]; then - cur_path="${cur_path}:${p}" - fi - done - stripped_path="${cur_path#:*}" - shift - done - restore_IFS - PATH="${stripped_path}" -} - -# @FUNCTION: _ebuild_arg_to_phase -# @DESCRIPTION: -# Translate a known ebuild(1) argument into the precise -# name of it's corresponding ebuild phase. -_ebuild_arg_to_phase() { - [ $# -ne 2 ] && die "expected exactly 2 args, got $#: $*" - local eapi=$1 - local arg=$2 - local phase_func="" - - case "$arg" in - pretend) - ! has $eapi 0 1 2 3 3_pre2 && \ - phase_func=pkg_pretend - ;; - setup) - phase_func=pkg_setup - ;; - nofetch) - phase_func=pkg_nofetch - ;; - unpack) - phase_func=src_unpack - ;; - prepare) - ! has $eapi 0 1 && \ - phase_func=src_prepare - ;; - configure) - ! has $eapi 0 1 && \ - phase_func=src_configure - ;; - compile) - phase_func=src_compile - ;; - test) - phase_func=src_test - ;; - install) - phase_func=src_install - ;; - preinst) - phase_func=pkg_preinst - ;; - postinst) - phase_func=pkg_postinst - ;; - prerm) - phase_func=pkg_prerm - ;; - postrm) - phase_func=pkg_postrm - ;; - esac - - [[ -z $phase_func ]] && return 1 - echo "$phase_func" - return 0 -} - -_ebuild_phase_funcs() { - [ $# -ne 2 ] && die "expected exactly 2 args, got $#: $*" - local eapi=$1 - local phase_func=$2 - local default_phases="pkg_nofetch src_unpack src_prepare src_configure - src_compile src_install src_test" - local x y default_func="" - - for x in pkg_nofetch src_unpack src_test ; do - declare -F $x >/dev/null || \ - eval "$x() { _eapi0_$x \"\$@\" ; }" - done - - case $eapi in - - 0|1) - - if ! declare -F src_compile >/dev/null ; then - case $eapi in - 0) - src_compile() { _eapi0_src_compile "$@" ; } - ;; - *) - src_compile() { _eapi1_src_compile "$@" ; } - ;; - esac - fi - - for x in $default_phases ; do - eval "default_$x() { - die \"default_$x() is not supported with EAPI='$eapi' during phase $phase_func\" - }" - done - - eval "default() { - die \"default() is not supported with EAPI='$eapi' during phase $phase_func\" - }" - - ;; - - *) - - declare -F src_configure >/dev/null || \ - src_configure() { _eapi2_src_configure "$@" ; } - - declare -F src_compile >/dev/null || \ - src_compile() { _eapi2_src_compile "$@" ; } - - has $eapi 2 3 3_pre2 || declare -F src_install >/dev/null || \ - src_install() { _eapi4_src_install "$@" ; } - - if has $phase_func $default_phases ; then - - _eapi2_pkg_nofetch () { _eapi0_pkg_nofetch "$@" ; } - _eapi2_src_unpack () { _eapi0_src_unpack "$@" ; } - _eapi2_src_prepare () { true ; } - _eapi2_src_test () { _eapi0_src_test "$@" ; } - _eapi2_src_install () { die "$FUNCNAME is not supported" ; } - - for x in $default_phases ; do - eval "default_$x() { _eapi2_$x \"\$@\" ; }" - done - - eval "default() { _eapi2_$phase_func \"\$@\" ; }" - - case $eapi in - 2|3) - ;; - *) - eval "default_src_install() { _eapi4_src_install \"\$@\" ; }" - [[ $phase_func = src_install ]] && \ - eval "default() { _eapi4_$phase_func \"\$@\" ; }" - ;; - esac - - else - - for x in $default_phases ; do - eval "default_$x() { - die \"default_$x() is not supported in phase $default_func\" - }" - done - - eval "default() { - die \"default() is not supported with EAPI='$eapi' during phase $phase_func\" - }" - - fi - - ;; - esac -} - -# Set given variables unless these variable have been already set (e.g. during emerge -# invocation) to values different than values set in make.conf. -set_unless_changed() { - if [[ $# -lt 1 ]]; then - die "${FUNCNAME}() requires at least 1 argument: VARIABLE=VALUE" - fi - - local argument value variable - for argument in "$@"; do - if [[ ${argument} != *=* ]]; then - die "${FUNCNAME}(): Argument '${argument}' has incorrect syntax" - fi - variable="${argument%%=*}" - value="${argument#*=}" - if eval "[[ \${${variable}} == \$(env -u ${variable} portageq envvar ${variable}) ]]"; then - eval "${variable}=\"\${value}\"" - fi - done -} - -# Unset given variables unless these variable have been set (e.g. during emerge -# invocation) to values different than values set in make.conf. -unset_unless_changed() { - if [[ $# -lt 1 ]]; then - die "${FUNCNAME}() requires at least 1 argument: VARIABLE" - fi - - local variable - for variable in "$@"; do - if eval "[[ \${${variable}} == \$(env -u ${variable} portageq envvar ${variable}) ]]"; then - unset ${variable} - fi - done -} - PORTAGE_BASHRCS_SOURCED=0 # @FUNCTION: source_all_bashrcs @@ -1716,222 +377,46 @@ source_all_bashrcs() { done fi - # We assume if people are changing shopts in their bashrc they do so at their - # own peril. This is the ONLY non-portage bit of code that can change shopts - # without a QA violation. - for x in "${PORTAGE_BASHRC}" "${PM_EBUILD_HOOK_DIR}"/${CATEGORY}/{${PN},${PN}:${SLOT},${P},${PF}}; do - if [ -r "${x}" ]; then - # If $- contains x, then tracing has already enabled elsewhere for some - # reason. We preserve it's state so as not to interfere. - if [ "$PORTAGE_DEBUG" != "1" ] || [ "${-/x/}" != "$-" ]; then - source "${x}" - else - set -x - source "${x}" - set +x - fi + if [ -r "${PORTAGE_BASHRC}" ] ; then + if [ "$PORTAGE_DEBUG" != "1" ] || [ "${-/x/}" != "$-" ]; then + source "${PORTAGE_BASHRC}" + else + set -x + source "${PORTAGE_BASHRC}" + set +x fi - done - - [ ! -z "${OCC}" ] && export CC="${OCC}" - [ ! -z "${OCXX}" ] && export CXX="${OCXX}" -} - -# Hardcoded bash lists are needed for backward compatibility with -# <portage-2.1.4 since they assume that a newly installed version -# of ebuild.sh will work for pkg_postinst, pkg_prerm, and pkg_postrm -# when portage is upgrading itself. - -PORTAGE_READONLY_METADATA="DEFINED_PHASES DEPEND DESCRIPTION - EAPI HOMEPAGE INHERITED IUSE REQUIRED_USE KEYWORDS LICENSE - PDEPEND PROVIDE RDEPEND RESTRICT SLOT SRC_URI" - -PORTAGE_READONLY_VARS="D EBUILD EBUILD_PHASE \ - EBUILD_SH_ARGS ECLASSDIR EMERGE_FROM FILESDIR MERGE_TYPE \ - PM_EBUILD_HOOK_DIR \ - PORTAGE_ACTUAL_DISTDIR PORTAGE_ARCHLIST PORTAGE_BASHRC \ - PORTAGE_BINPKG_FILE PORTAGE_BINPKG_TAR_OPTS PORTAGE_BINPKG_TMPFILE \ - PORTAGE_BIN_PATH PORTAGE_BUILDDIR PORTAGE_BUNZIP2_COMMAND \ - PORTAGE_BZIP2_COMMAND PORTAGE_COLORMAP PORTAGE_CONFIGROOT \ - PORTAGE_DEBUG PORTAGE_DEPCACHEDIR PORTAGE_EBUILD_EXIT_FILE \ - PORTAGE_GID PORTAGE_GRPNAME PORTAGE_INST_GID PORTAGE_INST_UID \ - PORTAGE_IPC_DAEMON PORTAGE_IUSE PORTAGE_LOG_FILE \ - PORTAGE_MUTABLE_FILTERED_VARS PORTAGE_PYM_PATH PORTAGE_PYTHON \ - PORTAGE_READONLY_METADATA PORTAGE_READONLY_VARS \ - PORTAGE_REPO_NAME PORTAGE_RESTRICT PORTAGE_SANDBOX_COMPAT_LEVEL \ - PORTAGE_SAVED_READONLY_VARS PORTAGE_SIGPIPE_STATUS \ - PORTAGE_TMPDIR PORTAGE_UPDATE_ENV PORTAGE_USERNAME \ - PORTAGE_VERBOSE PORTAGE_WORKDIR_MODE PORTDIR PORTDIR_OVERLAY \ - PROFILE_PATHS REPLACING_VERSIONS REPLACED_BY_VERSION T WORKDIR" - -PORTAGE_SAVED_READONLY_VARS="A CATEGORY P PF PN PR PV PVR" - -# Variables that portage sets but doesn't mark readonly. -# In order to prevent changed values from causing unexpected -# interference, they are filtered out of the environment when -# it is saved or loaded (any mutations do not persist). -PORTAGE_MUTABLE_FILTERED_VARS="AA HOSTNAME" - -# @FUNCTION: filter_readonly_variables -# @DESCRIPTION: [--filter-sandbox] [--allow-extra-vars] -# Read an environment from stdin and echo to stdout while filtering variables -# with names that are known to cause interference: -# -# * some specific variables for which bash does not allow assignment -# * some specific variables that affect portage or sandbox behavior -# * variable names that begin with a digit or that contain any -# non-alphanumeric characters that are not be supported by bash -# -# --filter-sandbox causes all SANDBOX_* variables to be filtered, which -# is only desired in certain cases, such as during preprocessing or when -# saving environment.bz2 for a binary or installed package. -# -# --filter-features causes the special FEATURES variable to be filtered. -# Generally, we want it to persist between phases since the user might -# want to modify it via bashrc to enable things like splitdebug and -# installsources for specific packages. They should be able to modify it -# in pre_pkg_setup() and have it persist all the way through the install -# phase. However, if FEATURES exist inside environment.bz2 then they -# should be overridden by current settings. -# -# --filter-locale causes locale related variables such as LANG and LC_* -# variables to be filtered. These variables should persist between phases, -# in case they are modified by the ebuild. However, the current user -# settings should be used when loading the environment from a binary or -# installed package. -# -# --filter-path causes the PATH variable to be filtered. This variable -# should persist between phases, in case it is modified by the ebuild. -# However, old settings should be overridden when loading the -# environment from a binary or installed package. -# -# ---allow-extra-vars causes some extra vars to be allowd through, such -# as ${PORTAGE_SAVED_READONLY_VARS} and ${PORTAGE_MUTABLE_FILTERED_VARS}. -# -# In bash-3.2_p20+ an attempt to assign BASH_*, FUNCNAME, GROUPS or any -# readonly variable cause the shell to exit while executing the "source" -# builtin command. To avoid this problem, this function filters those -# variables out and discards them. See bug #190128. -filter_readonly_variables() { - local x filtered_vars - local readonly_bash_vars="BASHOPTS BASHPID DIRSTACK EUID - FUNCNAME GROUPS PIPESTATUS PPID SHELLOPTS UID" - local bash_misc_vars="BASH BASH_.* COMP_WORDBREAKS HISTCMD - HISTFILE HOSTNAME HOSTTYPE IFS LINENO MACHTYPE OLDPWD - OPTERR OPTIND OSTYPE POSIXLY_CORRECT PS4 PWD RANDOM - SECONDS SHELL SHLVL" - local filtered_sandbox_vars="SANDBOX_ACTIVE SANDBOX_BASHRC - SANDBOX_DEBUG_LOG SANDBOX_DISABLED SANDBOX_LIB - SANDBOX_LOG SANDBOX_ON" - local misc_garbage_vars="_portage_filter_opts" - filtered_vars="$readonly_bash_vars $bash_misc_vars - $PORTAGE_READONLY_VARS $misc_garbage_vars" - - # Don't filter/interfere with prefix variables unless they are - # supported by the current EAPI. - case "${EAPI:-0}" in - 0|1|2) - ;; - *) - filtered_vars+=" ED EPREFIX EROOT" - ;; - esac - - if has --filter-sandbox $* ; then - filtered_vars="${filtered_vars} SANDBOX_.*" - else - filtered_vars="${filtered_vars} ${filtered_sandbox_vars}" - fi - if has --filter-features $* ; then - filtered_vars="${filtered_vars} FEATURES PORTAGE_FEATURES" fi - if has --filter-path $* ; then - filtered_vars+=" PATH" - fi - if has --filter-locale $* ; then - filtered_vars+=" LANG LC_ALL LC_COLLATE - LC_CTYPE LC_MESSAGES LC_MONETARY - LC_NUMERIC LC_PAPER LC_TIME" - fi - if ! has --allow-extra-vars $* ; then - filtered_vars=" - ${filtered_vars} - ${PORTAGE_SAVED_READONLY_VARS} - ${PORTAGE_MUTABLE_FILTERED_VARS} - " - fi - - "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}"/filter-bash-environment.py "${filtered_vars}" || die "filter-bash-environment.py failed" -} -# @FUNCTION: preprocess_ebuild_env -# @DESCRIPTION: -# Filter any readonly variables from ${T}/environment, source it, and then -# save it via save_ebuild_env(). This process should be sufficient to prevent -# any stale variables or functions from an arbitrary environment from -# interfering with the current environment. This is useful when an existing -# environment needs to be loaded from a binary or installed package. -preprocess_ebuild_env() { - local _portage_filter_opts="--filter-features --filter-locale --filter-path --filter-sandbox" - - # If environment.raw is present, this is a signal from the python side, - # indicating that the environment may contain stale FEATURES and - # SANDBOX_{DENY,PREDICT,READ,WRITE} variables that should be filtered out. - # Otherwise, we don't need to filter the environment. - [ -f "${T}/environment.raw" ] || return 0 - - filter_readonly_variables $_portage_filter_opts < "${T}"/environment \ - >> "$T/environment.filtered" || return $? - unset _portage_filter_opts - mv "${T}"/environment.filtered "${T}"/environment || return $? - rm -f "${T}/environment.success" || return $? - # WARNING: Code inside this subshell should avoid making assumptions - # about variables or functions after source "${T}"/environment has been - # called. Any variables that need to be relied upon should already be - # filtered out above. - ( - export SANDBOX_ON=1 - source "${T}/environment" || exit $? - # We have to temporarily disable sandbox since the - # SANDBOX_{DENY,READ,PREDICT,WRITE} values we've just loaded - # may be unusable (triggering in spurious sandbox violations) - # until we've merged them with our current values. - export SANDBOX_ON=0 - - # It's remotely possible that save_ebuild_env() has been overridden - # by the above source command. To protect ourselves, we override it - # here with our own version. ${PORTAGE_BIN_PATH} is safe to use here - # because it's already filtered above. - source "${PORTAGE_BIN_PATH}/isolated-functions.sh" || exit $? - - # Rely on save_ebuild_env() to filter out any remaining variables - # and functions that could interfere with the current environment. - save_ebuild_env || exit $? - >> "$T/environment.success" || exit $? - ) > "${T}/environment.filtered" - local retval - if [ -e "${T}/environment.success" ] ; then - filter_readonly_variables --filter-features < \ - "${T}/environment.filtered" > "${T}/environment" - retval=$? - else - retval=1 + if [[ $EBUILD_PHASE != depend ]] ; then + # The user's bashrc is the ONLY non-portage bit of code that can + # change shopts without a QA violation. + for x in "${PM_EBUILD_HOOK_DIR}"/${CATEGORY}/{${PN},${PN}:${SLOT},${P},${PF}}; do + if [ -r "${x}" ]; then + # If $- contains x, then tracing has already been enabled + # elsewhere for some reason. We preserve it's state so as + # not to interfere. + if [ "$PORTAGE_DEBUG" != "1" ] || [ "${-/x/}" != "$-" ]; then + source "${x}" + else + set -x + source "${x}" + set +x + fi + fi + done fi - rm -f "${T}"/environment.{filtered,raw,success} - return ${retval} + + [ ! -z "${OCC}" ] && export CC="${OCC}" + [ ! -z "${OCXX}" ] && export CXX="${OCXX}" } # === === === === === === === === === === === === === === === === === === # === === === === === functions end, main part begins === === === === === -# === === === === === functions end, main part begins === === === === === -# === === === === === functions end, main part begins === === === === === # === === === === === === === === === === === === === === === === === === export SANDBOX_ON="1" export S=${WORKDIR}/${P} -unset E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND - # Turn of extended glob matching so that g++ doesn't get incorrectly matched. shopt -u extglob @@ -1993,11 +478,12 @@ if ! has "$EBUILD_PHASE" clean cleanrm depend && \ # The environment may have been extracted from environment.bz2 or # may have come from another version of ebuild.sh or something. # In any case, preprocess it to prevent any potential interference. + # NOTE: export ${FOO}=... requires quoting, unlike normal exports preprocess_ebuild_env || \ die "error processing environment" # Colon separated SANDBOX_* variables need to be cumulative. for x in SANDBOX_DENY SANDBOX_READ SANDBOX_PREDICT SANDBOX_WRITE ; do - export PORTAGE_${x}=${!x} + export PORTAGE_${x}="${!x}" done PORTAGE_SANDBOX_ON=${SANDBOX_ON} export SANDBOX_ON=1 @@ -2011,13 +497,13 @@ if ! has "$EBUILD_PHASE" clean cleanrm depend && \ for x in SANDBOX_DENY SANDBOX_PREDICT SANDBOX_READ SANDBOX_WRITE ; do y="PORTAGE_${x}" if [ -z "${!x}" ] ; then - export ${x}=${!y} + export ${x}="${!y}" elif [ -n "${!y}" ] && [ "${!y}" != "${!x}" ] ; then # filter out dupes - export ${x}=$(printf "${!y}:${!x}" | tr ":" "\0" | \ - sort -z -u | tr "\0" ":") + export ${x}="$(printf "${!y}:${!x}" | tr ":" "\0" | \ + sort -z -u | tr "\0" ":")" fi - export ${x}=${!x%:} + export ${x}="${!x%:}" unset PORTAGE_${x} done unset x y @@ -2026,6 +512,10 @@ if ! has "$EBUILD_PHASE" clean cleanrm depend && \ [[ -n $EAPI ]] || EAPI=0 fi +if has "${EAPI:-0}" 4-python; then + shopt -s globstar +fi + if ! has "$EBUILD_PHASE" clean cleanrm ; then if [[ $EBUILD_PHASE = depend || ! -f $T/environment || \ -f $PORTAGE_BUILDDIR/.ebuild_changed ]] || \ @@ -2044,7 +534,7 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then # In order to ensure correct interaction between ebuilds and # eclasses, they need to be unset before this process of # interaction begins. - unset DEPEND RDEPEND PDEPEND INHERITED IUSE REQUIRED_USE \ + unset EAPI DEPEND RDEPEND PDEPEND INHERITED IUSE REQUIRED_USE \ ECLASS E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND if [[ $PORTAGE_DEBUG != 1 || ${-/x/} != $- ]] ; then @@ -2061,7 +551,7 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then rm "$PORTAGE_BUILDDIR/.ebuild_changed" fi - [[ -n $EAPI ]] || EAPI=0 + [ "${EAPI+set}" = set ] || EAPI=0 if has "$EAPI" 0 1 2 3 3_pre2 ; then export RDEPEND=${RDEPEND-${DEPEND}} @@ -2069,11 +559,11 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then fi # add in dependency info from eclasses - IUSE="${IUSE} ${E_IUSE}" - DEPEND="${DEPEND} ${E_DEPEND}" - RDEPEND="${RDEPEND} ${E_RDEPEND}" - PDEPEND="${PDEPEND} ${E_PDEPEND}" - REQUIRED_USE="${REQUIRED_USE} ${E_REQUIRED_USE}" + IUSE+="${IUSE:+ }${E_IUSE}" + DEPEND+="${DEPEND:+ }${E_DEPEND}" + RDEPEND+="${RDEPEND:+ }${E_RDEPEND}" + PDEPEND+="${PDEPEND:+ }${E_PDEPEND}" + REQUIRED_USE+="${REQUIRED_USE:+ }${E_REQUIRED_USE}" unset ECLASS E_IUSE E_REQUIRED_USE E_DEPEND E_RDEPEND E_PDEPEND \ __INHERITED_QA_CACHE @@ -2110,29 +600,11 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then if [[ $EBUILD_PHASE != depend ]] ; then - case "$EAPI" in - 0|1|2|3) - _ebuild_helpers_path="$PORTAGE_BIN_PATH/ebuild-helpers" - ;; - *) - _ebuild_helpers_path="$PORTAGE_BIN_PATH/ebuild-helpers/4:$PORTAGE_BIN_PATH/ebuild-helpers" - ;; - esac - - PATH=$_ebuild_helpers_path:$PREROOTPATH${PREROOTPATH:+:}/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin${ROOTPATH:+:}$ROOTPATH - unset _ebuild_helpers_path - - # Use default ABI libdir in accordance with bug #355283. - x=LIBDIR_${DEFAULT_ABI} - [[ -n $DEFAULT_ABI && -n ${!x} ]] && x=${!x} || x=lib - if has distcc $FEATURES ; then - PATH="/usr/$x/distcc/bin:$PATH" [[ -n $DISTCC_LOG ]] && addwrite "${DISTCC_LOG%/*}" fi if has ccache $FEATURES ; then - PATH="/usr/$x/ccache/bin:$PATH" if [[ -n $CCACHE_DIR ]] ; then addread "$CCACHE_DIR" @@ -2142,8 +614,6 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then [[ -n $CCACHE_SIZE ]] && ccache -M $CCACHE_SIZE &> /dev/null fi - unset x - if [[ -n $QA_PREBUILT ]] ; then # these ones support fnmatch patterns @@ -2153,7 +623,7 @@ if ! has "$EBUILD_PHASE" clean cleanrm ; then # these ones support regular expressions, so translate # fnmatch patterns to regular expressions - for x in QA_DT_HASH QA_DT_NEEDED QA_PRESTRIPPED QA_SONAME ; do + for x in QA_DT_NEEDED QA_FLAGS_IGNORED QA_PRESTRIPPED QA_SONAME ; do if [[ $(declare -p $x 2>/dev/null) = declare\ -a* ]] ; then eval "$x=(\"\${$x[@]}\" ${QA_PREBUILT//\*/.*})" else @@ -2183,241 +653,60 @@ then export DEBUGBUILD=1 fi -#a reasonable default for $S -[[ -z ${S} ]] && export S=${WORKDIR}/${P} +if [[ $EBUILD_PHASE = depend ]] ; then + export SANDBOX_ON="0" + set -f + + if [ -n "${dbkey}" ] ; then + if [ ! -d "${dbkey%/*}" ]; then + install -d -g ${PORTAGE_GID} -m2775 "${dbkey%/*}" + fi + # Make it group writable. 666&~002==664 + umask 002 + fi + + auxdbkeys="DEPEND RDEPEND SLOT SRC_URI RESTRICT HOMEPAGE LICENSE + DESCRIPTION KEYWORDS INHERITED IUSE REQUIRED_USE PDEPEND PROVIDE EAPI + PROPERTIES DEFINED_PHASES UNUSED_05 UNUSED_04 + UNUSED_03 UNUSED_02 UNUSED_01" -# Note: readonly variables interfere with preprocess_ebuild_env(), so -# declare them only after it has already run. -if [ "${EBUILD_PHASE}" != "depend" ] ; then + # The extra $(echo) commands remove newlines. + if [ -n "${dbkey}" ] ; then + > "${dbkey}" + for f in ${auxdbkeys} ; do + echo $(echo ${!f}) >> "${dbkey}" || exit $? + done + else + for f in ${auxdbkeys} ; do + echo $(echo ${!f}) 1>&9 || exit $? + done + exec 9>&- + fi + set +f +else + # Note: readonly variables interfere with preprocess_ebuild_env(), so + # declare them only after it has already run. declare -r $PORTAGE_READONLY_METADATA $PORTAGE_READONLY_VARS case "$EAPI" in 0|1|2) + [[ " ${FEATURES} " == *" force-prefix "* ]] && \ + declare -r ED EPREFIX EROOT ;; *) declare -r ED EPREFIX EROOT ;; esac -fi - -ebuild_main() { - - # Subshell/helper die support (must export for the die helper). - # Since this function is typically executed in a subshell, - # setup EBUILD_MASTER_PID to refer to the current $BASHPID, - # which seems to give the best results when further - # nested subshells call die. - export EBUILD_MASTER_PID=$BASHPID - trap 'exit 1' SIGTERM - - if [[ $EBUILD_PHASE != depend ]] ; then - # Force configure scripts that automatically detect ccache to - # respect FEATURES="-ccache". - has ccache $FEATURES || export CCACHE_DISABLE=1 - - local phase_func=$(_ebuild_arg_to_phase "$EAPI" "$EBUILD_PHASE") - [[ -n $phase_func ]] && _ebuild_phase_funcs "$EAPI" "$phase_func" - unset phase_func - fi - - source_all_bashrcs - - case ${EBUILD_SH_ARGS} in - nofetch) - ebuild_phase_with_hooks pkg_nofetch - ;; - prerm|postrm|postinst|config|info) - if has "$EBUILD_SH_ARGS" config info && \ - ! declare -F "pkg_$EBUILD_SH_ARGS" >/dev/null ; then - ewarn "pkg_${EBUILD_SH_ARGS}() is not defined: '${EBUILD##*/}'" - fi - export SANDBOX_ON="0" - if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then - ebuild_phase_with_hooks pkg_${EBUILD_SH_ARGS} - else - set -x - ebuild_phase_with_hooks pkg_${EBUILD_SH_ARGS} - set +x - fi - if [[ $EBUILD_PHASE == postinst ]] && [[ -n $PORTAGE_UPDATE_ENV ]]; then - # Update environment.bz2 in case installation phases - # need to pass some variables to uninstallation phases. - save_ebuild_env --exclude-init-phases | \ - filter_readonly_variables --filter-path \ - --filter-sandbox --allow-extra-vars \ - | ${PORTAGE_BZIP2_COMMAND} -c -f9 > "$PORTAGE_UPDATE_ENV" - assert "save_ebuild_env failed" - fi - ;; - unpack|prepare|configure|compile|test|clean|install) - if [[ ${SANDBOX_DISABLED:-0} = 0 ]] ; then - export SANDBOX_ON="1" - else - export SANDBOX_ON="0" - fi - - case "$EBUILD_SH_ARGS" in - configure|compile) - - local x - for x in ASFLAGS CCACHE_DIR CCACHE_SIZE \ - CFLAGS CXXFLAGS LDFLAGS LIBCFLAGS LIBCXXFLAGS ; do - [[ ${!x+set} = set ]] && export $x - done - unset x - - has distcc $FEATURES && [[ -n $DISTCC_DIR ]] && \ - [[ ${SANDBOX_WRITE/$DISTCC_DIR} = $SANDBOX_WRITE ]] && \ - addwrite "$DISTCC_DIR" - - x=LIBDIR_$ABI - [ -z "$PKG_CONFIG_PATH" -a -n "$ABI" -a -n "${!x}" ] && \ - export PKG_CONFIG_PATH=/usr/${!x}/pkgconfig - - if has noauto $FEATURES && \ - [[ ! -f $PORTAGE_BUILDDIR/.unpacked ]] ; then - echo - echo "!!! We apparently haven't unpacked..." \ - "This is probably not what you" - echo "!!! want to be doing... You are using" \ - "FEATURES=noauto so I'll assume" - echo "!!! that you know what you are doing..." \ - "You have 5 seconds to abort..." - echo - - local x - for x in 1 2 3 4 5 6 7 8; do - LC_ALL=C sleep 0.25 - done - - sleep 3 - fi - - cd "$PORTAGE_BUILDDIR" - if [ ! -d build-info ] ; then - mkdir build-info - cp "$EBUILD" "build-info/$PF.ebuild" - fi - - #our custom version of libtool uses $S and $D to fix - #invalid paths in .la files - export S D - - ;; - esac - if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then - dyn_${EBUILD_SH_ARGS} - else - set -x - dyn_${EBUILD_SH_ARGS} - set +x - fi - export SANDBOX_ON="0" - ;; - help|pretend|setup|preinst) - #pkg_setup needs to be out of the sandbox for tmp file creation; - #for example, awking and piping a file in /tmp requires a temp file to be created - #in /etc. If pkg_setup is in the sandbox, both our lilo and apache ebuilds break. - export SANDBOX_ON="0" - if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then - dyn_${EBUILD_SH_ARGS} - else - set -x - dyn_${EBUILD_SH_ARGS} - set +x - fi - ;; - depend) - export SANDBOX_ON="0" - set -f - - if [ -n "${dbkey}" ] ; then - if [ ! -d "${dbkey%/*}" ]; then - install -d -g ${PORTAGE_GID} -m2775 "${dbkey%/*}" - fi - # Make it group writable. 666&~002==664 - umask 002 - fi - - auxdbkeys="DEPEND RDEPEND SLOT SRC_URI RESTRICT HOMEPAGE LICENSE - DESCRIPTION KEYWORDS INHERITED IUSE REQUIRED_USE PDEPEND PROVIDE EAPI - PROPERTIES DEFINED_PHASES UNUSED_05 UNUSED_04 - UNUSED_03 UNUSED_02 UNUSED_01" - - #the extra $(echo) commands remove newlines - [ -n "${EAPI}" ] || EAPI=0 - - if [ -n "${dbkey}" ] ; then - > "${dbkey}" - for f in ${auxdbkeys} ; do - echo $(echo ${!f}) >> "${dbkey}" || exit $? - done - else - for f in ${auxdbkeys} ; do - echo $(echo ${!f}) 1>&9 || exit $? - done + if [[ -n $EBUILD_SH_ARGS ]] ; then + ( + # Don't allow subprocesses to inherit the pipe which + # emerge uses to monitor ebuild.sh. exec 9>&- - fi - set +f - ;; - _internal_test) - ;; - *) - export SANDBOX_ON="1" - echo "Unrecognized EBUILD_SH_ARGS: '${EBUILD_SH_ARGS}'" - echo - dyn_help - exit 1 - ;; - esac -} - -if [[ -s $SANDBOX_LOG ]] ; then - # We use SANDBOX_LOG to check for sandbox violations, - # so we ensure that there can't be a stale log to - # interfere with our logic. - x= - if [[ -n SANDBOX_ON ]] ; then - x=$SANDBOX_ON - export SANDBOX_ON=0 + ebuild_main ${EBUILD_SH_ARGS} + exit 0 + ) + exit $? fi - - rm -f "$SANDBOX_LOG" || \ - die "failed to remove stale sandbox log: '$SANDBOX_LOG'" - - if [[ -n $x ]] ; then - export SANDBOX_ON=$x - fi - unset x -fi - -if [[ $EBUILD_PHASE = depend ]] ; then - ebuild_main -elif [[ -n $EBUILD_SH_ARGS ]] ; then - ( - # Don't allow subprocesses to inherit the pipe which - # emerge uses to monitor ebuild.sh. - exec 9>&- - - ebuild_main - - # Save the env only for relevant phases. - if ! has "$EBUILD_SH_ARGS" clean help info nofetch ; then - umask 002 - save_ebuild_env | filter_readonly_variables \ - --filter-features > "$T/environment" - assert "save_ebuild_env failed" - chown portage:portage "$T/environment" &>/dev/null - chmod g+w "$T/environment" &>/dev/null - fi - [[ -n $PORTAGE_EBUILD_EXIT_FILE ]] && > "$PORTAGE_EBUILD_EXIT_FILE" - if [[ -n $PORTAGE_IPC_DAEMON ]] ; then - [[ ! -s $SANDBOX_LOG ]] - "$PORTAGE_BIN_PATH"/ebuild-ipc exit $? - fi - exit 0 - ) - exit $? fi # Do not exit when ebuild.sh is sourced by other scripts. diff --git a/portage_with_autodep/bin/egencache b/portage_with_autodep/bin/egencache index 1b4265d..2f53b40 100755 --- a/portage_with_autodep/bin/egencache +++ b/portage_with_autodep/bin/egencache @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2009-2011 Gentoo Foundation +# Copyright 2009-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -42,7 +42,7 @@ from portage.manifest import guessManifestFileType from portage.util import cmp_sort_key, writemsg_level from portage import cpv_getkey from portage.dep import Atom, isjustname -from portage.versions import pkgcmp, pkgsplit, vercmp +from portage.versions import pkgsplit, vercmp try: from xml.etree import ElementTree @@ -87,6 +87,9 @@ def parse_args(args): common.add_option("--portdir", help="override the portage tree location", dest="portdir") + common.add_option("--portdir-overlay", + help="override the PORTDIR_OVERLAY variable (requires that --repo is also specified)", + dest="portdir_overlay") common.add_option("--tolerant", action="store_true", help="exit successfully if only minor errors occurred") @@ -160,9 +163,17 @@ def parse_args(args): parser.error("Not a directory: --config-root='%s'" % \ (options.config_root,)) - if options.cache_dir is not None and not os.path.isdir(options.cache_dir): - parser.error("Not a directory: --cache-dir='%s'" % \ - (options.cache_dir,)) + if options.cache_dir is not None: + if not os.path.isdir(options.cache_dir): + parser.error("Not a directory: --cache-dir='%s'" % \ + (options.cache_dir,)) + if not os.access(options.cache_dir, os.W_OK): + parser.error("Write access denied: --cache-dir='%s'" % \ + (options.cache_dir,)) + + if options.portdir_overlay is not None and \ + options.repo is None: + parser.error("--portdir-overlay option requires --repo option") for atom in args: try: @@ -188,7 +199,12 @@ def parse_args(args): class GenCache(object): def __init__(self, portdb, cp_iter=None, max_jobs=None, max_load=None, rsync=False): + # The caller must set portdb.porttrees in order to constrain + # findname, cp_list, and cpv_list to the desired tree. + tree = portdb.porttrees[0] self._portdb = portdb + self._eclass_db = portdb.repositories.get_repo_for_location(tree).eclass_db + self._auxdbkeys = portdb._known_keys # We can globally cleanse stale cache only if we # iterate over every single cp. self._global_cleanse = cp_iter is None @@ -203,27 +219,67 @@ class GenCache(object): consumer=self._metadata_callback, max_jobs=max_jobs, max_load=max_load) self.returncode = os.EX_OK - metadbmodule = portdb.settings.load_best_module("portdbapi.metadbmodule") - self._trg_cache = metadbmodule(portdb.porttrees[0], - "metadata/cache", portage.auxdbkeys[:]) + conf = portdb.repositories.get_repo_for_location(tree) + self._trg_caches = tuple(conf.iter_pregenerated_caches( + self._auxdbkeys, force=True, readonly=False)) + if not self._trg_caches: + raise Exception("cache formats '%s' aren't supported" % + (" ".join(conf.cache_formats),)) + if rsync: - self._trg_cache.raise_stat_collision = True - try: - self._trg_cache.ec = \ - portdb._repo_info[portdb.porttrees[0]].eclass_db - except AttributeError: - pass + for trg_cache in self._trg_caches: + if hasattr(trg_cache, 'raise_stat_collision'): + trg_cache.raise_stat_collision = True + # Make _metadata_callback write this cache first, in case + # it raises a StatCollision and triggers mtime + # modification. + self._trg_caches = tuple([trg_cache] + + [x for x in self._trg_caches if x is not trg_cache]) + self._existing_nodes = set() - def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata): + def _metadata_callback(self, cpv, repo_path, metadata, + ebuild_hash, eapi_supported): self._existing_nodes.add(cpv) self._cp_missing.discard(cpv_getkey(cpv)) - if metadata is not None: + + # Since we're supposed to be able to efficiently obtain the + # EAPI from _parse_eapi_ebuild_head, we don't write cache + # entries for unsupported EAPIs. + if metadata is not None and eapi_supported: if metadata.get('EAPI') == '0': del metadata['EAPI'] + for trg_cache in self._trg_caches: + self._write_cache(trg_cache, + cpv, repo_path, metadata, ebuild_hash) + + def _write_cache(self, trg_cache, cpv, repo_path, metadata, ebuild_hash): + + if not hasattr(trg_cache, 'raise_stat_collision'): + # This cache does not avoid redundant writes automatically, + # so check for an identical existing entry before writing. + # This prevents unnecessary disk writes and can also prevent + # unnecessary rsync transfers. + try: + dest = trg_cache[cpv] + except (KeyError, CacheError): + pass + else: + if trg_cache.validate_entry(dest, + ebuild_hash, self._eclass_db): + identical = True + for k in self._auxdbkeys: + if dest.get(k, '') != metadata.get(k, ''): + identical = False + break + if identical: + return + try: + chf = trg_cache.validation_chf + metadata['_%s_' % chf] = getattr(ebuild_hash, chf) try: - self._trg_cache[cpv] = metadata + trg_cache[cpv] = metadata except StatCollision as sc: # If the content of a cache entry changes and neither the # file mtime nor size changes, it will prevent rsync from @@ -231,24 +287,30 @@ class GenCache(object): # exception from _setitem() if they detect this type of stat # collision. These exceptions are handled by bumping the # mtime on the ebuild (and the corresponding cache entry). - # See bug #139134. + # See bug #139134. It is convenient to include checks for + # redundant writes along with the internal StatCollision + # detection code, so for caches with the + # raise_stat_collision attribute, we do not need to + # explicitly check for redundant writes like we do for the + # other cache types above. max_mtime = sc.mtime - for ec, (loc, ec_mtime) in metadata['_eclasses_'].items(): - if max_mtime < ec_mtime: - max_mtime = ec_mtime + for ec, ec_hash in metadata['_eclasses_'].items(): + if max_mtime < ec_hash.mtime: + max_mtime = ec_hash.mtime if max_mtime == sc.mtime: max_mtime += 1 max_mtime = long(max_mtime) try: - os.utime(ebuild_path, (max_mtime, max_mtime)) + os.utime(ebuild_hash.location, (max_mtime, max_mtime)) except OSError as e: self.returncode |= 1 writemsg_level( "%s writing target: %s\n" % (cpv, e), level=logging.ERROR, noiselevel=-1) else: + ebuild_hash.mtime = max_mtime metadata['_mtime_'] = max_mtime - self._trg_cache[cpv] = metadata + trg_cache[cpv] = metadata self._portdb.auxdb[repo_path][cpv] = metadata except CacheError as ce: @@ -287,9 +349,12 @@ class GenCache(object): sys.exit(received_signal[0]) self.returncode |= self._regen.returncode - cp_missing = self._cp_missing - trg_cache = self._trg_cache + for trg_cache in self._trg_caches: + self._cleanse_cache(trg_cache) + + def _cleanse_cache(self, trg_cache): + cp_missing = self._cp_missing dead_nodes = set() if self._global_cleanse: try: @@ -443,12 +508,12 @@ class GenUseLocalDesc(object): errors='backslashreplace') output.write(_unicode_decode('\n')) else: - output.write(_unicode_decode(''' -# This file is deprecated as per GLEP 56 in favor of metadata.xml. Please add -# your descriptions to your package's metadata.xml ONLY. -# * generated automatically using egencache * + output.write(textwrap.dedent(_unicode_decode('''\ + # This file is deprecated as per GLEP 56 in favor of metadata.xml. Please add + # your descriptions to your package's metadata.xml ONLY. + # * generated automatically using egencache * -'''.lstrip())) + '''))) # The cmp function no longer exists in python3, so we'll # implement our own here under a slightly different name @@ -465,10 +530,20 @@ class GenUseLocalDesc(object): return 1 return (a > b) - (a < b) + class _MetadataTreeBuilder(ElementTree.TreeBuilder): + """ + Implements doctype() as required to avoid deprecation warnings + since Python >=2.7 + """ + def doctype(self, name, pubid, system): + pass + for cp in self._portdb.cp_all(): metadata_path = os.path.join(repo_path, cp, 'metadata.xml') try: - metadata = ElementTree.parse(metadata_path) + metadata = ElementTree.parse(metadata_path, + parser=ElementTree.XMLParser( + target=_MetadataTreeBuilder())) except IOError: pass except (ExpatError, EnvironmentError) as e: @@ -495,7 +570,7 @@ class GenUseLocalDesc(object): return cmp_func(atomb.operator, atoma.operator) # Version matching elif atoma.cpv != atomb.cpv: - return pkgcmp(pkgsplit(atoma.cpv), pkgsplit(atomb.cpv)) + return vercmp(atoma.version, atomb.version) # Versions match, let's fallback to operator matching else: return cmp_func(ops.get(atoma.operator, -1), @@ -620,12 +695,12 @@ class GenChangeLogs(object): self.returncode |= 2 return - output.write(_unicode_decode(''' -# ChangeLog for %s -# Copyright 1999-%s Gentoo Foundation; Distributed under the GPL v2 -# $Header: $ + output.write(textwrap.dedent(_unicode_decode('''\ + # ChangeLog for %s + # Copyright 1999-%s Gentoo Foundation; Distributed under the GPL v2 + # $Header: $ -''' % (cp, time.strftime('%Y'))).lstrip()) + ''' % (cp, time.strftime('%Y'))))) # now grab all the commits commits = self.grab(['git', 'rev-list', 'HEAD', '--', '.']).split() @@ -755,8 +830,6 @@ def egencache_main(args): parser, options, atoms = parse_args(args) config_root = options.config_root - if config_root is None: - config_root = '/' # The calling environment is ignored, so the program is # completely controlled by commandline arguments. @@ -764,6 +837,8 @@ def egencache_main(args): if options.repo is None: env['PORTDIR_OVERLAY'] = '' + elif options.portdir_overlay: + env['PORTDIR_OVERLAY'] = options.portdir_overlay if options.cache_dir is not None: env['PORTAGE_DEPCACHEDIR'] = options.cache_dir @@ -772,7 +847,7 @@ def egencache_main(args): env['PORTDIR'] = options.portdir settings = portage.config(config_root=config_root, - target_root='/', local_config=False, env=env) + local_config=False, env=env) default_opts = None if not options.ignore_default_opts: @@ -781,14 +856,11 @@ def egencache_main(args): if default_opts: parser, options, args = parse_args(default_opts + args) - if options.config_root is not None: - config_root = options.config_root - if options.cache_dir is not None: env['PORTAGE_DEPCACHEDIR'] = options.cache_dir settings = portage.config(config_root=config_root, - target_root='/', local_config=False, env=env) + local_config=False, env=env) if not options.update and not options.update_use_local_desc \ and not options.update_changelogs: @@ -796,14 +868,27 @@ def egencache_main(args): return 1 if options.update and 'metadata-transfer' not in settings.features: - writemsg_level("ecachegen: warning: " + \ - "automatically enabling FEATURES=metadata-transfer\n", - level=logging.WARNING, noiselevel=-1) settings.features.add('metadata-transfer') settings.lock() portdb = portage.portdbapi(mysettings=settings) + + if options.update: + if options.cache_dir is not None: + # already validated earlier + pass + else: + # We check write access after the portdbapi constructor + # has had an opportunity to create it. This ensures that + # we don't use the cache in the "volatile" mode which is + # undesirable for egencache. + if not os.access(settings["PORTAGE_DEPCACHEDIR"], os.W_OK): + writemsg_level("ecachegen: error: " + \ + "write access denied: %s\n" % (settings["PORTAGE_DEPCACHEDIR"],), + level=logging.ERROR, noiselevel=-1) + return 1 + if options.repo is not None: repo_path = portdb.getRepositoryPath(options.repo) if repo_path is None: @@ -813,6 +898,8 @@ def egencache_main(args): # Limit ebuilds to the specified repo. portdb.porttrees = [repo_path] + else: + portdb.porttrees = [portdb.porttree_root] ret = [os.EX_OK] diff --git a/portage_with_autodep/bin/emaint b/portage_with_autodep/bin/emaint index fdd01ed..1bee0fe 100755 --- a/portage_with_autodep/bin/emaint +++ b/portage_with_autodep/bin/emaint @@ -40,15 +40,15 @@ class WorldHandler(object): self.okay = [] from portage._sets import load_default_config setconfig = load_default_config(portage.settings, - portage.db[portage.settings["ROOT"]]) + portage.db[portage.settings['EROOT']]) self._sets = setconfig.getSets() def _check_world(self, onProgress): categories = set(portage.settings.categories) - myroot = portage.settings["ROOT"] - self.world_file = os.path.join(portage.settings["EROOT"], portage.const.WORLD_FILE) + eroot = portage.settings['EROOT'] + self.world_file = os.path.join(eroot, portage.const.WORLD_FILE) self.found = os.access(self.world_file, os.R_OK) - vardb = portage.db[myroot]["vartree"].dbapi + vardb = portage.db[eroot]["vartree"].dbapi from portage._sets import SETPREFIX sets = self._sets @@ -120,8 +120,8 @@ class BinhostHandler(object): name = staticmethod(name) def __init__(self): - myroot = portage.settings["ROOT"] - self._bintree = portage.db[myroot]["bintree"] + eroot = portage.settings['EROOT'] + self._bintree = portage.db[eroot]["bintree"] self._bintree.populate() self._pkgindex_file = self._bintree._pkgindex_file self._pkgindex = self._bintree._load_pkgindex() @@ -403,8 +403,8 @@ class MoveInstalled(MoveHandler): return "moveinst" name = staticmethod(name) def __init__(self): - myroot = portage.settings["ROOT"] - MoveHandler.__init__(self, portage.db[myroot]["vartree"], portage.db[myroot]["porttree"]) + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"]) class MoveBinary(MoveHandler): @@ -414,8 +414,8 @@ class MoveBinary(MoveHandler): return "movebin" name = staticmethod(name) def __init__(self): - myroot = portage.settings["ROOT"] - MoveHandler.__init__(self, portage.db[myroot]["bintree"], portage.db[myroot]["porttree"]) + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree']) class VdbKeyHandler(object): def name(): @@ -423,7 +423,7 @@ class VdbKeyHandler(object): name = staticmethod(name) def __init__(self): - self.list = portage.db["/"]["vartree"].dbapi.cpv_all() + self.list = portage.db[portage.settings["EROOT"]]["vartree"].dbapi.cpv_all() self.missing = [] self.keys = ["HOMEPAGE", "SRC_URI", "KEYWORDS", "DESCRIPTION"] diff --git a/portage_with_autodep/bin/emerge-webrsync b/portage_with_autodep/bin/emerge-webrsync index d933871..bfd9aa2 100755 --- a/portage_with_autodep/bin/emerge-webrsync +++ b/portage_with_autodep/bin/emerge-webrsync @@ -12,7 +12,7 @@ # # gpg key import -# KEY_ID=0x239C75C4 +# KEY_ID=0x96D8BF6D # gpg --homedir /etc/portage/gnupg --keyserver subkeys.pgp.net --recv-keys $KEY_ID # gpg --homedir /etc/portage/gnupg --edit-key $KEY_ID trust # @@ -27,11 +27,19 @@ wecho() { echo "${argv0}: warning: $*" 1>&2 ; } eecho() { echo "${argv0}: error: $*" 1>&2 ; } argv0=$0 -if ! type -P portageq > /dev/null ; then + +# Use portageq from the same directory/prefix as the current script, so +# that we don't have to rely on PATH including the current EPREFIX. +scriptpath=${BASH_SOURCE[0]} +if [ -x "${scriptpath%/*}/portageq" ]; then + portageq=${scriptpath%/*}/portageq +elif type -P portageq > /dev/null ; then + portageq=portageq +else eecho "could not find 'portageq'; aborting" exit 1 fi -eval $(portageq envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \ +eval $("${portageq}" envvar -v FEATURES FETCHCOMMAND GENTOO_MIRRORS \ PORTAGE_BIN_PATH PORTAGE_GPG_DIR \ PORTAGE_NICENESS PORTAGE_RSYNC_EXTRA_OPTS PORTAGE_TMPDIR PORTDIR \ SYNC http_proxy ftp_proxy) @@ -157,6 +165,7 @@ check_file_signature() { gpg --homedir "${PORTAGE_GPG_DIR}" --verify "$signature" "$file" && r=0 else eecho "cannot check signature: gpg binary not found" + exit 1 fi else r=0 @@ -211,6 +220,8 @@ sync_local() { emerge --metadata fi [ -x /etc/portage/bin/post_sync ] && /etc/portage/bin/post_sync + # --quiet suppresses output if there are no relevant news items + has news ${FEATURES} && emerge --check-news --quiet return 0 } diff --git a/portage_with_autodep/bin/emirrordist b/portage_with_autodep/bin/emirrordist new file mode 100755 index 0000000..8d93de9 --- /dev/null +++ b/portage_with_autodep/bin/emirrordist @@ -0,0 +1,13 @@ +#!/usr/bin/python +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import sys + +import portage +portage._internal_caller = True +portage._disable_legacy_globals() +from portage._emirrordist.main import emirrordist_main + +if __name__ == "__main__": + sys.exit(emirrordist_main(sys.argv[1:])) diff --git a/portage_with_autodep/bin/etc-update b/portage_with_autodep/bin/etc-update index 42518ad..1edc91f 100755 --- a/portage_with_autodep/bin/etc-update +++ b/portage_with_autodep/bin/etc-update @@ -1,8 +1,9 @@ #!/bin/bash -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Author Brandon Low <lostlogic@gentoo.org> +# Mike Frysinger <vapier@gentoo.org> # # Previous version (from which I've borrowed a few bits) by: # Jochem Kossen <j.kossen@home.nl> @@ -11,9 +12,7 @@ cd / -if type -P gsed >/dev/null ; then - sed() { gsed "$@"; } -fi +type -P gsed >/dev/null && sed() { gsed "$@"; } get_config() { # the sed here does: @@ -25,54 +24,92 @@ get_config() { # If there's more than one of the same configuration item, then # the store to the hold space clobbers previous value so the last # setting takes precedence. - local item=$1 - eval echo $(sed -n \ + local match=$1 + eval $(sed -n -r \ -e 's:[[:space:]]*#.*$::' \ - -e "/^[[:space:]]*$item[[:space:]]*=/{s:[^=]*=[[:space:]]*\([\"']\{0,1\}\)\(.*\)\1:\2:;h}" \ + -e "/^[[:space:]]*${match}[[:space:]]*=/{s:^([^=]*)=[[:space:]]*([\"']{0,1})(.*)\2:\1=\2\3\2:;H}" \ -e '${g;p}' \ "${PORTAGE_CONFIGROOT}"etc/etc-update.conf) } +cmd_var_is_valid() { + # return true if the first whitespace-separated token contained + # in "${1}" is an executable file, false otherwise + [[ -x $(type -P ${1%%[[:space:]]*}) ]] +} + diff_command() { local cmd=${diff_command//%file1/$1} ${cmd//%file2/$2} } +# Usage: do_mv_ln [options] <src> <dst> +# Files have to be the last two args, and has to be +# files so we can handle symlinked target sanely. +do_mv_ln() { + local opts=( ${@:1:$(( $# - 2 ))} ) + local src=${@:$(( $# - 1 )):1} + local dst=${@:$(( $# - 0 )):1} + + if [[ -L ${dst} ]] ; then #330221 + local lfile=$(readlink "${dst}") + [[ ${lfile} == /* ]] || lfile="${dst%/*}/${lfile}" + echo " Target is a symlink; replacing ${lfile}" + dst=${lfile} + fi + + mv "${opts[@]}" "${src}" "${dst}" +} + scan() { echo "Scanning Configuration files..." - rm -rf ${TMP}/files > /dev/null 2>&1 - mkdir ${TMP}/files || die "Failed mkdir command!" 1 + rm -rf "${TMP}"/files > /dev/null 2>&1 + mkdir "${TMP}"/files || die "Failed mkdir command!" count=0 input=0 local find_opts - local my_basename - - for path in ${CONFIG_PROTECT} ; do - path="${ROOT}${path}" - # Do not traverse hidden directories such as .svn or .git. - find_opts="-name .* -type d -prune -o -name ._cfg????_*" - if [ ! -d "${path}" ]; then - [ ! -f "${path}" ] && continue - my_basename="${path##*/}" + local path + + for path in ${SCAN_PATHS} ; do + path="${EROOT%/}${path}" + + if [[ ! -d ${path} ]] ; then + [[ ! -f ${path} ]] && continue + local my_basename="${path##*/}" path="${path%/*}" - find_opts="-maxdepth 1 -name ._cfg????_${my_basename}" + find_opts=( -maxdepth 1 -name "._cfg????_${my_basename}" ) + else + # Do not traverse hidden directories such as .svn or .git. + find_opts=( -name '.*' -type d -prune -o -name '._cfg????_*' ) fi + find_opts+=( ! -name '.*~' ! -iname '.*.bak' -print ) - ofile="" - # The below set -f turns off file name globbing in the ${find_opts} expansion. - for file in $(set -f ; find ${path}/ ${find_opts} \ - ! -name '.*~' ! -iname '.*.bak' -print | - sed -e "s:\(^.*/\)\(\._cfg[0-9]*_\)\(.*$\):\1\2\3\%\1%\2\%\3:" | - sort -t'%' -k2,2 -k4,4 -k3,3 | LANG=POSIX LC_ALL=POSIX cut -f1 -d'%'); do + if [ ! -w "${path}" ] ; then + [ -e "${path}" ] || continue + die "Need write access to ${path}" + fi - rpath=$(echo "${file/\/\///}" | sed -e "s:/[^/]*$::") - rfile=$(echo "${file/\/\///}" | sed -e "s:^.*/::") + local file ofile b=$'\001' + for file in $(find "${path}"/ "${find_opts[@]}" | + sed \ + -e 's://*:/:g' \ + -e "s:\(^.*/\)\(\._cfg[0-9]*_\)\(.*$\):\1\2\3$b\1$b\2$b\3:" | + sort -t"$b" -k2,2 -k4,4 -k3,3 | + LC_ALL=C cut -f1 -d"$b") + do + local rpath rfile cfg_file live_file + rpath=${file%/*} + rfile=${file##*/} + cfg_file="${rpath}/${rfile}" + live_file="${rpath}/${rfile:10}" + + local mpath for mpath in ${CONFIG_PROTECT_MASK}; do - mpath="${ROOT}${mpath}" - mpath=$(echo "${mpath/\/\///}") - if [[ "${rpath}" == "${mpath}"* ]]; then - mv ${rpath}/${rfile} ${rpath}/${rfile:10} - break + mpath="${EROOT%/}${mpath}" + if [[ "${rpath}" == "${mpath}"* ]] ; then + echo "Updating masked file: ${live_file}" + mv "${cfg_file}" "${live_file}" + continue 2 fi done if [[ ! -f ${file} ]] ; then @@ -81,45 +118,88 @@ scan() { fi if [[ "${ofile:10}" != "${rfile:10}" ]] || - [[ ${opath} != ${rpath} ]]; then + [[ ${opath} != ${rpath} ]] + then MATCHES=0 - if [[ "${EU_AUTOMERGE}" == "yes" ]]; then - if [ ! -e "${rpath}/${rfile}" ] || [ ! -e "${rpath}/${rfile:10}" ]; then + if [[ ${eu_automerge} == "yes" ]] ; then + if [[ ! -e ${cfg_file} || ! -e ${live_file} ]] ; then MATCHES=0 else - diff -Bbua ${rpath}/${rfile} ${rpath}/${rfile:10} | egrep '^[+-]' | egrep -v '^[+-][\t ]*#|^--- |^\+\+\+ ' | egrep -qv '^[-+][\t ]*$' - MATCHES=$? + diff -Bbua "${cfg_file}" "${live_file}" | \ + sed -n -r \ + -e '/^[+-]/{/^([+-][\t ]*(#|$)|-{3} |\+{3} )/d;q1}' + : $(( MATCHES = ($? == 0) )) fi - elif [[ -z $(diff -Nua ${rpath}/${rfile} ${rpath}/${rfile:10}| - grep "^[+-][^+-]"|grep -v '# .Header:.*') ]]; then - MATCHES=1 + + else + diff -Nbua "${cfg_file}" "${live_file}" | + sed -n \ + -e '/# .Header:/d' \ + -e '/^[+-][^+-]/q1' + : $(( MATCHES = ($? == 0) )) fi - if [[ "${MATCHES}" == "1" ]]; then - echo "Automerging trivial changes in: ${rpath}/${rfile:10}" - mv ${rpath}/${rfile} ${rpath}/${rfile:10} + + if [[ ${MATCHES} == 1 ]] ; then + echo "Automerging trivial changes in: ${live_file}" + do_mv_ln "${cfg_file}" "${live_file}" continue else - count=${count}+1 - echo "${rpath}/${rfile:10}" > ${TMP}/files/${count} - echo "${rpath}/${rfile}" >> ${TMP}/files/${count} + : $(( ++count )) + echo "${live_file}" > "${TMP}"/files/${count} + echo "${cfg_file}" >> "${TMP}"/files/${count} ofile="${rfile}" opath="${rpath}" continue fi fi - if [[ -z $(diff -Nua ${rpath}/${rfile} ${rpath}/${ofile}| - grep "^[+-][^+-]"|grep -v '# .Header:.*') ]]; then - mv ${rpath}/${rfile} ${rpath}/${ofile} - continue - else - echo "${rpath}/${rfile}" >> ${TMP}/files/${count} + if ! diff -Nbua "${cfg_file}" "${rpath}/${ofile}" | + sed -n \ + -e '/# .Header:/d' \ + -e '/^[+-][^+-]/q1' + then + echo "${cfg_file}" >> "${TMP}"/files/${count} ofile="${rfile}" opath="${rpath}" + else + mv "${cfg_file}" "${rpath}/${ofile}" + continue fi done done +} +parse_automode_flag() { + case $1 in + -9) + local reply + read -p "Are you sure that you want to delete all updates (type YES): " reply + if [[ ${reply} != "YES" ]] ; then + echo "Did not get a 'YES', so ignoring request" + return 1 + else + parse_automode_flag -7 + export rm_opts="" + fi + ;; + -7) + input=0 + export DELETE_ALL="yes" + ;; + -5) + parse_automode_flag -3 + export mv_opts=" ${mv_opts} " + mv_opts="${mv_opts// -i / }" + ;; + -3) + input=0 + export OVERWRITE_ALL="yes" + ;; + *) + return 1 + ;; + esac + return 0 } sel_file() { @@ -128,88 +208,77 @@ sel_file() { [[ ${input} == -1 ]] || \ [[ ${input} == -3 ]] do - local numfiles=$(ls ${TMP}/files|wc -l) - local numwidth=${#numfiles} - for file in $(ls ${TMP}/files|sort -n); do - if [[ ${isfirst} == 0 ]] ; then - isfirst=${file} - fi - numshow=$(printf "%${numwidth}i${PAR} " ${file}) - numupdates=$(( $(wc -l <${TMP}/files/${file}) - 1 )) - echo -n "${numshow}" - if [[ ${mode} == 0 ]] ; then - echo "$(head -n1 ${TMP}/files/${file}) (${numupdates})" - else - head -n1 ${TMP}/files/${file} - fi - done > ${TMP}/menuitems + local allfiles=( $(cd "${TMP}"/files/ && printf '%s\n' * | sort -n) ) + local isfirst=${allfiles[0]} - if [ "${OVERWRITE_ALL}" == "yes" ]; then - input=0 - elif [ "${DELETE_ALL}" == "yes" ]; then + # Optimize: no point in building the whole file list if + # we're not actually going to talk to the user. + if [[ ${OVERWRITE_ALL} == "yes" || ${DELETE_ALL} == "yes" ]] ; then input=0 else - [[ $CLEAR_TERM == yes ]] && clear - if [[ ${mode} == 0 ]] ; then - echo "The following is the list of files which need updating, each -configuration file is followed by a list of possible replacement files." - else - local my_title="Please select a file to update" - fi + local numfiles=${#allfiles[@]} + local numwidth=${#numfiles} + local file fullfile line + for file in "${allfiles[@]}" ; do + fullfile="${TMP}/files/${file}" + line=$(head -n1 "${fullfile}") + printf '%*i%s %s' ${numwidth} ${file} "${PAR}" "${line}" + if [[ ${mode} == 0 ]] ; then + local numupdates=$(( $(wc -l <"${fullfile}") - 1 )) + echo " (${numupdates})" + else + echo + fi + done > "${TMP}"/menuitems + + clear if [[ ${mode} == 0 ]] ; then - cat ${TMP}/menuitems - echo "Please select a file to edit by entering the corresponding number." - echo " (don't use -3, -5, -7 or -9 if you're unsure what to do)" - echo " (-1 to exit) (-3 to auto merge all remaining files)" - echo " (-5 to auto-merge AND not use 'mv -i')" - echo " (-7 to discard all updates)" - echo -n " (-9 to discard all updates AND not use 'rm -i'): " + cat <<-EOF + The following is the list of files which need updating, each + configuration file is followed by a list of possible replacement files. + $(<"${TMP}"/menuitems) + Please select a file to edit by entering the corresponding number. + (don't use -3, -5, -7 or -9 if you're unsure what to do) + (-1 to exit) (${_3_HELP_TEXT}) + (${_5_HELP_TEXT}) + (${_7_HELP_TEXT}) + EOF + printf " (${_9_HELP_TEXT}): " input=$(read_int) else - dialog --title "${title}" --menu "${my_title}" \ - 0 0 0 $(echo -e "-1 Exit\n$(<${TMP}/menuitems)") \ - 2> ${TMP}/input || die "User termination!" 0 - input=$(<${TMP}/input) + dialog \ + --title "${title}" \ + --menu "Please select a file to update" \ + 0 0 0 $(<"${TMP}"/menuitems) \ + 2> "${TMP}"/input \ + || die "$(<"${TMP}"/input)\n\nUser termination!" 0 + input=$(<"${TMP}"/input) fi - if [[ ${input} == -9 ]]; then - read -p "Are you sure that you want to delete all updates (type YES):" reply - if [[ ${reply} != "YES" ]]; then - continue - else - input=-7 - export rm_opts="" - fi - fi - if [[ ${input} == -7 ]]; then - input=0 - export DELETE_ALL="yes" - fi - if [[ ${input} == -5 ]] ; then - input=-3 - export mv_opts=" ${mv_opts} " - mv_opts="${mv_opts// -i / }" - fi - if [[ ${input} == -3 ]] ; then - input=0 - export OVERWRITE_ALL="yes" + : ${input:=0} + + if [[ ${input} != 0 ]] ; then + parse_automode_flag ${input} || continue fi fi # -3 automerge - if [[ -z ${input} ]] || [[ ${input} == 0 ]] ; then + if [[ ${input} == 0 ]] ; then input=${isfirst} fi done } user_special() { - if [ -r ${PORTAGE_CONFIGROOT}etc/etc-update.special ]; then - if [ -z "$1" ]; then - echo "ERROR: user_special() called without arguments" + local special="${PORTAGE_CONFIGROOT}etc/etc-update.special" + + if [[ -r ${special} ]] ; then + if [[ -z $1 ]] ; then + error "user_special() called without arguments" return 1 fi - while read -r pat; do - echo ${1} | grep "${pat}" > /dev/null && return 0 - done < ${PORTAGE_CONFIGROOT}etc/etc-update.special + local pat + while read -r pat ; do + echo "$1" | grep -q "${pat}" && return 0 + done < "${special}" fi return 1 } @@ -219,12 +288,12 @@ read_int() { # read. This is a workaround for odd behavior of bash when an attempt is # made to store a value such as "1y" into an integer-only variable. local my_input - while true; do + while : ; do read my_input # failed integer conversions will break a loop unless they're enclosed # in a subshell. - echo "${my_input}" | ( declare -i x; read x) 2>/dev/null && break - echo -n "Value '$my_input' is not valid. Please enter an integer value:" >&2 + echo "${my_input}" | (declare -i x; read x) 2>/dev/null && break + printf 'Value "%s" is not valid. Please enter an integer value: ' "${my_input}" >&2 done echo ${my_input} } @@ -233,141 +302,147 @@ do_file() { interactive_echo() { [ "${OVERWRITE_ALL}" != "yes" ] && [ "${DELETE_ALL}" != "yes" ] && echo; } interactive_echo local -i my_input - local -i fcount=0 - until (( $(wc -l < ${TMP}/files/${input}) < 2 )); do - my_input=0 - if (( $(wc -l < ${TMP}/files/${input}) == 2 )); then + local -i linecnt + local fullfile="${TMP}/files/${input}" + local ofile=$(head -n1 "${fullfile}") + + # Walk through all the pending updates for this one file. + linecnt=$(wc -l <"${fullfile}") + while (( linecnt > 1 )) ; do + if (( linecnt == 2 )) ; then + # Only one update ... keeps things simple. my_input=1 + else + my_input=0 fi - until (( ${my_input} > 0 )) && (( ${my_input} < $(wc -l < ${TMP}/files/${input}) )); do - fcount=0 - if [ "${OVERWRITE_ALL}" == "yes" ]; then - my_input=0 - elif [ "${DELETE_ALL}" == "yes" ]; then - my_input=0 - else - for line in $(<${TMP}/files/${input}); do - if (( ${fcount} > 0 )); then - echo -n "${fcount}${PAR} " - echo "${line}" - else - if [[ ${mode} == 0 ]] ; then - echo "Below are the new config files for ${line}:" - else - local my_title="Please select a file to process for ${line}" - fi - fi - fcount=${fcount}+1 - done > ${TMP}/menuitems + # Optimize: no point in scanning the file list when we know + # we're just going to consume all the ones available. + if [[ ${OVERWRITE_ALL} == "yes" || ${DELETE_ALL} == "yes" ]] ; then + my_input=1 + fi - if [[ ${mode} == 0 ]] ; then - cat ${TMP}/menuitems - echo -n "Please select a file to process (-1 to exit this file): " - my_input=$(read_int) - else - dialog --title "${title}" --menu "${my_title}" \ - 0 0 0 $(echo -e "$(<${TMP}/menuitems)\n${fcount} Exit") \ - 2> ${TMP}/input || die "User termination!" 0 - my_input=$(<${TMP}/input) + # Figure out which file they wish to operate on. + while (( my_input <= 0 || my_input >= linecnt )) ; do + local fcount=0 + for line in $(<"${fullfile}"); do + if (( fcount > 0 )); then + printf '%i%s %s\n' ${fcount} "${PAR}" "${line}" fi - fi # OVERWRITE_ALL + : $(( ++fcount )) + done > "${TMP}"/menuitems + + if [[ ${mode} == 0 ]] ; then + echo "Below are the new config files for ${ofile}:" + cat "${TMP}"/menuitems + echo -n "Please select a file to process (-1 to exit this file): " + my_input=$(read_int) + else + dialog \ + --title "${title}" \ + --menu "Please select a file to process for ${ofile}" \ + 0 0 0 $(<"${TMP}"/menuitems) \ + 2> "${TMP}"/input \ + || die "$(<"${TMP}"/input)\n\nUser termination!" 0 + my_input=$(<"${TMP}"/input) + fi if [[ ${my_input} == 0 ]] ; then + # Auto select the first file. my_input=1 elif [[ ${my_input} == -1 ]] ; then input=0 return - elif [[ ${my_input} == ${fcount} ]] ; then - break fi done - if [[ ${my_input} == ${fcount} ]] ; then - break - fi - - fcount=${my_input}+1 - - file=$(sed -e "${fcount}p;d" ${TMP}/files/${input}) - ofile=$(head -n1 ${TMP}/files/${input}) + # First line is the old file while the rest are the config files. + : $(( ++my_input )) + local file=$(sed -n -e "${my_input}p" "${fullfile}") do_cfg "${file}" "${ofile}" - sed -e "${fcount}!p;d" ${TMP}/files/${input} > ${TMP}/files/sed - mv ${TMP}/files/sed ${TMP}/files/${input} + sed -i -e "${my_input}d" "${fullfile}" - if [[ ${my_input} == -1 ]] ; then - break - fi + : $(( --linecnt )) done + interactive_echo - rm ${TMP}/files/${input} - count=${count}-1 + rm "${fullfile}" + : $(( --count )) } -do_cfg() { +show_diff() { + clear + local file1=$1 file2=$2 + if [[ ${using_editor} == 0 ]] ; then + ( + echo "Showing differences between ${file1} and ${file2}" + diff_command "${file1}" "${file2}" + ) | ${pager} + else + echo "Beginning of differences between ${file1} and ${file2}" + diff_command "${file1}" "${file2}" + echo "End of differences between ${file1} and ${file2}" + fi +} - local file="${1}" - local ofile="${2}" +do_cfg() { + local file=$1 + local ofile=$2 local -i my_input=0 - until (( ${my_input} == -1 )) || [ ! -f ${file} ]; do + until (( my_input == -1 )) || [ ! -f "${file}" ] ; do if [[ "${OVERWRITE_ALL}" == "yes" ]] && ! user_special "${ofile}"; then my_input=1 elif [[ "${DELETE_ALL}" == "yes" ]] && ! user_special "${ofile}"; then my_input=2 else - [[ $CLEAR_TERM == yes ]] && clear - if [ "${using_editor}" == 0 ]; then - ( - echo "Showing differences between ${ofile} and ${file}" - diff_command "${ofile}" "${file}" - ) | ${pager} - else - echo "Beginning of differences between ${ofile} and ${file}" - diff_command "${ofile}" "${file}" - echo "End of differences between ${ofile} and ${file}" - fi - if [ -L "${file}" ]; then - echo - echo "-------------------------------------------------------------" - echo "NOTE: File is a symlink to another file. REPLACE recommended." - echo " The original file may simply have moved. Please review." - echo "-------------------------------------------------------------" - echo + show_diff "${ofile}" "${file}" + if [[ -L ${file} ]] ; then + cat <<-EOF + + ------------------------------------------------------------- + NOTE: File is a symlink to another file. REPLACE recommended. + The original file may simply have moved. Please review. + ------------------------------------------------------------- + + EOF fi - echo -n "File: ${file} -1) Replace original with update -2) Delete update, keeping original as is -3) Interactively merge original with update -4) Show differences again -5) Save update as example config -Please select from the menu above (-1 to ignore this update): " + cat <<-EOF + + File: ${file} + 1) Replace original with update + 2) Delete update, keeping original as is + 3) Interactively merge original with update + 4) Show differences again + 5) Save update as example config + EOF + printf 'Please select from the menu above (-1 to ignore this update): ' my_input=$(read_int) fi case ${my_input} in - 1) echo "Replacing ${ofile} with ${file}" - mv ${mv_opts} ${file} ${ofile} - [ -n "${OVERWRITE_ALL}" ] && my_input=-1 - continue - ;; - 2) echo "Deleting ${file}" - rm ${rm_opts} ${file} - [ -n "${DELETE_ALL}" ] && my_input=-1 - continue - ;; - 3) do_merge "${file}" "${ofile}" - my_input=${?} -# [ ${my_input} == 255 ] && my_input=-1 - continue - ;; - 4) continue - ;; - 5) do_distconf "${file}" "${ofile}" - ;; - *) continue - ;; + 1) echo "Replacing ${ofile} with ${file}" + do_mv_ln ${mv_opts} "${file}" "${ofile}" + [ -n "${OVERWRITE_ALL}" ] && my_input=-1 + continue + ;; + 2) echo "Deleting ${file}" + rm ${rm_opts} "${file}" + [ -n "${DELETE_ALL}" ] && my_input=-1 + continue + ;; + 3) do_merge "${file}" "${ofile}" + my_input=${?} +# [ ${my_input} == 255 ] && my_input=-1 + continue + ;; + 4) continue + ;; + 5) do_distconf "${file}" "${ofile}" + ;; + *) continue + ;; esac done } @@ -393,57 +468,48 @@ do_merge() { # need to make sure the full /path/to/ exists ahead of time mkdir -p "${mfile%/*}" - until (( ${my_input} == -1 )); do + until (( my_input == -1 )); do echo "Merging ${file} and ${ofile}" $(echo "${merge_command}" | sed -e "s:%merged:${mfile}:g" \ -e "s:%orig:${ofile}:g" \ -e "s:%new:${file}:g") - until (( ${my_input} == -1 )); do - echo -n "1) Replace ${ofile} with merged file -2) Show differences between merged file and original -3) Remerge original with update -4) Edit merged file -5) Return to the previous menu -Please select from the menu above (-1 to exit, losing this merge): " + until (( my_input == -1 )); do + cat <<-EOF + 1) Replace ${ofile} with merged file + 2) Show differences between merged file and original + 3) Remerge original with update + 4) Edit merged file + 5) Return to the previous menu + EOF + printf 'Please select from the menu above (-1 to exit, losing this merge): ' my_input=$(read_int) case ${my_input} in - 1) echo "Replacing ${ofile} with ${mfile}" - if [[ ${USERLAND} == BSD ]] ; then - chown "$(stat -f %Su:%Sg "${ofile}")" "${mfile}" - chmod $(stat -f %Mp%Lp "${ofile}") "${mfile}" - else - chown --reference="${ofile}" "${mfile}" - chmod --reference="${ofile}" "${mfile}" - fi - mv ${mv_opts} "${mfile}" "${ofile}" - rm ${rm_opts} "${file}" - return 255 - ;; - 2) - [[ $CLEAR_TERM == yes ]] && clear - if [ "${using_editor}" == 0 ]; then - ( - echo "Showing differences between ${ofile} and ${mfile}" - diff_command "${ofile}" "${mfile}" - ) | ${pager} + 1) echo "Replacing ${ofile} with ${mfile}" + if [[ ${USERLAND} == BSD ]] ; then + chown "$(stat -f %Su:%Sg "${ofile}")" "${mfile}" + chmod $(stat -f %Mp%Lp "${ofile}") "${mfile}" else - echo "Beginning of differences between ${ofile} and ${mfile}" - diff_command "${ofile}" "${mfile}" - echo "End of differences between ${ofile} and ${mfile}" + chown --reference="${ofile}" "${mfile}" + chmod --reference="${ofile}" "${mfile}" fi - continue - ;; - 3) break - ;; - 4) ${EDITOR:-nano -w} "${mfile}" - continue - ;; - 5) rm ${rm_opts} "${mfile}" - return 0 - ;; - *) continue - ;; + do_mv_ln ${mv_opts} "${mfile}" "${ofile}" + rm ${rm_opts} "${file}" + return 255 + ;; + 2) show_diff "${ofile}" "${mfile}" + continue + ;; + 3) break + ;; + 4) ${EDITOR:-nano -w} "${mfile}" + continue + ;; + 5) rm ${rm_opts} "${mfile}" + return 0 + ;; + *) continue + ;; esac done done @@ -455,21 +521,15 @@ do_distconf() { # search for any previously saved distribution config # files and number the current one accordingly - local file="${1}" - local ofile="${2}" + local file=$1 ofile=$2 local -i count - local -i fill local suffix local efile - for ((count = 0; count <= 9999; count++)); do - suffix=".dist_" - for ((fill = 4 - ${#count}; fill > 0; fill--)); do - suffix+="0" - done - suffix+="${count}" + for (( count = 0; count <= 9999; ++count )) ; do + suffix=$(printf ".dist_%04i" ${count}) efile="${ofile}${suffix}" - if [[ ! -f ${efile} ]]; then + if [[ ! -f ${efile} ]] ; then mv ${mv_opts} "${file}" "${efile}" break elif diff_command "${file}" "${efile}" &> /dev/null; then @@ -480,35 +540,51 @@ do_distconf() { done } +error() { echo "etc-update: ERROR: $*" 1>&2 ; return 1 ; } die() { trap SIGTERM trap SIGINT + local msg=$1 exitcode=${2:-1} - if [ "$2" -eq 0 ]; then - echo "Exiting: ${1}" + if [ ${exitcode} -eq 0 ] ; then + printf 'Exiting: %b\n' "${msg}" scan > /dev/null [ ${count} -gt 0 ] && echo "NOTE: ${count} updates remaining" else - echo "ERROR: ${1}" + error "${msg}" fi rm -rf "${TMP}" - exit ${2} + exit ${exitcode} } +_3_HELP_TEXT="-3 to auto merge all files" +_5_HELP_TEXT="-5 to auto-merge AND not use 'mv -i'" +_7_HELP_TEXT="-7 to discard all updates" +_9_HELP_TEXT="-9 to discard all updates AND not use 'rm -i'" usage() { cat <<-EOF etc-update: Handle configuration file updates - Usage: etc-update [options] + Usage: etc-update [options] [paths to scan] + + If no paths are specified, then \${CONFIG_PROTECT} will be used. Options: -d, --debug Enable shell debugging -h, --help Show help and run away + -p, --preen Automerge trivial changes only and quit + -v, --verbose Show settings and such along the way -V, --version Show version and trundle away + + --automode <mode> + ${_3_HELP_TEXT} + ${_5_HELP_TEXT} + ${_7_HELP_TEXT} + ${_9_HELP_TEXT} EOF - [[ -n ${*:2} ]] && printf "\nError: %s\n" "${*:2}" 1>&2 + [[ $# -gt 1 ]] && printf "\nError: %s\n" "${*:2}" 1>&2 exit ${1:-0} } @@ -517,98 +593,121 @@ usage() { # Run the script # +declare -i count=0 +declare input=0 +declare title="Gentoo's etc-update tool!" + +PREEN=false SET_X=false +VERBOSE=false while [[ -n $1 ]] ; do case $1 in -d|--debug) SET_X=true;; -h|--help) usage;; - -V|--version) emerge --version ; exit 0;; - *) usage 1 "Invalid option '$1'";; + -p|--preen) PREEN=true;; + -v|--verbose) VERBOSE=true;; + -V|--version) emerge --version; exit 0;; + --automode) parse_automode_flag $2 && shift || usage 1 "Invalid mode '$2'";; + -*) usage 1 "Invalid option '$1'";; + *) break;; esac shift done ${SET_X} && set -x -type portageq > /dev/null || exit $? -eval $(portageq envvar -v CONFIG_PROTECT \ - CONFIG_PROTECT_MASK PORTAGE_CONFIGROOT PORTAGE_TMPDIR ROOT USERLAND) +type portageq >/dev/null || die "missing portageq" +portage_vars=( + CONFIG_PROTECT{,_MASK} + PORTAGE_CONFIGROOT + PORTAGE_INST_{G,U}ID + PORTAGE_TMPDIR + EROOT + USERLAND + NOCOLOR +) +eval $(portageq envvar -v ${portage_vars[@]}) export PORTAGE_TMPDIR +SCAN_PATHS=${*:-${CONFIG_PROTECT}} TMP="${PORTAGE_TMPDIR}/etc-update-$$" -trap "die terminated 1" SIGTERM -trap "die interrupted 1" SIGINT +trap "die terminated" SIGTERM +trap "die interrupted" SIGINT -[ -w ${PORTAGE_CONFIGROOT}etc ] || die "Need write access to ${PORTAGE_CONFIGROOT}etc" 1 -#echo $PORTAGE_TMPDIR -#echo $CONFIG_PROTECT -#echo $CONFIG_PROTECT_MASK -#export PORTAGE_TMPDIR=$(/usr/lib/portage/bin/portageq envvar PORTAGE_TMPDIR) - -rm -rf "${TMP}" 2> /dev/null -mkdir "${TMP}" || die "failed to create temp dir" 1 +rm -rf "${TMP}" 2>/dev/null +mkdir "${TMP}" || die "failed to create temp dir" # make sure we have a secure directory to work in -chmod 0700 "${TMP}" || die "failed to set perms on temp dir" 1 -chown ${UID:-0}:${GID:-0} "${TMP}" || die "failed to set ownership on temp dir" 1 - -# I need the CONFIG_PROTECT value -#CONFIG_PROTECT=$(/usr/lib/portage/bin/portageq envvar CONFIG_PROTECT) -#CONFIG_PROTECT_MASK=$(/usr/lib/portage/bin/portageq envvar CONFIG_PROTECT_MASK) - -# load etc-config's configuration -CLEAR_TERM=$(get_config clear_term) -EU_AUTOMERGE=$(get_config eu_automerge) -rm_opts=$(get_config rm_opts) -mv_opts=$(get_config mv_opts) -cp_opts=$(get_config cp_opts) -pager=$(get_config pager) -diff_command=$(get_config diff_command) -using_editor=$(get_config using_editor) -merge_command=$(get_config merge_command) -declare -i mode=$(get_config mode) -[[ -z ${mode} ]] && mode=0 -[[ -z ${pager} ]] && pager="cat" - -if [ "${using_editor}" == 0 ]; then +chmod 0700 "${TMP}" || die "failed to set perms on temp dir" +chown ${PORTAGE_INST_UID:-0}:${PORTAGE_INST_GID:-0} "${TMP}" || \ + die "failed to set ownership on temp dir" + +# Get all the user settings from etc-update.conf +cfg_vars=( + clear_term + eu_automerge + rm_opts + mv_opts + pager + diff_command + using_editor + merge_command + mode +) +# default them all to "" +eval ${cfg_vars[@]/%/=} +# then extract them all from the conf in one shot +# (ugly var at end is due to printf appending a '|' to last item) +get_config "($(printf '%s|' "${cfg_vars[@]}")NOVARFOROLDMEN)" + +# finally setup any specific defaults +: ${mode:="0"} +if ! cmd_var_is_valid "${pager}" ; then + pager=${PAGER} + cmd_var_is_valid "${pager}" || pager=cat +fi + +[[ ${clear_term} == "yes" ]] || clear() { :; } + +if [[ ${using_editor} == "0" ]] ; then # Sanity check to make sure diff exists and works echo > "${TMP}"/.diff-test-1 echo > "${TMP}"/.diff-test-2 - + if ! diff_command "${TMP}"/.diff-test-1 "${TMP}"/.diff-test-2 ; then - die "'${diff_command}' does not seem to work, aborting" 1 + die "'${diff_command}' does not seem to work, aborting" fi else - if ! type ${diff_command%% *} >/dev/null; then - die "'${diff_command}' does not seem to work, aborting" 1 + # NOTE: cmd_var_is_valid doesn't work with diff_command="eval emacs..." + # because it uses type -P. + if ! type ${diff_command%%[[:space:]]*} >/dev/null; then + die "'${diff_command}' does not seem to work, aborting" fi fi -if [[ ${mode} == "1" ]] ; then - if ! type dialog >/dev/null || ! dialog --help >/dev/null ; then - die "mode=1 and 'dialog' not found or not executable, aborting" 1 - fi -fi - -#echo "rm_opts: $rm_opts, mv_opts: $mv_opts, cp_opts: $cp_opts" -#echo "pager: $pager, diff_command: $diff_command, merge_command: $merge_command" - -if (( ${mode} == 0 )); then +if [[ ${mode} == "0" ]] ; then PAR=")" else PAR="" + if ! type dialog >/dev/null || ! dialog --help >/dev/null ; then + die "mode=1 and 'dialog' not found or not executable, aborting" + fi fi -declare -i count=0 -declare input=0 -declare title="Gentoo's etc-update tool!" +if ${VERBOSE} ; then + for v in ${portage_vars[@]} ${cfg_vars[@]} TMP SCAN_PATHS ; do + echo "${v}=${!v}" + done +fi scan -until (( ${input} == -1 )); do - if (( ${count} == 0 )); then +${PREEN} && exit 0 + +until (( input == -1 )); do + if (( count == 0 )); then die "Nothing left to do; exiting. :)" 0 fi sel_file - if (( ${input} != -1 )); then + if (( input != -1 )); then do_file fi done diff --git a/portage_with_autodep/bin/fixpackages b/portage_with_autodep/bin/fixpackages index 5e1df70..dc43ed2 100755 --- a/portage_with_autodep/bin/fixpackages +++ b/portage_with_autodep/bin/fixpackages @@ -1,11 +1,12 @@ #!/usr/bin/python -# Copyright 1999-2006 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function -import os,sys -os.environ["PORTAGE_CALLER"]="fixpackages" +import os +import sys + try: import portage except ImportError: @@ -38,7 +39,7 @@ except (OSError, ValueError) as e: portage.writemsg("!!! %s\n" % str(e)) del e -_global_updates(mytrees, mtimedb["updates"]) +_global_updates(mytrees, mtimedb["updates"], if_mtime_changed=False) print() print("Done.") diff --git a/portage_with_autodep/bin/glsa-check b/portage_with_autodep/bin/glsa-check index 2f2d555..a840c32 100755 --- a/portage_with_autodep/bin/glsa-check +++ b/portage_with_autodep/bin/glsa-check @@ -103,8 +103,9 @@ elif mode == "list" and not params: # delay this for speed increase from portage.glsa import * -vardb = portage.db[portage.settings["ROOT"]]["vartree"].dbapi -portdb = portage.db["/"]["porttree"].dbapi +eroot = portage.settings['EROOT'] +vardb = portage.db[eroot]["vartree"].dbapi +portdb = portage.db[eroot]["porttree"].dbapi # build glsa lists completelist = get_glsa_list(portage.settings) diff --git a/portage_with_autodep/bin/helper-functions.sh b/portage_with_autodep/bin/helper-functions.sh new file mode 100755 index 0000000..afe14af --- /dev/null +++ b/portage_with_autodep/bin/helper-functions.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright 1999-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# For routines we want to use in ebuild-helpers/ but don't want to +# expose to the general ebuild environment. + +source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}"/isolated-functions.sh + +# +# API functions for doing parallel processing +# +numjobs() { + # Copied from eutils.eclass:makeopts_jobs() + local jobs=$(echo " ${MAKEOPTS} " | \ + sed -r -n 's:.*[[:space:]](-j|--jobs[=[:space:]])[[:space:]]*([0-9]+).*:\2:p') + echo ${jobs:-1} +} + +multijob_init() { + # Setup a pipe for children to write their pids to when they finish. + mj_control_pipe=$(mktemp -t multijob.XXXXXX) + rm "${mj_control_pipe}" + mkfifo "${mj_control_pipe}" + exec {mj_control_fd}<>${mj_control_pipe} + rm -f "${mj_control_pipe}" + + # See how many children we can fork based on the user's settings. + mj_max_jobs=$(numjobs) + mj_num_jobs=0 +} + +multijob_child_init() { + trap 'echo ${BASHPID} $? >&'${mj_control_fd} EXIT + trap 'exit 1' INT TERM +} + +multijob_finish_one() { + local pid ret + read -r -u ${mj_control_fd} pid ret + : $(( --mj_num_jobs )) + return ${ret} +} + +multijob_finish() { + local ret=0 + while [[ ${mj_num_jobs} -gt 0 ]] ; do + multijob_finish_one + : $(( ret |= $? )) + done + # Let bash clean up its internal child tracking state. + wait + return ${ret} +} + +multijob_post_fork() { + : $(( ++mj_num_jobs )) + if [[ ${mj_num_jobs} -ge ${mj_max_jobs} ]] ; then + multijob_finish_one + fi + return $? +} diff --git a/portage_with_autodep/bin/install.py b/portage_with_autodep/bin/install.py new file mode 100755 index 0000000..2c6dfbe --- /dev/null +++ b/portage_with_autodep/bin/install.py @@ -0,0 +1,253 @@ +#!/usr/bin/python +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import stat +import sys +import subprocess +import traceback + +import portage +from portage.util._argparse import ArgumentParser +from portage.util.movefile import _copyxattr +from portage.exception import OperationNotSupported + +# Change back to original cwd _after_ all imports (bug #469338). +os.chdir(os.environ["__PORTAGE_HELPER_CWD"]) + +def parse_args(args): + """ + Parse the command line arguments using optparse for python 2.6 compatibility + Args: + args: a list of the white space delimited command line + Returns: + tuple of the Namespace of parsed options, and a list of order parameters + """ + parser = ArgumentParser(add_help=False) + + parser.add_argument( + "-b", + action="store_true", + dest="shortopt_b" + ) + parser.add_argument( + "--backup", + action="store", + dest="backup" + ) + parser.add_argument( + "-c", + action="store_true", + dest="shortopt_c" + ) + parser.add_argument( + "--compare", + "-C", + action="store_true", + dest="compare" + ) + parser.add_argument( + "--directory", + "-d", + action="store_true", + dest="directory" + ) + parser.add_argument( + "-D", + action="store_true", + dest="shortopt_D" + ) + parser.add_argument( + "--owner", + "-o", + action="store", + dest="owner" + ) + parser.add_argument( + "--group", + "-g", + action="store", + dest="group" + ) + parser.add_argument( + "--mode", + "-m", + action="store", + dest="mode" + ) + parser.add_argument( + "--preserve-timestamps", + "-p", + action="store_true", + dest="preserve_timestamps" + ) + parser.add_argument( + "--strip", + "-s", + action="store_true", + dest="strip" + ) + parser.add_argument( + "--strip-program", + action="store", + dest="strip_program" + ) + parser.add_argument( + "--suffix", + "-S", + action="store", + dest="suffix" + ) + parser.add_argument( + "--target-directory", + "-t", + action="store", + dest="target_directory" + ) + parser.add_argument( + "--no-target-directory", + "-T", + action="store_true", + dest="no_target_directory" + ) + parser.add_argument( + "--context", + "-Z", + action="store", + dest="context" + ) + parser.add_argument( + "--verbose", + "-v", + action="store_true", + dest="verbose" + ) + parser.add_argument( + "--help", + action="store_true", + dest="help" + ) + parser.add_argument( + "--version", + action="store_true", + dest="version" + ) + + # Use parse_known_args for maximum compatibility with + # getopt handling of non-option file arguments. Note + # that parser.add_argument("files", nargs='+') would + # be subtly incompatible because it requires that all + # of the file arguments be grouped sequentially. Also + # note that we have to explicitly call add_argument + # for known options in order for argparse to correctly + # separate option arguments from file arguments in all + # cases (it also allows for optparse compatibility). + parsed_args = parser.parse_known_args() + + opts = parsed_args[0] + files = parsed_args[1] + files = [f for f in files if f != "--"] # filter out "--" + + return (opts, files) + + +def copy_xattrs(opts, files): + """ + Copy the extended attributes using portage.util.movefile._copyxattr + Args: + opts: Namespace of the parsed command line otions + files: list of ordered command line parameters which should be files/directories + Returns: + system exit code + """ + if opts.directory or not files: + return os.EX_OK + + if opts.target_directory is None: + source, target = files[:-1], files[-1] + target_is_directory = os.path.isdir(target) + else: + source, target = files, opts.target_directory + target_is_directory = True + + exclude = os.environ.get("PORTAGE_XATTR_EXCLUDE", "security.* system.nfs4_acl") + + try: + if target_is_directory: + for s in source: + abs_path = os.path.join(target, os.path.basename(s)) + _copyxattr(s, abs_path, exclude=exclude) + else: + _copyxattr(source[0], target, exclude=exclude) + return os.EX_OK + + except OperationNotSupported: + traceback.print_exc() + return os.EX_OSERR + + +def Which(filename, path=None, exclude=None): + """ + Find the absolute path of 'filename' in a given search 'path' + Args: + filename: basename of the file + path: colon delimited search path + exclude: path of file to exclude + """ + if path is None: + path = os.environ.get('PATH', '') + + if exclude is not None: + st = os.stat(exclude) + exclude = (st.st_ino, st.st_dev) + + for p in path.split(':'): + p = os.path.join(p, filename) + if os.access(p, os.X_OK): + try: + st = os.stat(p) + except OSError: + # file disappeared? + pass + else: + if stat.S_ISREG(st.st_mode) and \ + (exclude is None or exclude != (st.st_ino, st.st_dev)): + return p + + return None + + +def main(args): + opts, files = parse_args(args) + install_binary = Which('install', exclude=os.environ["__PORTAGE_HELPER_PATH"]) + if install_binary is None: + sys.stderr.write("install: command not found\n") + return 127 + + cmdline = [install_binary] + cmdline += args + + if sys.hexversion >= 0x3000000: + # We can't trust that the filesystem encoding (locale dependent) + # correctly matches the arguments, so use surrogateescape to + # pass through the original argv bytes for Python 3. + fs_encoding = sys.getfilesystemencoding() + cmdline = [x.encode(fs_encoding, 'surrogateescape') for x in cmdline] + files = [x.encode(fs_encoding, 'surrogateescape') for x in files] + if opts.target_directory is not None: + opts.target_directory = \ + opts.target_directory.encode(fs_encoding, 'surrogateescape') + + returncode = subprocess.call(cmdline) + if returncode == os.EX_OK: + returncode = copy_xattrs(opts, files) + if returncode != os.EX_OK: + portage.util.writemsg("!!! install: copy_xattrs failed with the " + "following arguments: %s\n" % + " ".join(portage._shell_quote(x) for x in args), noiselevel=-1) + return returncode + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/portage_with_autodep/bin/isolated-functions.sh b/portage_with_autodep/bin/isolated-functions.sh index 65bb1d5..dbf988b 100644..100755 --- a/portage_with_autodep/bin/isolated-functions.sh +++ b/portage_with_autodep/bin/isolated-functions.sh @@ -173,8 +173,8 @@ die() { | while read -r n ; do eerror " ${n#RETAIN-LEADING-SPACE}" ; done eerror fi - eerror "If you need support, post the output of 'emerge --info =$CATEGORY/$PF'," - eerror "the complete build log and the output of 'emerge -pqv =$CATEGORY/$PF'." + eerror "If you need support, post the output of \`emerge --info '=$CATEGORY/$PF'\`," + eerror "the complete build log and the output of \`emerge -pqv '=$CATEGORY/$PF'\`." if [[ -n ${EBUILD_OVERLAY_ECLASSES} ]] ; then eerror "This ebuild used the following eclasses from overlays:" local x @@ -203,7 +203,12 @@ die() { fi fi - if [[ "${EBUILD_PHASE/depend}" == "${EBUILD_PHASE}" ]] ; then + # Only call die hooks here if we are executed via ebuild.sh or + # misc-functions.sh, since those are the only cases where the environment + # contains the hook functions. When necessary (like for helpers_die), die + # hooks are automatically called later by a misc-functions.sh invocation. + if has ${BASH_SOURCE[$main_index]##*/} ebuild.sh misc-functions.sh && \ + [[ ${EBUILD_PHASE} != depend ]] ; then local x for x in $EBUILD_DEATH_HOOKS; do ${x} "$@" >&2 1>&2 @@ -211,8 +216,15 @@ die() { > "$PORTAGE_BUILDDIR/.die_hooks" fi - [[ -n ${PORTAGE_LOG_FILE} ]] \ - && eerror "The complete build log is located at '${PORTAGE_LOG_FILE}'." + if [[ -n ${PORTAGE_LOG_FILE} ]] ; then + eerror "The complete build log is located at '${PORTAGE_LOG_FILE}'." + if [[ ${PORTAGE_LOG_FILE} != ${T}/* ]] ; then + # Display path to symlink in ${T}, as requested in bug #412865. + local log_ext=log + [[ ${PORTAGE_LOG_FILE} != *.log ]] && log_ext+=.${PORTAGE_LOG_FILE##*.} + eerror "For convenience, a symlink to the build log is located at '${T}/build.${log_ext}'." + fi + fi if [ -f "${T}/environment" ] ; then eerror "The ebuild environment file is located at '${T}/environment'." elif [ -d "${T}" ] ; then @@ -222,6 +234,7 @@ die() { } > "${T}/die.env" eerror "The ebuild environment file is located at '${T}/die.env'." fi + eerror "Working directory: '$(pwd)'" eerror "S: '${S}'" [[ -n $PORTAGE_EBUILD_EXIT_FILE ]] && > "$PORTAGE_EBUILD_EXIT_FILE" @@ -392,57 +405,6 @@ eend() { return ${retval} } -KV_major() { - [[ -z $1 ]] && return 1 - - local KV=$@ - echo "${KV%%.*}" -} - -KV_minor() { - [[ -z $1 ]] && return 1 - - local KV=$@ - KV=${KV#*.} - echo "${KV%%.*}" -} - -KV_micro() { - [[ -z $1 ]] && return 1 - - local KV=$@ - KV=${KV#*.*.} - echo "${KV%%[^[:digit:]]*}" -} - -KV_to_int() { - [[ -z $1 ]] && return 1 - - local KV_MAJOR=$(KV_major "$1") - local KV_MINOR=$(KV_minor "$1") - local KV_MICRO=$(KV_micro "$1") - local KV_int=$(( KV_MAJOR * 65536 + KV_MINOR * 256 + KV_MICRO )) - - # We make version 2.2.0 the minimum version we will handle as - # a sanity check ... if its less, we fail ... - if [[ ${KV_int} -ge 131584 ]] ; then - echo "${KV_int}" - return 0 - fi - - return 1 -} - -_RC_GET_KV_CACHE="" -get_KV() { - [[ -z ${_RC_GET_KV_CACHE} ]] \ - && _RC_GET_KV_CACHE=$(uname -r) - - echo $(KV_to_int "${_RC_GET_KV_CACHE}") - - return $? -} - unset_colors() { COLS=80 ENDCOL= @@ -457,7 +419,13 @@ unset_colors() { set_colors() { COLS=${COLUMNS:-0} # bash's internal COLUMNS variable - (( COLS == 0 )) && COLS=$(set -- $(stty size 2>/dev/null) ; echo $2) + # Avoid wasteful stty calls during the "depend" phases. + # If stdout is a pipe, the parent process can export COLUMNS + # if it's relevant. Use an extra subshell for stty calls, in + # order to redirect "/dev/tty: No such device or address" + # error from bash to /dev/null. + [[ $COLS == 0 && $EBUILD_PHASE != depend ]] && \ + COLS=$(set -- $( ( stty size </dev/tty ) 2>/dev/null || echo 24 80 ) ; echo $2) (( COLS > 0 )) || (( COLS = 80 )) # Now, ${ENDCOL} will move us to the end of the @@ -471,8 +439,8 @@ set_colors() { BAD=$'\e[31;01m' HILITE=$'\e[36;01m' BRACKET=$'\e[34;01m' + NORMAL=$'\e[0m' fi - NORMAL=$'\e[0m' } RC_ENDCOL="yes" @@ -536,95 +504,4 @@ has() { return 1 } -# @FUNCTION: save_ebuild_env -# @DESCRIPTION: -# echo the current environment to stdout, filtering out redundant info. -# -# --exclude-init-phases causes pkg_nofetch and src_* phase functions to -# be excluded from the output. These function are not needed for installation -# or removal of the packages, and can therefore be safely excluded. -# -save_ebuild_env() { - ( - - if has --exclude-init-phases $* ; then - unset S _E_DOCDESTTREE_ _E_EXEDESTTREE_ - if [[ -n $PYTHONPATH ]] ; then - export PYTHONPATH=${PYTHONPATH/${PORTAGE_PYM_PATH}:} - [[ -z $PYTHONPATH ]] && unset PYTHONPATH - fi - fi - - # misc variables inherited from the calling environment - unset COLORTERM DISPLAY EDITOR LESS LESSOPEN LOGNAME LS_COLORS PAGER \ - TERM TERMCAP USER ftp_proxy http_proxy no_proxy - - # other variables inherited from the calling environment - unset CVS_RSH ECHANGELOG_USER GPG_AGENT_INFO \ - SSH_AGENT_PID SSH_AUTH_SOCK STY WINDOW XAUTHORITY - - # CCACHE and DISTCC config - unset ${!CCACHE_*} ${!DISTCC_*} - - # There's no need to bloat environment.bz2 with internally defined - # functions and variables, so filter them out if possible. - - for x in pkg_setup pkg_nofetch src_unpack src_prepare src_configure \ - src_compile src_test src_install pkg_preinst pkg_postinst \ - pkg_prerm pkg_postrm ; do - unset -f default_$x _eapi{0,1,2,3,4}_$x - done - unset x - - unset -f assert assert_sigpipe_ok dump_trace die diefunc \ - quiet_mode vecho elog_base eqawarn elog \ - esyslog einfo einfon ewarn eerror ebegin _eend eend KV_major \ - KV_minor KV_micro KV_to_int get_KV unset_colors set_colors has \ - has_phase_defined_up_to \ - hasg hasgq hasv hasq qa_source qa_call \ - addread addwrite adddeny addpredict _sb_append_var \ - lchown lchgrp esyslog use usev useq has_version portageq \ - best_version use_with use_enable register_die_hook \ - keepdir unpack strip_duplicate_slashes econf einstall \ - dyn_setup dyn_unpack dyn_clean into insinto exeinto docinto \ - insopts diropts exeopts libopts docompress \ - abort_handler abort_prepare abort_configure abort_compile \ - abort_test abort_install dyn_prepare dyn_configure \ - dyn_compile dyn_test dyn_install \ - dyn_preinst dyn_help debug-print debug-print-function \ - debug-print-section inherit EXPORT_FUNCTIONS remove_path_entry \ - save_ebuild_env filter_readonly_variables preprocess_ebuild_env \ - set_unless_changed unset_unless_changed source_all_bashrcs \ - ebuild_main ebuild_phase ebuild_phase_with_hooks \ - _ebuild_arg_to_phase _ebuild_phase_funcs default \ - _pipestatus \ - ${QA_INTERCEPTORS} - - # portage config variables and variables set directly by portage - unset ACCEPT_LICENSE BAD BRACKET BUILD_PREFIX COLS \ - DISTCC_DIR DISTDIR DOC_SYMLINKS_DIR \ - EBUILD_FORCE_TEST EBUILD_MASTER_PID \ - ECLASS_DEPTH ENDCOL FAKEROOTKEY \ - GOOD HILITE HOME \ - LAST_E_CMD LAST_E_LEN LD_PRELOAD MISC_FUNCTIONS_ARGS MOPREFIX \ - NOCOLOR NORMAL PKGDIR PKGUSE PKG_LOGDIR PKG_TMPDIR \ - PORTAGE_BASHRCS_SOURCED PORTAGE_NONFATAL PORTAGE_QUIET \ - PORTAGE_SANDBOX_DENY PORTAGE_SANDBOX_PREDICT \ - PORTAGE_SANDBOX_READ PORTAGE_SANDBOX_WRITE PREROOTPATH \ - QA_INTERCEPTORS \ - RC_DEFAULT_INDENT RC_DOT_PATTERN RC_ENDCOL RC_INDENTATION \ - ROOT ROOTPATH RPMDIR TEMP TMP TMPDIR USE_EXPAND \ - WARN XARGS _RC_GET_KV_CACHE - - # user config variables - unset DOC_SYMLINKS_DIR INSTALL_MASK PKG_INSTALL_MASK - - declare -p - declare -fp - if [[ ${BASH_VERSINFO[0]} == 3 ]]; then - export - fi - ) -} - true diff --git a/portage_with_autodep/bin/lock-helper.py b/portage_with_autodep/bin/lock-helper.py index 5f3ea9f..dfb8876 100755 --- a/portage_with_autodep/bin/lock-helper.py +++ b/portage_with_autodep/bin/lock-helper.py @@ -1,15 +1,16 @@ #!/usr/bin/python -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os import sys sys.path.insert(0, os.environ['PORTAGE_PYM_PATH']) import portage +portage._disable_legacy_globals() def main(args): - if args and sys.hexversion < 0x3000000 and not isinstance(args[0], unicode): + if args and isinstance(args[0], bytes): for i, x in enumerate(args): args[i] = portage._unicode_decode(x, errors='strict') diff --git a/portage_with_autodep/bin/misc-functions.sh b/portage_with_autodep/bin/misc-functions.sh index 8c191ff..564af85 100755 --- a/portage_with_autodep/bin/misc-functions.sh +++ b/portage_with_autodep/bin/misc-functions.sh @@ -17,7 +17,9 @@ shift $# source "${PORTAGE_BIN_PATH:-/usr/lib/portage/bin}/ebuild.sh" install_symlink_html_docs() { - cd "${D}" || die "cd failed" + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + cd "${ED}" || die "cd failed" #symlink the html documentation (if DOC_SYMLINKS_DIR is set in make.conf) if [ -n "${DOC_SYMLINKS_DIR}" ] ; then local mydocdir docdir @@ -64,11 +66,13 @@ canonicalize() { prepcompress() { local -a include exclude incl_d incl_f local f g i real_f real_d + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac # Canonicalize path names and check for their existence. - real_d=$(canonicalize "${D}") + real_d=$(canonicalize "${ED}") for (( i = 0; i < ${#PORTAGE_DOCOMPRESS[@]}; i++ )); do - real_f=$(canonicalize "${D}${PORTAGE_DOCOMPRESS[i]}") + real_f=$(canonicalize "${ED}${PORTAGE_DOCOMPRESS[i]}") f=${real_f#"${real_d}"} if [[ ${real_f} != "${f}" ]] && [[ -d ${real_f} || -f ${real_f} ]] then @@ -79,7 +83,7 @@ prepcompress() { fi done for (( i = 0; i < ${#PORTAGE_DOCOMPRESS_SKIP[@]}; i++ )); do - real_f=$(canonicalize "${D}${PORTAGE_DOCOMPRESS_SKIP[i]}") + real_f=$(canonicalize "${ED}${PORTAGE_DOCOMPRESS_SKIP[i]}") f=${real_f#"${real_d}"} if [[ ${real_f} != "${f}" ]] && [[ -d ${real_f} || -f ${real_f} ]] then @@ -128,7 +132,7 @@ prepcompress() { # Split the include list into directories and files for f in "${include[@]}"; do - if [[ -d ${D}${f} ]]; then + if [[ -d ${ED}${f} ]]; then incl_d[${#incl_d[@]}]=${f} else incl_f[${#incl_f[@]}]=${f} @@ -138,15 +142,101 @@ prepcompress() { # Queue up for compression. # ecompress{,dir} doesn't like to be called with empty argument lists. [[ ${#incl_d[@]} -gt 0 ]] && ecompressdir --queue "${incl_d[@]}" - [[ ${#incl_f[@]} -gt 0 ]] && ecompress --queue "${incl_f[@]/#/${D}}" + [[ ${#incl_f[@]} -gt 0 ]] && ecompress --queue "${incl_f[@]/#/${ED}}" [[ ${#exclude[@]} -gt 0 ]] && ecompressdir --ignore "${exclude[@]}" return 0 } install_qa_check() { - local f x + local f i qa_var x + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local EPREFIX= ED=${D} ;; esac + + cd "${ED}" || die "cd failed" + + # Merge QA_FLAGS_IGNORED and QA_DT_HASH into a single array, since + # QA_DT_HASH is deprecated. + qa_var="QA_FLAGS_IGNORED_${ARCH/-/_}" + eval "[[ -n \${!qa_var} ]] && QA_FLAGS_IGNORED=(\"\${${qa_var}[@]}\")" + if [[ ${#QA_FLAGS_IGNORED[@]} -eq 1 ]] ; then + local shopts=$- + set -o noglob + QA_FLAGS_IGNORED=(${QA_FLAGS_IGNORED}) + set +o noglob + set -${shopts} + fi + + qa_var="QA_DT_HASH_${ARCH/-/_}" + eval "[[ -n \${!qa_var} ]] && QA_DT_HASH=(\"\${${qa_var}[@]}\")" + if [[ ${#QA_DT_HASH[@]} -eq 1 ]] ; then + local shopts=$- + set -o noglob + QA_DT_HASH=(${QA_DT_HASH}) + set +o noglob + set -${shopts} + fi + + if [[ -n ${QA_DT_HASH} ]] ; then + QA_FLAGS_IGNORED=("${QA_FLAGS_IGNORED[@]}" "${QA_DT_HASH[@]}") + unset QA_DT_HASH + fi - cd "${D}" || die "cd failed" + # Merge QA_STRICT_FLAGS_IGNORED and QA_STRICT_DT_HASH, since + # QA_STRICT_DT_HASH is deprecated + if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] && \ + [ "${QA_STRICT_DT_HASH-unset}" != unset ] ; then + QA_STRICT_FLAGS_IGNORED=1 + unset QA_STRICT_DT_HASH + fi + + # Check for files built without respecting *FLAGS. Note that + # -frecord-gcc-switches must be in all *FLAGS variables, in + # order to avoid false positive results here. + # NOTE: This check must execute before prepall/prepstrip, since + # prepstrip strips the .GCC.command.line sections. + if type -P scanelf > /dev/null && ! has binchecks ${RESTRICT} && \ + [[ "${CFLAGS}" == *-frecord-gcc-switches* ]] && \ + [[ "${CXXFLAGS}" == *-frecord-gcc-switches* ]] && \ + [[ "${FFLAGS}" == *-frecord-gcc-switches* ]] && \ + [[ "${FCFLAGS}" == *-frecord-gcc-switches* ]] ; then + rm -f "${T}"/scanelf-ignored-CFLAGS.log + for x in $(scanelf -qyRF '%k %p' -k \!.GCC.command.line "${ED}" | \ + sed -e "s:\!.GCC.command.line ::") ; do + # Separate out file types that are known to support + # .GCC.command.line sections, using the `file` command + # similar to how prepstrip uses it. + f=$(file "${x}") || continue + [[ -z ${f} ]] && continue + if [[ ${f} == *"SB executable"* || + ${f} == *"SB shared object"* ]] ; then + echo "${x}" >> "${T}"/scanelf-ignored-CFLAGS.log + fi + done + + if [[ -f "${T}"/scanelf-ignored-CFLAGS.log ]] ; then + + if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] ; then + for x in "${QA_FLAGS_IGNORED[@]}" ; do + sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-CFLAGS.log + done + fi + # Filter anything under /usr/lib/debug/ in order to avoid + # duplicate warnings for splitdebug files. + sed -e "s#^usr/lib/debug/.*##" -e "/^\$/d" -e "s#^#/#" \ + -i "${T}"/scanelf-ignored-CFLAGS.log + f=$(<"${T}"/scanelf-ignored-CFLAGS.log) + if [[ -n ${f} ]] ; then + vecho -ne '\n' + eqawarn "${BAD}QA Notice: Files built without respecting CFLAGS have been detected${NORMAL}" + eqawarn " Please include the following list of files in your report:" + eqawarn "${f}" + vecho -ne '\n' + sleep 1 + else + rm -f "${T}"/scanelf-ignored-CFLAGS.log + fi + fi + fi export STRIP_MASK prepall @@ -154,9 +244,12 @@ install_qa_check() { ecompressdir --dequeue ecompress --dequeue + # Prefix specific checks + [[ ${ED} != ${D} ]] && install_qa_check_prefix + f= for x in etc/app-defaults usr/man usr/info usr/X11R6 usr/doc usr/locale ; do - [[ -d $D/$x ]] && f+=" $x\n" + [[ -d ${ED}/$x ]] && f+=" $x\n" done if [[ -n $f ]] ; then @@ -165,18 +258,30 @@ install_qa_check() { eqawarn "$f" fi + if [[ -d ${ED}/etc/udev/rules.d ]] ; then + f= + for x in $(ls "${ED}/etc/udev/rules.d") ; do + f+=" etc/udev/rules.d/$x\n" + done + if [[ -n $f ]] ; then + eqawarn "QA Notice: udev rules should be installed in /lib/udev/rules.d:" + eqawarn + eqawarn "$f" + fi + fi + # Now we look for all world writable files. - local i - for i in $(find "${D}/" -type f -perm -2); do - vecho "QA Security Notice:" - vecho "- ${i:${#D}:${#i}} will be a world writable file." + local unsafe_files=$(find "${ED}" -type f -perm -2 | sed -e "s:^${ED}:- :") + if [[ -n ${unsafe_files} ]] ; then + vecho "QA Security Notice: world writable file(s):" + vecho "${unsafe_files}" vecho "- This may or may not be a security problem, most of the time it is one." vecho "- Please double check that $PF really needs a world writeable bit and file bugs accordingly." sleep 1 - done + fi if type -P scanelf > /dev/null && ! has binchecks ${RESTRICT}; then - local qa_var insecure_rpath=0 tmp_quiet=${PORTAGE_QUIET} + local insecure_rpath=0 tmp_quiet=${PORTAGE_QUIET} local x # display warnings when using stricter because we die afterwards @@ -196,7 +301,7 @@ install_qa_check() { if [[ -n "${ROOT}" && "${ROOT}" != "/" ]]; then forbidden_dirs+=" ${ROOT}" fi - local dir l rpath_files=$(scanelf -F '%F:%r' -qBR "${D}") + local dir l rpath_files=$(scanelf -F '%F:%r' -qBR "${ED}") f="" for dir in ${forbidden_dirs}; do for l in $(echo "${rpath_files}" | grep -E ":${dir}|::|: "); do @@ -210,7 +315,7 @@ install_qa_check() { # Reject set*id binaries with $ORIGIN in RPATH #260331 x=$( - find "${D}" -type f \( -perm -u+s -o -perm -g+s \) -print0 | \ + find "${ED}" -type f \( -perm -u+s -o -perm -g+s \) -print0 | \ xargs -0 scanelf -qyRF '%r %p' | grep '$ORIGIN' ) @@ -236,7 +341,7 @@ install_qa_check() { [[ -n ${!qa_var} ]] && QA_TEXTRELS=${!qa_var} [[ -n ${QA_STRICT_TEXTRELS} ]] && QA_TEXTRELS="" export QA_TEXTRELS="${QA_TEXTRELS} lib*/modules/*.ko" - f=$(scanelf -qyRF '%t %p' "${D}" | grep -v 'usr/lib/debug/') + f=$(scanelf -qyRF '%t %p' "${ED}" | grep -v 'usr/lib/debug/') if [[ -n ${f} ]] ; then scanelf -qyRAF '%T %p' "${PORTAGE_BUILDDIR}"/ &> "${T}"/scanelf-textrel.log vecho -ne '\n' @@ -276,7 +381,7 @@ install_qa_check() { [[ -n ${QA_STRICT_WX_LOAD} ]] && QA_WX_LOAD="" export QA_EXECSTACK="${QA_EXECSTACK} lib*/modules/*.ko" export QA_WX_LOAD="${QA_WX_LOAD} lib*/modules/*.ko" - f=$(scanelf -qyRAF '%e %p' "${D}" | grep -v 'usr/lib/debug/') + f=$(scanelf -qyRAF '%e %p' "${ED}" | grep -v 'usr/lib/debug/') ;; esac ;; @@ -300,26 +405,15 @@ install_qa_check() { fi # Check for files built without respecting LDFLAGS - if [[ "${LDFLAGS}" == *,--hash-style=gnu* ]] && [[ "${PN}" != *-bin ]] ; then - qa_var="QA_DT_HASH_${ARCH/-/_}" - eval "[[ -n \${!qa_var} ]] && QA_DT_HASH=(\"\${${qa_var}[@]}\")" - f=$(scanelf -qyRF '%k %p' -k .hash "${D}" | sed -e "s:\.hash ::") + if [[ "${LDFLAGS}" == *,--hash-style=gnu* ]] && \ + ! has binchecks ${RESTRICT} ; then + f=$(scanelf -qyRF '%k %p' -k .hash "${ED}" | sed -e "s:\.hash ::") if [[ -n ${f} ]] ; then echo "${f}" > "${T}"/scanelf-ignored-LDFLAGS.log - if [ "${QA_STRICT_DT_HASH-unset}" == unset ] ; then - if [[ ${#QA_DT_HASH[@]} -gt 1 ]] ; then - for x in "${QA_DT_HASH[@]}" ; do - sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-LDFLAGS.log - done - else - local shopts=$- - set -o noglob - for x in ${QA_DT_HASH} ; do - sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-LDFLAGS.log - done - set +o noglob - set -${shopts} - fi + if [ "${QA_STRICT_FLAGS_IGNORED-unset}" = unset ] ; then + for x in "${QA_FLAGS_IGNORED[@]}" ; do + sed -e "s#^${x#/}\$##" -i "${T}"/scanelf-ignored-LDFLAGS.log + done fi # Filter anything under /usr/lib/debug/ in order to avoid # duplicate warnings for splitdebug files. @@ -339,39 +433,6 @@ install_qa_check() { fi fi - # Save NEEDED information after removing self-contained providers - rm -f "$PORTAGE_BUILDDIR"/build-info/NEEDED{,.ELF.2} - scanelf -qyRF '%a;%p;%S;%r;%n' "${D}" | { while IFS= read -r l; do - arch=${l%%;*}; l=${l#*;} - obj="/${l%%;*}"; l=${l#*;} - soname=${l%%;*}; l=${l#*;} - rpath=${l%%;*}; l=${l#*;}; [ "${rpath}" = " - " ] && rpath="" - needed=${l%%;*}; l=${l#*;} - if [ -z "${rpath}" -o -n "${rpath//*ORIGIN*}" ]; then - # object doesn't contain $ORIGIN in its runpath attribute - echo "${obj} ${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED - echo "${arch:3};${obj};${soname};${rpath};${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2 - else - dir=${obj%/*} - # replace $ORIGIN with the dirname of the current object for the lookup - opath=$(echo :${rpath}: | sed -e "s#.*:\(.*\)\$ORIGIN\(.*\):.*#\1${dir}\2#") - sneeded=$(echo ${needed} | tr , ' ') - rneeded="" - for lib in ${sneeded}; do - found=0 - for path in ${opath//:/ }; do - [ -e "${D}/${path}/${lib}" ] && found=1 && break - done - [ "${found}" -eq 0 ] && rneeded="${rneeded},${lib}" - done - rneeded=${rneeded:1} - if [ -n "${rneeded}" ]; then - echo "${obj} ${rneeded}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED - echo "${arch:3};${obj};${soname};${rpath};${rneeded}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2 - fi - fi - done } - if [[ ${insecure_rpath} -eq 1 ]] ; then die "Aborting due to serious QA concerns with RUNPATH/RPATH" elif [[ -n ${die_msg} ]] && has stricter ${FEATURES} ; then @@ -381,7 +442,7 @@ install_qa_check() { # Check for shared libraries lacking SONAMEs qa_var="QA_SONAME_${ARCH/-/_}" eval "[[ -n \${!qa_var} ]] && QA_SONAME=(\"\${${qa_var}[@]}\")" - f=$(scanelf -ByF '%S %p' "${D}"{,usr/}lib*/lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${D}:/:") + f=$(scanelf -ByF '%S %p' "${ED}"{,usr/}lib*/lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") if [[ -n ${f} ]] ; then echo "${f}" > "${T}"/scanelf-missing-SONAME.log if [[ "${QA_STRICT_SONAME-unset}" == unset ]] ; then @@ -415,7 +476,7 @@ install_qa_check() { # Check for shared libraries lacking NEEDED entries qa_var="QA_DT_NEEDED_${ARCH/-/_}" eval "[[ -n \${!qa_var} ]] && QA_DT_NEEDED=(\"\${${qa_var}[@]}\")" - f=$(scanelf -ByF '%n %p' "${D}"{,usr/}lib*/lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${D}:/:") + f=$(scanelf -ByF '%n %p' "${ED}"{,usr/}lib*/lib*.so* | gawk '$2 == "" { print }' | sed -e "s:^[[:space:]]${ED}:/:") if [[ -n ${f} ]] ; then echo "${f}" > "${T}"/scanelf-missing-NEEDED.log if [[ "${QA_STRICT_DT_NEEDED-unset}" == unset ]] ; then @@ -449,7 +510,35 @@ install_qa_check() { PORTAGE_QUIET=${tmp_quiet} fi - local unsafe_files=$(find "${D}" -type f '(' -perm -2002 -o -perm -4002 ')') + # Create NEEDED.ELF.2 regardless of RESTRICT=binchecks, since this info is + # too useful not to have (it's required for things like preserve-libs), and + # it's tempting for ebuild authors to set RESTRICT=binchecks for packages + # containing pre-built binaries. + if type -P scanelf > /dev/null ; then + # Save NEEDED information after removing self-contained providers + rm -f "$PORTAGE_BUILDDIR"/build-info/NEEDED{,.ELF.2} + scanelf -qyRF '%a;%p;%S;%r;%n' "${D}" | { while IFS= read -r l; do + arch=${l%%;*}; l=${l#*;} + obj="/${l%%;*}"; l=${l#*;} + soname=${l%%;*}; l=${l#*;} + rpath=${l%%;*}; l=${l#*;}; [ "${rpath}" = " - " ] && rpath="" + needed=${l%%;*}; l=${l#*;} + echo "${obj} ${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED + echo "${arch:3};${obj};${soname};${rpath};${needed}" >> "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2 + done } + + [ -n "${QA_SONAME_NO_SYMLINK}" ] && \ + echo "${QA_SONAME_NO_SYMLINK}" > \ + "${PORTAGE_BUILDDIR}"/build-info/QA_SONAME_NO_SYMLINK + + if has binchecks ${RESTRICT} && \ + [ -s "${PORTAGE_BUILDDIR}/build-info/NEEDED.ELF.2" ] ; then + eqawarn "QA Notice: RESTRICT=binchecks prevented checks on these ELF files:" + eqawarn "$(while read -r x; do x=${x#*;} ; x=${x%%;*} ; echo "${x#${EPREFIX}}" ; done < "${PORTAGE_BUILDDIR}"/build-info/NEEDED.ELF.2)" + fi + fi + + local unsafe_files=$(find "${ED}" -type f '(' -perm -2002 -o -perm -4002 ')' | sed -e "s:^${ED}:/:") if [[ -n ${unsafe_files} ]] ; then eqawarn "QA Notice: Unsafe files detected (set*id and world writable)" eqawarn "${unsafe_files}" @@ -469,8 +558,8 @@ install_qa_check() { # Sanity check syntax errors in init.d scripts local d for d in /etc/conf.d /etc/init.d ; do - [[ -d ${D}/${d} ]] || continue - for i in "${D}"/${d}/* ; do + [[ -d ${ED}/${d} ]] || continue + for i in "${ED}"/${d}/* ; do [[ -L ${i} ]] && continue # if empty conf.d/init.d dir exists (baselayout), then i will be "/etc/conf.d/*" and not exist [[ ! -e ${i} ]] && continue @@ -478,20 +567,27 @@ install_qa_check() { done done + # Look for leaking LDFLAGS into pkg-config files + f=$(egrep -sH '^Libs.*-Wl,(-O[012]|--hash-style)' "${ED}"/usr/*/pkgconfig/*.pc) + if [[ -n ${f} ]] ; then + eqawarn "QA Notice: pkg-config files with wrong LDFLAGS detected:" + eqawarn "${f//${D}}" + fi + # this should help to ensure that all (most?) shared libraries are executable # and that all libtool scripts / static libraries are not executable local j - for i in "${D}"opt/*/lib{,32,64} \ - "${D}"lib{,32,64} \ - "${D}"usr/lib{,32,64} \ - "${D}"usr/X11R6/lib{,32,64} ; do + for i in "${ED}"opt/*/lib{,32,64} \ + "${ED}"lib{,32,64} \ + "${ED}"usr/lib{,32,64} \ + "${ED}"usr/X11R6/lib{,32,64} ; do [[ ! -d ${i} ]] && continue for j in "${i}"/*.so.* "${i}"/*.so ; do [[ ! -e ${j} ]] && continue [[ -L ${j} ]] && continue [[ -x ${j} ]] && continue - vecho "making executable: ${j#${D}}" + vecho "making executable: ${j#${ED}}" chmod +x "${j}" done @@ -499,7 +595,7 @@ install_qa_check() { [[ ! -e ${j} ]] && continue [[ -L ${j} ]] && continue [[ ! -x ${j} ]] && continue - vecho "removing executable bit: ${j#${D}}" + vecho "removing executable bit: ${j#${ED}}" chmod -x "${j}" done @@ -523,7 +619,7 @@ install_qa_check() { # http://bugs.gentoo.org/4411 abort="no" local a s - for a in "${D}"usr/lib*/*.a ; do + for a in "${ED}"usr/lib*/*.a ; do s=${a%.a}.so if [[ ! -e ${s} ]] ; then s=${s%usr/*}${s##*/usr/} @@ -537,7 +633,7 @@ install_qa_check() { [[ ${abort} == "yes" ]] && die "add those ldscripts" # Make sure people don't store libtool files or static libs in /lib - f=$(ls "${D}"lib*/*.{a,la} 2>/dev/null) + f=$(ls "${ED}"lib*/*.{a,la} 2>/dev/null) if [[ -n ${f} ]] ; then vecho -ne '\n' eqawarn "QA Notice: Excessive files found in the / partition" @@ -548,9 +644,9 @@ install_qa_check() { # Verify that the libtool files don't contain bogus $D entries. local abort=no gentoo_bug=no always_overflow=no - for a in "${D}"usr/lib*/*.la ; do + for a in "${ED}"usr/lib*/*.la ; do s=${a##*/} - if grep -qs "${D}" "${a}" ; then + if grep -qs "${ED}" "${a}" ; then vecho -ne '\n' eqawarn "QA Notice: ${s} appears to contain PORTAGE_TMPDIR paths" abort="yes" @@ -621,8 +717,8 @@ install_qa_check() { #esac if [[ $always_overflow = yes ]] ; then eerror - eerror "QA Notice: Package has poor programming practices which may compile" - eerror " fine but exhibit random runtime failures." + eerror "QA Notice: Package triggers severe warnings which indicate that it" + eerror " may exhibit random runtime failures." eerror eerror "${f}" eerror @@ -631,8 +727,8 @@ install_qa_check() { eerror else vecho -ne '\n' - eqawarn "QA Notice: Package has poor programming practices which may compile" - eqawarn " fine but exhibit random runtime failures." + eqawarn "QA Notice: Package triggers severe warnings which indicate that it" + eqawarn " may exhibit random runtime failures." eqawarn "${f}" vecho -ne '\n' fi @@ -658,8 +754,8 @@ install_qa_check() { if [[ $gentoo_bug = yes ]] ; then eerror - eerror "QA Notice: Package has poor programming practices which may compile" - eerror " but will almost certainly crash on 64bit architectures." + eerror "QA Notice: Package triggers severe warnings which indicate that it" + eerror " will almost certainly crash on 64bit architectures." eerror eerror "${f}" eerror @@ -668,8 +764,8 @@ install_qa_check() { eerror else vecho -ne '\n' - eqawarn "QA Notice: Package has poor programming practices which may compile" - eqawarn " but will almost certainly crash on 64bit architectures." + eqawarn "QA Notice: Package triggers severe warnings which indicate that it" + eqawarn " will almost certainly crash on 64bit architectures." eqawarn "${f}" vecho -ne '\n' fi @@ -678,7 +774,7 @@ install_qa_check() { if [[ ${abort} == "yes" ]] ; then if [[ $gentoo_bug = yes || $always_overflow = yes ]] ; then die "install aborted due to" \ - "poor programming practices shown above" + "severe warnings shown above" else echo "Please do not file a Gentoo bug and instead" \ "report the above QA issues directly to the upstream" \ @@ -686,13 +782,13 @@ install_qa_check() { while read -r line ; do eqawarn "${line}" ; done eqawarn "Homepage: ${HOMEPAGE}" has stricter ${FEATURES} && die "install aborted due to" \ - "poor programming practices shown above" + "severe warnings shown above" fi fi fi # Portage regenerates this on the installed system. - rm -f "${D}"/usr/share/info/dir{,.gz,.bz2} + rm -f "${ED}"/usr/share/info/dir{,.gz,.bz2} || die "rm failed!" if has multilib-strict ${FEATURES} && \ [[ -x /usr/bin/file && -x /usr/bin/find ]] && \ @@ -701,15 +797,15 @@ install_qa_check() { local abort=no dir file firstrun=yes MULTILIB_STRICT_EXEMPT=$(echo ${MULTILIB_STRICT_EXEMPT} | sed -e 's:\([(|)]\):\\\1:g') for dir in ${MULTILIB_STRICT_DIRS} ; do - [[ -d ${D}/${dir} ]] || continue - for file in $(find ${D}/${dir} -type f | grep -v "^${D}/${dir}/${MULTILIB_STRICT_EXEMPT}"); do + [[ -d ${ED}/${dir} ]] || continue + for file in $(find ${ED}/${dir} -type f | grep -v "^${ED}/${dir}/${MULTILIB_STRICT_EXEMPT}"); do if file ${file} | egrep -q "${MULTILIB_STRICT_DENY}" ; then if [[ ${firstrun} == yes ]] ; then echo "Files matching a file type that is not allowed:" firstrun=no fi abort=yes - echo " ${file#${D}//}" + echo " ${file#${ED}//}" fi done done @@ -718,7 +814,7 @@ install_qa_check() { # ensure packages don't install systemd units automagically if ! has systemd ${INHERITED} && \ - [[ -d "${D}"/lib/systemd/system ]] + [[ -d "${ED}"/lib/systemd/system ]] then eqawarn "QA Notice: package installs systemd unit files (/lib/systemd/system)" eqawarn " but does not inherit systemd.eclass." @@ -727,6 +823,124 @@ install_qa_check() { fi } +install_qa_check_prefix() { + if [[ -d ${ED}/${D} ]] ; then + find "${ED}/${D}" | \ + while read i ; do + eqawarn "QA Notice: /${i##${ED}/${D}} installed in \${ED}/\${D}" + done + die "Aborting due to QA concerns: files installed in ${ED}/${D}" + fi + + if [[ -d ${ED}/${EPREFIX} ]] ; then + find "${ED}/${EPREFIX}/" | \ + while read i ; do + eqawarn "QA Notice: ${i#${D}} double prefix" + done + die "Aborting due to QA concerns: double prefix files installed" + fi + + if [[ -d ${D} ]] ; then + INSTALLTOD=$(find ${D%/} | egrep -v "^${ED}" | sed -e "s|^${D%/}||" | awk '{if (length($0) <= length("'"${EPREFIX}"'")) { if (substr("'"${EPREFIX}"'", 1, length($0)) != $0) {print $0;} } else if (substr($0, 1, length("'"${EPREFIX}"'")) != "'"${EPREFIX}"'") {print $0;} }') + if [[ -n ${INSTALLTOD} ]] ; then + eqawarn "QA Notice: the following files are outside of the prefix:" + eqawarn "${INSTALLTOD}" + die "Aborting due to QA concerns: there are files installed outside the prefix" + fi + fi + + # all further checks rely on ${ED} existing + [[ -d ${ED} ]] || return + + # this does not really belong here, but it's closely tied to + # the code below; many runscripts generate positives here, and we + # know they don't work (bug #196294) so as long as that one + # remains an issue, simply remove them as they won't work + # anyway, avoid etc/init.d/functions.sh from being thrown away + if [[ ( -d "${ED}"/etc/conf.d || -d "${ED}"/etc/init.d ) && ! -f "${ED}"/etc/init.d/functions.sh ]] ; then + ewarn "removed /etc/init.d and /etc/conf.d directories until bug #196294 has been resolved" + rm -Rf "${ED}"/etc/{conf,init}.d + fi + + # check shebangs, bug #282539 + rm -f "${T}"/non-prefix-shebangs-errs + local WHITELIST=" /usr/bin/env " + # this is hell expensive, but how else? + find "${ED}" -executable \! -type d -print0 \ + | xargs -0 grep -H -n -m1 "^#!" \ + | while read f ; + do + local fn=${f%%:*} + local pos=${f#*:} ; pos=${pos%:*} + local line=${f##*:} + # shebang always appears on the first line ;) + [[ ${pos} != 1 ]] && continue + local oldIFS=${IFS} + IFS=$'\r'$'\n'$'\t'" " + line=( ${line#"#!"} ) + IFS=${oldIFS} + [[ ${WHITELIST} == *" ${line[0]} "* ]] && continue + local fp=${fn#${D}} ; fp=/${fp%/*} + # line[0] can be an absolutised path, bug #342929 + local eprefix=$(canonicalize ${EPREFIX}) + local rf=${fn} + # in case we deal with a symlink, make sure we don't replace it + # with a real file (sed -i does that) + if [[ -L ${fn} ]] ; then + rf=$(readlink ${fn}) + [[ ${rf} != /* ]] && rf=${fn%/*}/${rf} + # ignore symlinks pointing to outside prefix + # as seen in sys-devel/native-cctools + [[ $(canonicalize "/${rf#${D}}") != ${eprefix}/* ]] && continue + fi + # does the shebang start with ${EPREFIX}, and does it exist? + if [[ ${line[0]} == ${EPREFIX}/* || ${line[0]} == ${eprefix}/* ]] ; then + if [[ ! -e ${ROOT%/}${line[0]} && ! -e ${D%/}${line[0]} ]] ; then + # hmm, refers explicitly to $EPREFIX, but doesn't exist, + # if it's in PATH that's wrong in any case + if [[ ":${PATH}:" == *":${fp}:"* ]] ; then + echo "${fn#${D}}:${line[0]} (explicit EPREFIX but target not found)" \ + >> "${T}"/non-prefix-shebangs-errs + else + eqawarn "${fn#${D}} has explicit EPREFIX in shebang but target not found (${line[0]})" + fi + fi + continue + fi + # unprefixed shebang, is the script directly in $PATH? + if [[ ":${PATH}:" == *":${fp}:"* ]] ; then + if [[ -e ${EROOT}${line[0]} || -e ${ED}${line[0]} ]] ; then + # is it unprefixed, but we can just fix it because a + # prefixed variant exists + eqawarn "prefixing shebang of ${fn#${D}}" + # statement is made idempotent on purpose, because + # symlinks may point to the same target, and hence the + # same real file may be sedded multiple times since we + # read the shebangs in one go upfront for performance + # reasons + sed -i -e '1s:^#! \?'"${line[0]}"':#!'"${EPREFIX}"${line[0]}':' "${rf}" + continue + else + # this is definitely wrong: script in $PATH and invalid shebang + echo "${fn#${D}}:${line[0]} (script ${fn##*/} installed in PATH but interpreter ${line[0]} not found)" \ + >> "${T}"/non-prefix-shebangs-errs + fi + else + # unprefixed/invalid shebang, but outside $PATH, this may be + # intended (e.g. config.guess) so remain silent by default + has stricter ${FEATURES} && \ + eqawarn "invalid shebang in ${fn#${D}}: ${line[0]}" + fi + done + if [[ -e "${T}"/non-prefix-shebangs-errs ]] ; then + eqawarn "QA Notice: the following files use invalid (possible non-prefixed) shebangs:" + while read line ; do + eqawarn " ${line}" + done < "${T}"/non-prefix-shebangs-errs + rm -f "${T}"/non-prefix-shebangs-errs + die "Aborting due to QA concerns: invalid shebangs found" + fi +} install_mask() { local root="$1" @@ -758,6 +972,9 @@ preinst_mask() { return 1 fi + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + # Make sure $PWD is not ${D} so that we don't leave gmon.out files # in there in case any tools were built with -pg in CFLAGS. cd "${T}" @@ -770,11 +987,11 @@ preinst_mask() { fi done - install_mask "${D}" "${INSTALL_MASK}" + install_mask "${ED}" "${INSTALL_MASK}" # remove share dir if unnessesary if has nodoc $FEATURES || has noman $FEATURES || has noinfo $FEATURES; then - rmdir "${D}usr/share" &> /dev/null + rmdir "${ED}usr/share" &> /dev/null fi } @@ -783,29 +1000,33 @@ preinst_sfperms() { eerror "${FUNCNAME}: D is unset" return 1 fi + + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + # Smart FileSystem Permissions if has sfperms $FEATURES; then local i - find "${D}" -type f -perm -4000 -print0 | \ + find "${ED}" -type f -perm -4000 -print0 | \ while read -r -d $'\0' i ; do if [ -n "$(find "$i" -perm -2000)" ] ; then - ebegin ">>> SetUID and SetGID: [chmod o-r] /${i#${D}}" + ebegin ">>> SetUID and SetGID: [chmod o-r] /${i#${ED}}" chmod o-r "$i" eend $? else - ebegin ">>> SetUID: [chmod go-r] /${i#${D}}" + ebegin ">>> SetUID: [chmod go-r] /${i#${ED}}" chmod go-r "$i" eend $? fi done - find "${D}" -type f -perm -2000 -print0 | \ + find "${ED}" -type f -perm -2000 -print0 | \ while read -r -d $'\0' i ; do if [ -n "$(find "$i" -perm -4000)" ] ; then # This case is already handled # by the SetUID check above. true else - ebegin ">>> SetGID: [chmod o-r] /${i#${D}}" + ebegin ">>> SetGID: [chmod o-r] /${i#${ED}}" chmod o-r "$i" eend $? fi @@ -818,6 +1039,10 @@ preinst_suid_scan() { eerror "${FUNCNAME}: D is unset" return 1 fi + + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + # total suid control. if has suidctl $FEATURES; then local i sfconf x @@ -826,10 +1051,10 @@ preinst_suid_scan() { # to files outside of the sandbox, but this # can easly be bypassed using the addwrite() function addwrite "${sfconf}" - vecho ">>> Performing suid scan in ${D}" - for i in $(find "${D}" -type f \( -perm -4000 -o -perm -2000 \) ); do + vecho ">>> Performing suid scan in ${ED}" + for i in $(find "${ED}" -type f \( -perm -4000 -o -perm -2000 \) ); do if [ -s "${sfconf}" ]; then - install_path=/${i#${D}} + install_path=/${i#${ED}} if grep -q "^${install_path}\$" "${sfconf}" ; then vecho "- ${install_path} is an approved suid file" else @@ -839,7 +1064,7 @@ preinst_suid_scan() { chmod ugo-s "${i}" grep "^#${install_path}$" "${sfconf}" > /dev/null || { vecho ">>> Appending commented out entry to ${sfconf} for ${PF}" - echo "## ${ls_ret%${D}*}${install_path}" >> "${sfconf}" + echo "## ${ls_ret%${ED}*}${install_path}" >> "${sfconf}" echo "#${install_path}" >> "${sfconf}" # no delwrite() eh? # delwrite ${sconf} @@ -861,13 +1086,15 @@ preinst_selinux_labels() { # SELinux file labeling (needs to always be last in dyn_preinst) # only attempt to label if setfiles is executable # and 'context' is available on selinuxfs. - if [ -f /selinux/context -a -x /usr/sbin/setfiles -a -x /usr/sbin/selinuxconfig ]; then + if [ -f /selinux/context -o -f /sys/fs/selinux/context ] && \ + [ -x /usr/sbin/setfiles -a -x /usr/sbin/selinuxconfig ]; then vecho ">>> Setting SELinux security labels" ( eval "$(/usr/sbin/selinuxconfig)" || \ die "Failed to determine SELinux policy paths."; - addwrite /selinux/context; + addwrite /selinux/context + addwrite /sys/fs/selinux/context /usr/sbin/setfiles "${file_contexts_path}" -r "${D}" "${D}" ) || die "Failed to set SELinux security labels." @@ -880,10 +1107,30 @@ preinst_selinux_labels() { } dyn_package() { + local PROOT + + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local EPREFIX= ED=${D} ;; esac + # Make sure $PWD is not ${D} so that we don't leave gmon.out files # in there in case any tools were built with -pg in CFLAGS. + cd "${T}" - install_mask "${PORTAGE_BUILDDIR}/image" "${PKG_INSTALL_MASK}" + + if [[ -n ${PKG_INSTALL_MASK} ]] ; then + PROOT=${T}/packaging/ + # make a temporary copy of ${D} so that any modifications we do that + # are binpkg specific, do not influence the actual installed image. + rm -rf "${PROOT}" || die "failed removing stale package tree" + cp -pPR $(cp --help | grep -qs -e-l && echo -l) \ + "${D}" "${PROOT}" \ + || die "failed creating packaging tree" + + install_mask "${PROOT%/}${EPREFIX}/" "${PKG_INSTALL_MASK}" + else + PROOT=${D} + fi + local tar_options="" [[ $PORTAGE_VERBOSE = 1 ]] && tar_options+=" -v" # Sandbox is disabled in case the user wants to use a symlink @@ -892,7 +1139,7 @@ dyn_package() { [ -z "${PORTAGE_BINPKG_TMPFILE}" ] && \ die "PORTAGE_BINPKG_TMPFILE is unset" mkdir -p "${PORTAGE_BINPKG_TMPFILE%/*}" || die "mkdir failed" - tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${D}" . | \ + tar $tar_options -cf - $PORTAGE_BINPKG_TAR_OPTS -C "${PROOT}" . | \ $PORTAGE_BZIP2_COMMAND -c > "$PORTAGE_BINPKG_TMPFILE" assert "failed to pack binary package: '$PORTAGE_BINPKG_TMPFILE'" PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ @@ -913,6 +1160,9 @@ dyn_package() { [ -n "${md5_hash}" ] && \ echo ${md5_hash} > "${PORTAGE_BUILDDIR}"/build-info/BINPKGMD5 vecho ">>> Done." + + # cleanup our temp tree + [[ -n ${PKG_INSTALL_MASK} ]] && rm -rf "${PROOT}" cd "${PORTAGE_BUILDDIR}" >> "$PORTAGE_BUILDDIR/.packaged" || \ die "Failed to create $PORTAGE_BUILDDIR/.packaged" @@ -957,10 +1207,14 @@ __END1__ } dyn_rpm() { + + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local EPREFIX= ;; esac + cd "${T}" || die "cd failed" local machine_name=$(uname -m) - local dest_dir=/usr/src/rpm/RPMS/${machine_name} - addwrite /usr/src/rpm + local dest_dir=${EPREFIX}/usr/src/rpm/RPMS/${machine_name} + addwrite ${EPREFIX}/usr/src/rpm addwrite "${RPMDIR}" dyn_spec rpmbuild -bb --clean --rmsource "${PF}.spec" || die "Failed to integrate rpm spec file" @@ -985,6 +1239,21 @@ success_hooks() { done } +install_hooks() { + local hooks_dir="${PORTAGE_CONFIGROOT}etc/portage/hooks/install" + local fp + local ret=0 + shopt -s nullglob + for fp in "${hooks_dir}"/*; do + if [ -x "$fp" ]; then + "$fp" + ret=$(( $ret | $? )) + fi + done + shopt -u nullglob + return $ret +} + if [ -n "${MISC_FUNCTIONS_ARGS}" ]; then source_all_bashrcs [ "$PORTAGE_DEBUG" == "1" ] && set -x diff --git a/portage_with_autodep/bin/phase-functions.sh b/portage_with_autodep/bin/phase-functions.sh new file mode 100755 index 0000000..ce251ce --- /dev/null +++ b/portage_with_autodep/bin/phase-functions.sh @@ -0,0 +1,1001 @@ +#!/bin/bash +# Copyright 1999-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# Hardcoded bash lists are needed for backward compatibility with +# <portage-2.1.4 since they assume that a newly installed version +# of ebuild.sh will work for pkg_postinst, pkg_prerm, and pkg_postrm +# when portage is upgrading itself. + +PORTAGE_READONLY_METADATA="DEFINED_PHASES DEPEND DESCRIPTION + EAPI HOMEPAGE INHERITED IUSE REQUIRED_USE KEYWORDS LICENSE + PDEPEND PROVIDE RDEPEND REPOSITORY RESTRICT SLOT SRC_URI" + +PORTAGE_READONLY_VARS="D EBUILD EBUILD_PHASE \ + EBUILD_SH_ARGS ECLASSDIR EMERGE_FROM FILESDIR MERGE_TYPE \ + PM_EBUILD_HOOK_DIR \ + PORTAGE_ACTUAL_DISTDIR PORTAGE_ARCHLIST PORTAGE_BASHRC \ + PORTAGE_BINPKG_FILE PORTAGE_BINPKG_TAR_OPTS PORTAGE_BINPKG_TMPFILE \ + PORTAGE_BIN_PATH PORTAGE_BUILDDIR PORTAGE_BUNZIP2_COMMAND \ + PORTAGE_BZIP2_COMMAND PORTAGE_COLORMAP PORTAGE_CONFIGROOT \ + PORTAGE_DEBUG PORTAGE_DEPCACHEDIR PORTAGE_EBUILD_EXIT_FILE \ + PORTAGE_GID PORTAGE_GRPNAME PORTAGE_INST_GID PORTAGE_INST_UID \ + PORTAGE_IPC_DAEMON PORTAGE_IUSE PORTAGE_LOG_FILE \ + PORTAGE_MUTABLE_FILTERED_VARS PORTAGE_OVERRIDE_EPREFIX \ + PORTAGE_PYM_PATH PORTAGE_PYTHON \ + PORTAGE_READONLY_METADATA PORTAGE_READONLY_VARS \ + PORTAGE_REPO_NAME PORTAGE_RESTRICT \ + PORTAGE_SAVED_READONLY_VARS PORTAGE_SIGPIPE_STATUS \ + PORTAGE_TMPDIR PORTAGE_UPDATE_ENV PORTAGE_USERNAME \ + PORTAGE_VERBOSE PORTAGE_WORKDIR_MODE PORTDIR PORTDIR_OVERLAY \ + PROFILE_PATHS REPLACING_VERSIONS REPLACED_BY_VERSION T WORKDIR \ + __PORTAGE_TEST_HARDLINK_LOCKS" + +PORTAGE_SAVED_READONLY_VARS="A CATEGORY P PF PN PR PV PVR" + +# Variables that portage sets but doesn't mark readonly. +# In order to prevent changed values from causing unexpected +# interference, they are filtered out of the environment when +# it is saved or loaded (any mutations do not persist). +PORTAGE_MUTABLE_FILTERED_VARS="AA HOSTNAME" + +# @FUNCTION: filter_readonly_variables +# @DESCRIPTION: [--filter-sandbox] [--allow-extra-vars] +# Read an environment from stdin and echo to stdout while filtering variables +# with names that are known to cause interference: +# +# * some specific variables for which bash does not allow assignment +# * some specific variables that affect portage or sandbox behavior +# * variable names that begin with a digit or that contain any +# non-alphanumeric characters that are not be supported by bash +# +# --filter-sandbox causes all SANDBOX_* variables to be filtered, which +# is only desired in certain cases, such as during preprocessing or when +# saving environment.bz2 for a binary or installed package. +# +# --filter-features causes the special FEATURES variable to be filtered. +# Generally, we want it to persist between phases since the user might +# want to modify it via bashrc to enable things like splitdebug and +# installsources for specific packages. They should be able to modify it +# in pre_pkg_setup() and have it persist all the way through the install +# phase. However, if FEATURES exist inside environment.bz2 then they +# should be overridden by current settings. +# +# --filter-locale causes locale related variables such as LANG and LC_* +# variables to be filtered. These variables should persist between phases, +# in case they are modified by the ebuild. However, the current user +# settings should be used when loading the environment from a binary or +# installed package. +# +# --filter-path causes the PATH variable to be filtered. This variable +# should persist between phases, in case it is modified by the ebuild. +# However, old settings should be overridden when loading the +# environment from a binary or installed package. +# +# ---allow-extra-vars causes some extra vars to be allowd through, such +# as ${PORTAGE_SAVED_READONLY_VARS} and ${PORTAGE_MUTABLE_FILTERED_VARS}. +# This is enabled automatically if EMERGE_FROM=binary, since it preserves +# variables from when the package was originally built. +# +# In bash-3.2_p20+ an attempt to assign BASH_*, FUNCNAME, GROUPS or any +# readonly variable cause the shell to exit while executing the "source" +# builtin command. To avoid this problem, this function filters those +# variables out and discards them. See bug #190128. +filter_readonly_variables() { + local x filtered_vars + local readonly_bash_vars="BASHOPTS BASHPID DIRSTACK EUID + FUNCNAME GROUPS PIPESTATUS PPID SHELLOPTS UID" + local bash_misc_vars="BASH BASH_.* COLUMNS COMP_WORDBREAKS HISTCMD + HISTFILE HOSTNAME HOSTTYPE IFS LINENO MACHTYPE OLDPWD + OPTERR OPTIND OSTYPE POSIXLY_CORRECT PS4 PWD RANDOM + SECONDS SHELL SHLVL _" + local filtered_sandbox_vars="SANDBOX_ACTIVE SANDBOX_BASHRC + SANDBOX_DEBUG_LOG SANDBOX_DISABLED SANDBOX_LIB + SANDBOX_LOG SANDBOX_ON" + # Untrusted due to possible application of package renames to binpkgs + local binpkg_untrusted_vars="CATEGORY P PF PN PR PV PVR" + local misc_garbage_vars="_portage_filter_opts" + filtered_vars="$readonly_bash_vars $bash_misc_vars + $PORTAGE_READONLY_VARS $misc_garbage_vars" + + # Don't filter/interfere with prefix variables unless they are + # supported by the current EAPI. + case "${EAPI:-0}" in + 0|1|2) + [[ " ${FEATURES} " == *" force-prefix "* ]] && \ + filtered_vars+=" ED EPREFIX EROOT" + ;; + *) + filtered_vars+=" ED EPREFIX EROOT" + ;; + esac + + if has --filter-sandbox $* ; then + filtered_vars="${filtered_vars} SANDBOX_.*" + else + filtered_vars="${filtered_vars} ${filtered_sandbox_vars}" + fi + if has --filter-features $* ; then + filtered_vars="${filtered_vars} FEATURES PORTAGE_FEATURES" + fi + if has --filter-path $* ; then + filtered_vars+=" PATH" + fi + if has --filter-locale $* ; then + filtered_vars+=" LANG LC_ALL LC_COLLATE + LC_CTYPE LC_MESSAGES LC_MONETARY + LC_NUMERIC LC_PAPER LC_TIME" + fi + if ! has --allow-extra-vars $* ; then + if [ "${EMERGE_FROM}" = binary ] ; then + # preserve additional variables from build time, + # while excluding untrusted variables + filtered_vars+=" ${binpkg_untrusted_vars}" + else + filtered_vars+=" ${PORTAGE_SAVED_READONLY_VARS}" + filtered_vars+=" ${PORTAGE_MUTABLE_FILTERED_VARS}" + fi + fi + + "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}"/filter-bash-environment.py "${filtered_vars}" || die "filter-bash-environment.py failed" +} + +# @FUNCTION: preprocess_ebuild_env +# @DESCRIPTION: +# Filter any readonly variables from ${T}/environment, source it, and then +# save it via save_ebuild_env(). This process should be sufficient to prevent +# any stale variables or functions from an arbitrary environment from +# interfering with the current environment. This is useful when an existing +# environment needs to be loaded from a binary or installed package. +preprocess_ebuild_env() { + local _portage_filter_opts="--filter-features --filter-locale --filter-path --filter-sandbox" + + # If environment.raw is present, this is a signal from the python side, + # indicating that the environment may contain stale FEATURES and + # SANDBOX_{DENY,PREDICT,READ,WRITE} variables that should be filtered out. + # Otherwise, we don't need to filter the environment. + [ -f "${T}/environment.raw" ] || return 0 + + filter_readonly_variables $_portage_filter_opts < "${T}"/environment \ + >> "$T/environment.filtered" || return $? + unset _portage_filter_opts + mv "${T}"/environment.filtered "${T}"/environment || return $? + rm -f "${T}/environment.success" || return $? + # WARNING: Code inside this subshell should avoid making assumptions + # about variables or functions after source "${T}"/environment has been + # called. Any variables that need to be relied upon should already be + # filtered out above. + ( + export SANDBOX_ON=1 + source "${T}/environment" || exit $? + # We have to temporarily disable sandbox since the + # SANDBOX_{DENY,READ,PREDICT,WRITE} values we've just loaded + # may be unusable (triggering in spurious sandbox violations) + # until we've merged them with our current values. + export SANDBOX_ON=0 + + # It's remotely possible that save_ebuild_env() has been overridden + # by the above source command. To protect ourselves, we override it + # here with our own version. ${PORTAGE_BIN_PATH} is safe to use here + # because it's already filtered above. + source "${PORTAGE_BIN_PATH}/save-ebuild-env.sh" || exit $? + + # Rely on save_ebuild_env() to filter out any remaining variables + # and functions that could interfere with the current environment. + save_ebuild_env || exit $? + >> "$T/environment.success" || exit $? + ) > "${T}/environment.filtered" + local retval + if [ -e "${T}/environment.success" ] ; then + filter_readonly_variables --filter-features < \ + "${T}/environment.filtered" > "${T}/environment" + retval=$? + else + retval=1 + fi + rm -f "${T}"/environment.{filtered,raw,success} + return ${retval} +} + +ebuild_phase() { + declare -F "$1" >/dev/null && qa_call $1 +} + +ebuild_phase_with_hooks() { + local x phase_name=${1} + for x in {pre_,,post_}${phase_name} ; do + ebuild_phase ${x} + done +} + +dyn_pretend() { + if [[ -e $PORTAGE_BUILDDIR/.pretended ]] ; then + vecho ">>> It appears that '$PF' is already pretended; skipping." + vecho ">>> Remove '$PORTAGE_BUILDDIR/.pretended' to force pretend." + return 0 + fi + ebuild_phase pre_pkg_pretend + ebuild_phase pkg_pretend + >> "$PORTAGE_BUILDDIR/.pretended" || \ + die "Failed to create $PORTAGE_BUILDDIR/.pretended" + ebuild_phase post_pkg_pretend +} + +dyn_setup() { + if [[ -e $PORTAGE_BUILDDIR/.setuped ]] ; then + vecho ">>> It appears that '$PF' is already setup; skipping." + vecho ">>> Remove '$PORTAGE_BUILDDIR/.setuped' to force setup." + return 0 + fi + ebuild_phase pre_pkg_setup + ebuild_phase pkg_setup + >> "$PORTAGE_BUILDDIR/.setuped" || \ + die "Failed to create $PORTAGE_BUILDDIR/.setuped" + ebuild_phase post_pkg_setup +} + +dyn_unpack() { + if [[ -f ${PORTAGE_BUILDDIR}/.unpacked ]] ; then + vecho ">>> WORKDIR is up-to-date, keeping..." + return 0 + fi + if [ ! -d "${WORKDIR}" ]; then + install -m${PORTAGE_WORKDIR_MODE:-0700} -d "${WORKDIR}" || die "Failed to create dir '${WORKDIR}'" + fi + cd "${WORKDIR}" || die "Directory change failed: \`cd '${WORKDIR}'\`" + ebuild_phase pre_src_unpack + vecho ">>> Unpacking source..." + ebuild_phase src_unpack + >> "$PORTAGE_BUILDDIR/.unpacked" || \ + die "Failed to create $PORTAGE_BUILDDIR/.unpacked" + vecho ">>> Source unpacked in ${WORKDIR}" + ebuild_phase post_src_unpack +} + +dyn_clean() { + if [ -z "${PORTAGE_BUILDDIR}" ]; then + echo "Aborting clean phase because PORTAGE_BUILDDIR is unset!" + return 1 + elif [ ! -d "${PORTAGE_BUILDDIR}" ] ; then + return 0 + fi + if has chflags $FEATURES ; then + chflags -R noschg,nouchg,nosappnd,nouappnd "${PORTAGE_BUILDDIR}" + chflags -R nosunlnk,nouunlnk "${PORTAGE_BUILDDIR}" 2>/dev/null + fi + + rm -rf "${PORTAGE_BUILDDIR}/image" "${PORTAGE_BUILDDIR}/homedir" + rm -f "${PORTAGE_BUILDDIR}/.installed" + + if [[ $EMERGE_FROM = binary ]] || \ + ! has keeptemp $FEATURES && ! has keepwork $FEATURES ; then + rm -rf "${T}" + fi + + if [[ $EMERGE_FROM = binary ]] || ! has keepwork $FEATURES; then + rm -f "$PORTAGE_BUILDDIR"/.{ebuild_changed,logid,pretended,setuped,unpacked,prepared} \ + "$PORTAGE_BUILDDIR"/.{configured,compiled,tested,packaged} \ + "$PORTAGE_BUILDDIR"/.die_hooks \ + "$PORTAGE_BUILDDIR"/.ipc_{in,out,lock} \ + "$PORTAGE_BUILDDIR"/.exit_status + + rm -rf "${PORTAGE_BUILDDIR}/build-info" + rm -rf "${WORKDIR}" + fi + + if [ -f "${PORTAGE_BUILDDIR}/.unpacked" ]; then + find "${PORTAGE_BUILDDIR}" -type d ! -regex "^${WORKDIR}" | sort -r | tr "\n" "\0" | $XARGS -0 rmdir &>/dev/null + fi + + # do not bind this to doebuild defined DISTDIR; don't trust doebuild, and if mistakes are made it'll + # result in it wiping the users distfiles directory (bad). + rm -rf "${PORTAGE_BUILDDIR}/distdir" + + # Some kernels, such as Solaris, return EINVAL when an attempt + # is made to remove the current working directory. + cd "$PORTAGE_BUILDDIR"/../.. + rmdir "$PORTAGE_BUILDDIR" 2>/dev/null + + true +} + +abort_handler() { + local msg + if [ "$2" != "fail" ]; then + msg="${EBUILD}: ${1} aborted; exiting." + else + msg="${EBUILD}: ${1} failed; exiting." + fi + echo + echo "$msg" + echo + eval ${3} + #unset signal handler + trap - SIGINT SIGQUIT +} + +abort_prepare() { + abort_handler src_prepare $1 + rm -f "$PORTAGE_BUILDDIR/.prepared" + exit 1 +} + +abort_configure() { + abort_handler src_configure $1 + rm -f "$PORTAGE_BUILDDIR/.configured" + exit 1 +} + +abort_compile() { + abort_handler "src_compile" $1 + rm -f "${PORTAGE_BUILDDIR}/.compiled" + exit 1 +} + +abort_test() { + abort_handler "dyn_test" $1 + rm -f "${PORTAGE_BUILDDIR}/.tested" + exit 1 +} + +abort_install() { + abort_handler "src_install" $1 + rm -rf "${PORTAGE_BUILDDIR}/image" + exit 1 +} + +has_phase_defined_up_to() { + local phase + for phase in unpack prepare configure compile install; do + has ${phase} ${DEFINED_PHASES} && return 0 + [[ ${phase} == $1 ]] && return 1 + done + # We shouldn't actually get here + return 1 +} + +dyn_prepare() { + + if [[ -e $PORTAGE_BUILDDIR/.prepared ]] ; then + vecho ">>> It appears that '$PF' is already prepared; skipping." + vecho ">>> Remove '$PORTAGE_BUILDDIR/.prepared' to force prepare." + return 0 + fi + + if [[ -d $S ]] ; then + cd "${S}" + elif has $EAPI 0 1 2 3 3_pre2 ; then + cd "${WORKDIR}" + elif [[ -z ${A} ]] && ! has_phase_defined_up_to prepare; then + cd "${WORKDIR}" + else + die "The source directory '${S}' doesn't exist" + fi + + trap abort_prepare SIGINT SIGQUIT + + ebuild_phase pre_src_prepare + vecho ">>> Preparing source in $PWD ..." + ebuild_phase src_prepare + >> "$PORTAGE_BUILDDIR/.prepared" || \ + die "Failed to create $PORTAGE_BUILDDIR/.prepared" + vecho ">>> Source prepared." + ebuild_phase post_src_prepare + + trap - SIGINT SIGQUIT +} + +dyn_configure() { + + if [[ -e $PORTAGE_BUILDDIR/.configured ]] ; then + vecho ">>> It appears that '$PF' is already configured; skipping." + vecho ">>> Remove '$PORTAGE_BUILDDIR/.configured' to force configuration." + return 0 + fi + + if [[ -d $S ]] ; then + cd "${S}" + elif has $EAPI 0 1 2 3 3_pre2 ; then + cd "${WORKDIR}" + elif [[ -z ${A} ]] && ! has_phase_defined_up_to configure; then + cd "${WORKDIR}" + else + die "The source directory '${S}' doesn't exist" + fi + + trap abort_configure SIGINT SIGQUIT + + ebuild_phase pre_src_configure + + vecho ">>> Configuring source in $PWD ..." + ebuild_phase src_configure + >> "$PORTAGE_BUILDDIR/.configured" || \ + die "Failed to create $PORTAGE_BUILDDIR/.configured" + vecho ">>> Source configured." + + ebuild_phase post_src_configure + + trap - SIGINT SIGQUIT +} + +dyn_compile() { + + if [[ -e $PORTAGE_BUILDDIR/.compiled ]] ; then + vecho ">>> It appears that '${PF}' is already compiled; skipping." + vecho ">>> Remove '$PORTAGE_BUILDDIR/.compiled' to force compilation." + return 0 + fi + + if [[ -d $S ]] ; then + cd "${S}" + elif has $EAPI 0 1 2 3 3_pre2 ; then + cd "${WORKDIR}" + elif [[ -z ${A} ]] && ! has_phase_defined_up_to compile; then + cd "${WORKDIR}" + else + die "The source directory '${S}' doesn't exist" + fi + + trap abort_compile SIGINT SIGQUIT + + if has distcc $FEATURES && has distcc-pump $FEATURES ; then + if [[ -z $INCLUDE_SERVER_PORT ]] || [[ ! -w $INCLUDE_SERVER_PORT ]] ; then + eval $(pump --startup) + trap "pump --shutdown" EXIT + fi + fi + + ebuild_phase pre_src_compile + + vecho ">>> Compiling source in $PWD ..." + ebuild_phase src_compile + >> "$PORTAGE_BUILDDIR/.compiled" || \ + die "Failed to create $PORTAGE_BUILDDIR/.compiled" + vecho ">>> Source compiled." + + ebuild_phase post_src_compile + + trap - SIGINT SIGQUIT +} + +dyn_test() { + + if [[ -e $PORTAGE_BUILDDIR/.tested ]] ; then + vecho ">>> It appears that ${PN} has already been tested; skipping." + vecho ">>> Remove '${PORTAGE_BUILDDIR}/.tested' to force test." + return + fi + + if [ "${EBUILD_FORCE_TEST}" == "1" ] ; then + # If USE came from ${T}/environment then it might not have USE=test + # like it's supposed to here. + ! has test ${USE} && export USE="${USE} test" + fi + + trap "abort_test" SIGINT SIGQUIT + if [ -d "${S}" ]; then + cd "${S}" + else + cd "${WORKDIR}" + fi + + if ! has test $FEATURES && [ "${EBUILD_FORCE_TEST}" != "1" ]; then + vecho ">>> Test phase [not enabled]: ${CATEGORY}/${PF}" + elif has test $RESTRICT; then + einfo "Skipping make test/check due to ebuild restriction." + vecho ">>> Test phase [explicitly disabled]: ${CATEGORY}/${PF}" + else + local save_sp=${SANDBOX_PREDICT} + addpredict / + ebuild_phase pre_src_test + ebuild_phase src_test + >> "$PORTAGE_BUILDDIR/.tested" || \ + die "Failed to create $PORTAGE_BUILDDIR/.tested" + ebuild_phase post_src_test + SANDBOX_PREDICT=${save_sp} + fi + + trap - SIGINT SIGQUIT +} + +dyn_install() { + [ -z "$PORTAGE_BUILDDIR" ] && die "${FUNCNAME}: PORTAGE_BUILDDIR is unset" + if has noauto $FEATURES ; then + rm -f "${PORTAGE_BUILDDIR}/.installed" + elif [[ -e $PORTAGE_BUILDDIR/.installed ]] ; then + vecho ">>> It appears that '${PF}' is already installed; skipping." + vecho ">>> Remove '${PORTAGE_BUILDDIR}/.installed' to force install." + return 0 + fi + trap "abort_install" SIGINT SIGQUIT + ebuild_phase pre_src_install + + _x=${ED} + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) _x=${D} ;; esac + rm -rf "${D}" + mkdir -p "${_x}" + unset _x + + if [[ -d $S ]] ; then + cd "${S}" + elif has $EAPI 0 1 2 3 3_pre2 ; then + cd "${WORKDIR}" + elif [[ -z ${A} ]] && ! has_phase_defined_up_to install; then + cd "${WORKDIR}" + else + die "The source directory '${S}' doesn't exist" + fi + + vecho + vecho ">>> Install ${PF} into ${D} category ${CATEGORY}" + #our custom version of libtool uses $S and $D to fix + #invalid paths in .la files + export S D + + # Reset exeinto(), docinto(), insinto(), and into() state variables + # in case the user is running the install phase multiple times + # consecutively via the ebuild command. + export DESTTREE=/usr + export INSDESTTREE="" + export _E_EXEDESTTREE_="" + export _E_DOCDESTTREE_="" + + ebuild_phase src_install + >> "$PORTAGE_BUILDDIR/.installed" || \ + die "Failed to create $PORTAGE_BUILDDIR/.installed" + vecho ">>> Completed installing ${PF} into ${D}" + vecho + ebuild_phase post_src_install + + cd "${PORTAGE_BUILDDIR}"/build-info + set -f + local f x + IFS=$' \t\n\r' + for f in CATEGORY DEFINED_PHASES FEATURES INHERITED IUSE \ + PF PKGUSE SLOT KEYWORDS HOMEPAGE DESCRIPTION ; do + x=$(echo -n ${!f}) + [[ -n $x ]] && echo "$x" > $f + done + if [[ $CATEGORY != virtual ]] ; then + for f in ASFLAGS CBUILD CC CFLAGS CHOST CTARGET CXX \ + CXXFLAGS EXTRA_ECONF EXTRA_EINSTALL EXTRA_MAKE \ + LDFLAGS LIBCFLAGS LIBCXXFLAGS ; do + x=$(echo -n ${!f}) + [[ -n $x ]] && echo "$x" > $f + done + fi + echo "${USE}" > USE + echo "${EAPI:-0}" > EAPI + + # Save EPREFIX, since it makes it easy to use chpathtool to + # adjust the content of a binary package so that it will + # work in a different EPREFIX from the one is was built for. + case "${EAPI:-0}" in + 0|1|2) + [[ " ${FEATURES} " == *" force-prefix "* ]] && \ + [ -n "${EPREFIX}" ] && echo "${EPREFIX}" > EPREFIX + ;; + *) + [ -n "${EPREFIX}" ] && echo "${EPREFIX}" > EPREFIX + ;; + esac + + set +f + + # local variables can leak into the saved environment. + unset f + + save_ebuild_env --exclude-init-phases | filter_readonly_variables \ + --filter-path --filter-sandbox --allow-extra-vars > environment + assert "save_ebuild_env failed" + + ${PORTAGE_BZIP2_COMMAND} -f9 environment + + cp "${EBUILD}" "${PF}.ebuild" + [ -n "${PORTAGE_REPO_NAME}" ] && echo "${PORTAGE_REPO_NAME}" > repository + if has nostrip ${FEATURES} ${RESTRICT} || has strip ${RESTRICT} + then + >> DEBUGBUILD + fi + trap - SIGINT SIGQUIT +} + +dyn_preinst() { + if [ -z "${D}" ]; then + eerror "${FUNCNAME}: D is unset" + return 1 + fi + ebuild_phase_with_hooks pkg_preinst +} + +dyn_help() { + echo + echo "Portage" + echo "Copyright 1999-2010 Gentoo Foundation" + echo + echo "How to use the ebuild command:" + echo + echo "The first argument to ebuild should be an existing .ebuild file." + echo + echo "One or more of the following options can then be specified. If more" + echo "than one option is specified, each will be executed in order." + echo + echo " help : show this help screen" + echo " pretend : execute package specific pretend actions" + echo " setup : execute package specific setup actions" + echo " fetch : download source archive(s) and patches" + echo " digest : create a manifest file for the package" + echo " manifest : create a manifest file for the package" + echo " unpack : unpack sources (auto-dependencies if needed)" + echo " prepare : prepare sources (auto-dependencies if needed)" + echo " configure : configure sources (auto-fetch/unpack if needed)" + echo " compile : compile sources (auto-fetch/unpack/configure if needed)" + echo " test : test package (auto-fetch/unpack/configure/compile if needed)" + echo " preinst : execute pre-install instructions" + echo " postinst : execute post-install instructions" + echo " install : install the package to the temporary install directory" + echo " qmerge : merge image into live filesystem, recording files in db" + echo " merge : do fetch, unpack, compile, install and qmerge" + echo " prerm : execute pre-removal instructions" + echo " postrm : execute post-removal instructions" + echo " unmerge : remove package from live filesystem" + echo " config : execute package specific configuration actions" + echo " package : create a tarball package in ${PKGDIR}/All" + echo " rpm : build a RedHat RPM package" + echo " clean : clean up all source and temporary files" + echo + echo "The following settings will be used for the ebuild process:" + echo + echo " package : ${PF}" + echo " slot : ${SLOT}" + echo " category : ${CATEGORY}" + echo " description : ${DESCRIPTION}" + echo " system : ${CHOST}" + echo " c flags : ${CFLAGS}" + echo " c++ flags : ${CXXFLAGS}" + echo " make flags : ${MAKEOPTS}" + echo -n " build mode : " + if has nostrip ${FEATURES} ${RESTRICT} || has strip ${RESTRICT} ; + then + echo "debug (large)" + else + echo "production (stripped)" + fi + echo " merge to : ${ROOT}" + echo + if [ -n "$USE" ]; then + echo "Additionally, support for the following optional features will be enabled:" + echo + echo " ${USE}" + fi + echo +} + +# @FUNCTION: _ebuild_arg_to_phase +# @DESCRIPTION: +# Translate a known ebuild(1) argument into the precise +# name of it's corresponding ebuild phase. +_ebuild_arg_to_phase() { + [ $# -ne 2 ] && die "expected exactly 2 args, got $#: $*" + local eapi=$1 + local arg=$2 + local phase_func="" + + case "$arg" in + pretend) + ! has $eapi 0 1 2 3 3_pre2 && \ + phase_func=pkg_pretend + ;; + setup) + phase_func=pkg_setup + ;; + nofetch) + phase_func=pkg_nofetch + ;; + unpack) + phase_func=src_unpack + ;; + prepare) + ! has $eapi 0 1 && \ + phase_func=src_prepare + ;; + configure) + ! has $eapi 0 1 && \ + phase_func=src_configure + ;; + compile) + phase_func=src_compile + ;; + test) + phase_func=src_test + ;; + install) + phase_func=src_install + ;; + preinst) + phase_func=pkg_preinst + ;; + postinst) + phase_func=pkg_postinst + ;; + prerm) + phase_func=pkg_prerm + ;; + postrm) + phase_func=pkg_postrm + ;; + esac + + [[ -z $phase_func ]] && return 1 + echo "$phase_func" + return 0 +} + +_ebuild_phase_funcs() { + [ $# -ne 2 ] && die "expected exactly 2 args, got $#: $*" + local eapi=$1 + local phase_func=$2 + local default_phases="pkg_nofetch src_unpack src_prepare src_configure + src_compile src_install src_test" + local x y default_func="" + + for x in pkg_nofetch src_unpack src_test ; do + declare -F $x >/dev/null || \ + eval "$x() { _eapi0_$x \"\$@\" ; }" + done + + case $eapi in + + 0|1) + + if ! declare -F src_compile >/dev/null ; then + case $eapi in + 0) + src_compile() { _eapi0_src_compile "$@" ; } + ;; + *) + src_compile() { _eapi1_src_compile "$@" ; } + ;; + esac + fi + + for x in $default_phases ; do + eval "default_$x() { + die \"default_$x() is not supported with EAPI='$eapi' during phase $phase_func\" + }" + done + + eval "default() { + die \"default() is not supported with EAPI='$eapi' during phase $phase_func\" + }" + + ;; + + *) + + declare -F src_configure >/dev/null || \ + src_configure() { _eapi2_src_configure "$@" ; } + + declare -F src_compile >/dev/null || \ + src_compile() { _eapi2_src_compile "$@" ; } + + has $eapi 2 3 3_pre2 || declare -F src_install >/dev/null || \ + src_install() { _eapi4_src_install "$@" ; } + + if has $phase_func $default_phases ; then + + _eapi2_pkg_nofetch () { _eapi0_pkg_nofetch "$@" ; } + _eapi2_src_unpack () { _eapi0_src_unpack "$@" ; } + _eapi2_src_prepare () { true ; } + _eapi2_src_test () { _eapi0_src_test "$@" ; } + _eapi2_src_install () { die "$FUNCNAME is not supported" ; } + + for x in $default_phases ; do + eval "default_$x() { _eapi2_$x \"\$@\" ; }" + done + + eval "default() { _eapi2_$phase_func \"\$@\" ; }" + + case $eapi in + 2|3) + ;; + *) + eval "default_src_install() { _eapi4_src_install \"\$@\" ; }" + [[ $phase_func = src_install ]] && \ + eval "default() { _eapi4_$phase_func \"\$@\" ; }" + ;; + esac + + else + + for x in $default_phases ; do + eval "default_$x() { + die \"default_$x() is not supported in phase $default_func\" + }" + done + + eval "default() { + die \"default() is not supported with EAPI='$eapi' during phase $phase_func\" + }" + + fi + + ;; + esac +} + +ebuild_main() { + + # Subshell/helper die support (must export for the die helper). + # Since this function is typically executed in a subshell, + # setup EBUILD_MASTER_PID to refer to the current $BASHPID, + # which seems to give the best results when further + # nested subshells call die. + export EBUILD_MASTER_PID=$BASHPID + trap 'exit 1' SIGTERM + + #a reasonable default for $S + [[ -z ${S} ]] && export S=${WORKDIR}/${P} + + if [[ -s $SANDBOX_LOG ]] ; then + # We use SANDBOX_LOG to check for sandbox violations, + # so we ensure that there can't be a stale log to + # interfere with our logic. + local x= + if [[ -n SANDBOX_ON ]] ; then + x=$SANDBOX_ON + export SANDBOX_ON=0 + fi + + rm -f "$SANDBOX_LOG" || \ + die "failed to remove stale sandbox log: '$SANDBOX_LOG'" + + if [[ -n $x ]] ; then + export SANDBOX_ON=$x + fi + unset x + fi + + # Force configure scripts that automatically detect ccache to + # respect FEATURES="-ccache". + has ccache $FEATURES || export CCACHE_DISABLE=1 + + local phase_func=$(_ebuild_arg_to_phase "$EAPI" "$EBUILD_PHASE") + [[ -n $phase_func ]] && _ebuild_phase_funcs "$EAPI" "$phase_func" + unset phase_func + + source_all_bashrcs + + case ${1} in + nofetch) + ebuild_phase_with_hooks pkg_nofetch + ;; + prerm|postrm|postinst|config|info) + if has "${1}" config info && \ + ! declare -F "pkg_${1}" >/dev/null ; then + ewarn "pkg_${1}() is not defined: '${EBUILD##*/}'" + fi + export SANDBOX_ON="0" + if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then + ebuild_phase_with_hooks pkg_${1} + else + set -x + ebuild_phase_with_hooks pkg_${1} + set +x + fi + if [[ $EBUILD_PHASE == postinst ]] && [[ -n $PORTAGE_UPDATE_ENV ]]; then + # Update environment.bz2 in case installation phases + # need to pass some variables to uninstallation phases. + save_ebuild_env --exclude-init-phases | \ + filter_readonly_variables --filter-path \ + --filter-sandbox --allow-extra-vars \ + | ${PORTAGE_BZIP2_COMMAND} -c -f9 > "$PORTAGE_UPDATE_ENV" + assert "save_ebuild_env failed" + fi + ;; + unpack|prepare|configure|compile|test|clean|install) + if [[ ${SANDBOX_DISABLED:-0} = 0 ]] ; then + export SANDBOX_ON="1" + else + export SANDBOX_ON="0" + fi + + case "${1}" in + configure|compile) + + local x + for x in ASFLAGS CCACHE_DIR CCACHE_SIZE \ + CFLAGS CXXFLAGS LDFLAGS LIBCFLAGS LIBCXXFLAGS ; do + [[ ${!x+set} = set ]] && export $x + done + unset x + + has distcc $FEATURES && [[ -n $DISTCC_DIR ]] && \ + [[ ${SANDBOX_WRITE/$DISTCC_DIR} = $SANDBOX_WRITE ]] && \ + addwrite "$DISTCC_DIR" + + x=LIBDIR_$ABI + [ -z "$PKG_CONFIG_PATH" -a -n "$ABI" -a -n "${!x}" ] && \ + export PKG_CONFIG_PATH=/usr/${!x}/pkgconfig + + if has noauto $FEATURES && \ + [[ ! -f $PORTAGE_BUILDDIR/.unpacked ]] ; then + echo + echo "!!! We apparently haven't unpacked..." \ + "This is probably not what you" + echo "!!! want to be doing... You are using" \ + "FEATURES=noauto so I'll assume" + echo "!!! that you know what you are doing..." \ + "You have 5 seconds to abort..." + echo + + local x + for x in 1 2 3 4 5 6 7 8; do + LC_ALL=C sleep 0.25 + done + + sleep 3 + fi + + cd "$PORTAGE_BUILDDIR" + if [ ! -d build-info ] ; then + mkdir build-info + cp "$EBUILD" "build-info/$PF.ebuild" + fi + + #our custom version of libtool uses $S and $D to fix + #invalid paths in .la files + export S D + + ;; + esac + + if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then + dyn_${1} + else + set -x + dyn_${1} + set +x + fi + export SANDBOX_ON="0" + ;; + help|pretend|setup|preinst) + #pkg_setup needs to be out of the sandbox for tmp file creation; + #for example, awking and piping a file in /tmp requires a temp file to be created + #in /etc. If pkg_setup is in the sandbox, both our lilo and apache ebuilds break. + export SANDBOX_ON="0" + if [ "${PORTAGE_DEBUG}" != "1" ] || [ "${-/x/}" != "$-" ]; then + dyn_${1} + else + set -x + dyn_${1} + set +x + fi + ;; + _internal_test) + ;; + *) + export SANDBOX_ON="1" + echo "Unrecognized arg '${1}'" + echo + dyn_help + exit 1 + ;; + esac + + # Save the env only for relevant phases. + if ! has "${1}" clean help info nofetch ; then + umask 002 + save_ebuild_env | filter_readonly_variables \ + --filter-features > "$T/environment" + assert "save_ebuild_env failed" + chown portage:portage "$T/environment" &>/dev/null + chmod g+w "$T/environment" &>/dev/null + fi + [[ -n $PORTAGE_EBUILD_EXIT_FILE ]] && > "$PORTAGE_EBUILD_EXIT_FILE" + if [[ -n $PORTAGE_IPC_DAEMON ]] ; then + [[ ! -s $SANDBOX_LOG ]] + "$PORTAGE_BIN_PATH"/ebuild-ipc exit $? + fi +} diff --git a/portage_with_autodep/bin/phase-helpers.sh b/portage_with_autodep/bin/phase-helpers.sh new file mode 100755 index 0000000..946520b --- /dev/null +++ b/portage_with_autodep/bin/phase-helpers.sh @@ -0,0 +1,663 @@ +#!/bin/bash +# Copyright 1999-2011 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +export DESTTREE=/usr +export INSDESTTREE="" +export _E_EXEDESTTREE_="" +export _E_DOCDESTTREE_="" +export INSOPTIONS="-m0644" +export EXEOPTIONS="-m0755" +export LIBOPTIONS="-m0644" +export DIROPTIONS="-m0755" +export MOPREFIX=${PN} +declare -a PORTAGE_DOCOMPRESS=( /usr/share/{doc,info,man} ) +declare -a PORTAGE_DOCOMPRESS_SKIP=( /usr/share/doc/${PF}/html ) + +into() { + if [ "$1" == "/" ]; then + export DESTTREE="" + else + export DESTTREE=$1 + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if [ ! -d "${ED}${DESTTREE}" ]; then + install -d "${ED}${DESTTREE}" + local ret=$? + if [[ $ret -ne 0 ]] ; then + helpers_die "${FUNCNAME[0]} failed" + return $ret + fi + fi + fi +} + +insinto() { + if [ "$1" == "/" ]; then + export INSDESTTREE="" + else + export INSDESTTREE=$1 + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if [ ! -d "${ED}${INSDESTTREE}" ]; then + install -d "${ED}${INSDESTTREE}" + local ret=$? + if [[ $ret -ne 0 ]] ; then + helpers_die "${FUNCNAME[0]} failed" + return $ret + fi + fi + fi +} + +exeinto() { + if [ "$1" == "/" ]; then + export _E_EXEDESTTREE_="" + else + export _E_EXEDESTTREE_="$1" + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if [ ! -d "${ED}${_E_EXEDESTTREE_}" ]; then + install -d "${ED}${_E_EXEDESTTREE_}" + local ret=$? + if [[ $ret -ne 0 ]] ; then + helpers_die "${FUNCNAME[0]} failed" + return $ret + fi + fi + fi +} + +docinto() { + if [ "$1" == "/" ]; then + export _E_DOCDESTTREE_="" + else + export _E_DOCDESTTREE_="$1" + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if [ ! -d "${ED}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" ]; then + install -d "${ED}usr/share/doc/${PF}/${_E_DOCDESTTREE_}" + local ret=$? + if [[ $ret -ne 0 ]] ; then + helpers_die "${FUNCNAME[0]} failed" + return $ret + fi + fi + fi +} + +insopts() { + export INSOPTIONS="$@" + + # `install` should never be called with '-s' ... + has -s ${INSOPTIONS} && die "Never call insopts() with -s" +} + +diropts() { + export DIROPTIONS="$@" +} + +exeopts() { + export EXEOPTIONS="$@" + + # `install` should never be called with '-s' ... + has -s ${EXEOPTIONS} && die "Never call exeopts() with -s" +} + +libopts() { + export LIBOPTIONS="$@" + + # `install` should never be called with '-s' ... + has -s ${LIBOPTIONS} && die "Never call libopts() with -s" +} + +docompress() { + has "${EAPI}" 0 1 2 3 && die "'docompress' not supported in this EAPI" + + local f g + if [[ $1 = "-x" ]]; then + shift + for f; do + f=$(strip_duplicate_slashes "${f}"); f=${f%/} + [[ ${f:0:1} = / ]] || f="/${f}" + for g in "${PORTAGE_DOCOMPRESS_SKIP[@]}"; do + [[ ${f} = "${g}" ]] && continue 2 + done + PORTAGE_DOCOMPRESS_SKIP[${#PORTAGE_DOCOMPRESS_SKIP[@]}]=${f} + done + else + for f; do + f=$(strip_duplicate_slashes "${f}"); f=${f%/} + [[ ${f:0:1} = / ]] || f="/${f}" + for g in "${PORTAGE_DOCOMPRESS[@]}"; do + [[ ${f} = "${g}" ]] && continue 2 + done + PORTAGE_DOCOMPRESS[${#PORTAGE_DOCOMPRESS[@]}]=${f} + done + fi +} + +# adds ".keep" files so that dirs aren't auto-cleaned +keepdir() { + dodir "$@" + local x + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + if [ "$1" == "-R" ] || [ "$1" == "-r" ]; then + shift + find "$@" -type d -printf "${ED}%p/.keep_${CATEGORY}_${PN}-${SLOT}\n" \ + | tr "\n" "\0" | \ + while read -r -d $'\0' ; do + >> "$REPLY" || \ + die "Failed to recursively create .keep files" + done + else + for x in "$@"; do + >> "${ED}${x}/.keep_${CATEGORY}_${PN}-${SLOT}" || \ + die "Failed to create .keep in ${ED}${x}" + done + fi +} + + +useq() { + has $EBUILD_PHASE prerm postrm || eqawarn \ + "QA Notice: The 'useq' function is deprecated (replaced by 'use')" + use ${1} +} + +usev() { + if use ${1}; then + echo "${1#!}" + return 0 + fi + return 1 +} + +use() { + local u=$1 + local found=0 + + # if we got something like '!flag', then invert the return value + if [[ ${u:0:1} == "!" ]] ; then + u=${u:1} + found=1 + fi + + if [[ $EBUILD_PHASE = depend ]] ; then + # TODO: Add a registration interface for eclasses to register + # any number of phase hooks, so that global scope eclass + # initialization can by migrated to phase hooks in new EAPIs. + # Example: add_phase_hook before pkg_setup $ECLASS_pre_pkg_setup + #if [[ -n $EAPI ]] && ! has "$EAPI" 0 1 2 3 ; then + # die "use() called during invalid phase: $EBUILD_PHASE" + #fi + true + + # Make sure we have this USE flag in IUSE + elif [[ -n $PORTAGE_IUSE && -n $EBUILD_PHASE ]] ; then + [[ $u =~ $PORTAGE_IUSE ]] || \ + eqawarn "QA Notice: USE Flag '${u}' not" \ + "in IUSE for ${CATEGORY}/${PF}" + fi + + if has ${u} ${USE} ; then + return ${found} + else + return $((!found)) + fi +} + +use_with() { + if [ -z "$1" ]; then + echo "!!! use_with() called without a parameter." >&2 + echo "!!! use_with <USEFLAG> [<flagname> [value]]" >&2 + return 1 + fi + + if ! has "${EAPI:-0}" 0 1 2 3 ; then + local UW_SUFFIX=${3+=$3} + else + local UW_SUFFIX=${3:+=$3} + fi + local UWORD=${2:-$1} + + if use $1; then + echo "--with-${UWORD}${UW_SUFFIX}" + else + echo "--without-${UWORD}" + fi + return 0 +} + +use_enable() { + if [ -z "$1" ]; then + echo "!!! use_enable() called without a parameter." >&2 + echo "!!! use_enable <USEFLAG> [<flagname> [value]]" >&2 + return 1 + fi + + if ! has "${EAPI:-0}" 0 1 2 3 ; then + local UE_SUFFIX=${3+=$3} + else + local UE_SUFFIX=${3:+=$3} + fi + local UWORD=${2:-$1} + + if use $1; then + echo "--enable-${UWORD}${UE_SUFFIX}" + else + echo "--disable-${UWORD}" + fi + return 0 +} + +unpack() { + local srcdir + local x + local y + local myfail + local eapi=${EAPI:-0} + [ -z "$*" ] && die "Nothing passed to the 'unpack' command" + + for x in "$@"; do + vecho ">>> Unpacking ${x} to ${PWD}" + y=${x%.*} + y=${y##*.} + + if [[ ${x} == "./"* ]] ; then + srcdir="" + elif [[ ${x} == ${DISTDIR%/}/* ]] ; then + die "Arguments to unpack() cannot begin with \${DISTDIR}." + elif [[ ${x} == "/"* ]] ; then + die "Arguments to unpack() cannot be absolute" + else + srcdir="${DISTDIR}/" + fi + [[ ! -s ${srcdir}${x} ]] && die "${x} does not exist" + + _unpack_tar() { + if [ "${y}" == "tar" ]; then + $1 -c -- "$srcdir$x" | tar xof - + assert_sigpipe_ok "$myfail" + else + local cwd_dest=${x##*/} + cwd_dest=${cwd_dest%.*} + $1 -c -- "${srcdir}${x}" > "${cwd_dest}" || die "$myfail" + fi + } + + myfail="failure unpacking ${x}" + case "${x##*.}" in + tar) + tar xof "$srcdir$x" || die "$myfail" + ;; + tgz) + tar xozf "$srcdir$x" || die "$myfail" + ;; + tbz|tbz2) + ${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d} -c -- "$srcdir$x" | tar xof - + assert_sigpipe_ok "$myfail" + ;; + ZIP|zip|jar) + # unzip will interactively prompt under some error conditions, + # as reported in bug #336285 + ( set +x ; while true ; do echo n || break ; done ) | \ + unzip -qo "${srcdir}${x}" || die "$myfail" + ;; + gz|Z|z) + _unpack_tar "gzip -d" + ;; + bz2|bz) + _unpack_tar "${PORTAGE_BUNZIP2_COMMAND:-${PORTAGE_BZIP2_COMMAND} -d}" + ;; + 7Z|7z) + local my_output + my_output="$(7z x -y "${srcdir}${x}")" + if [ $? -ne 0 ]; then + echo "${my_output}" >&2 + die "$myfail" + fi + ;; + RAR|rar) + unrar x -idq -o+ "${srcdir}${x}" || die "$myfail" + ;; + LHa|LHA|lha|lzh) + lha xfq "${srcdir}${x}" || die "$myfail" + ;; + a) + ar x "${srcdir}${x}" || die "$myfail" + ;; + deb) + # Unpacking .deb archives can not always be done with + # `ar`. For instance on AIX this doesn't work out. If + # we have `deb2targz` installed, prefer it over `ar` for + # that reason. We just make sure on AIX `deb2targz` is + # installed. + if type -P deb2targz > /dev/null; then + y=${x##*/} + local created_symlink=0 + if [ ! "$srcdir$x" -ef "$y" ] ; then + # deb2targz always extracts into the same directory as + # the source file, so create a symlink in the current + # working directory if necessary. + ln -sf "$srcdir$x" "$y" || die "$myfail" + created_symlink=1 + fi + deb2targz "$y" || die "$myfail" + if [ $created_symlink = 1 ] ; then + # Clean up the symlink so the ebuild + # doesn't inadvertently install it. + rm -f "$y" + fi + mv -f "${y%.deb}".tar.gz data.tar.gz || die "$myfail" + else + ar x "$srcdir$x" || die "$myfail" + fi + ;; + lzma) + _unpack_tar "lzma -d" + ;; + xz) + if has $eapi 0 1 2 ; then + vecho "unpack ${x}: file format not recognized. Ignoring." + else + _unpack_tar "xz -d" + fi + ;; + *) + vecho "unpack ${x}: file format not recognized. Ignoring." + ;; + esac + done + # Do not chmod '.' since it's probably ${WORKDIR} and PORTAGE_WORKDIR_MODE + # should be preserved. + find . -mindepth 1 -maxdepth 1 ! -type l -print0 | \ + ${XARGS} -0 chmod -fR a+rX,u+w,g-w,o-w +} + +econf() { + local x + + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local EPREFIX= ;; esac + + _hasg() { + local x s=$1 + shift + for x ; do [[ ${x} == ${s} ]] && echo "${x}" && return 0 ; done + return 1 + } + + _hasgq() { _hasg "$@" >/dev/null ; } + + local phase_func=$(_ebuild_arg_to_phase "$EAPI" "$EBUILD_PHASE") + if [[ -n $phase_func ]] ; then + if has "$EAPI" 0 1 ; then + [[ $phase_func != src_compile ]] && \ + eqawarn "QA Notice: econf called in" \ + "$phase_func instead of src_compile" + else + [[ $phase_func != src_configure ]] && \ + eqawarn "QA Notice: econf called in" \ + "$phase_func instead of src_configure" + fi + fi + + : ${ECONF_SOURCE:=.} + if [ -x "${ECONF_SOURCE}/configure" ]; then + if [[ -n $CONFIG_SHELL && \ + "$(head -n1 "$ECONF_SOURCE/configure")" =~ ^'#!'[[:space:]]*/bin/sh([[:space:]]|$) ]] ; then + sed -e "1s:^#![[:space:]]*/bin/sh:#!$CONFIG_SHELL:" -i "$ECONF_SOURCE/configure" || \ + die "Substition of shebang in '$ECONF_SOURCE/configure' failed" + fi + if [ -e "${EPREFIX}"/usr/share/gnuconfig/ ]; then + find "${WORKDIR}" -type f '(' \ + -name config.guess -o -name config.sub ')' -print0 | \ + while read -r -d $'\0' x ; do + vecho " * econf: updating ${x/${WORKDIR}\/} with ${EPREFIX}/usr/share/gnuconfig/${x##*/}" + cp -f "${EPREFIX}"/usr/share/gnuconfig/"${x##*/}" "${x}" + done + fi + + # EAPI=4 adds --disable-dependency-tracking to econf + if ! has "$EAPI" 0 1 2 3 3_pre2 && \ + "${ECONF_SOURCE}/configure" --help 2>/dev/null | \ + grep -q disable-dependency-tracking ; then + set -- --disable-dependency-tracking "$@" + fi + + # if the profile defines a location to install libs to aside from default, pass it on. + # if the ebuild passes in --libdir, they're responsible for the conf_libdir fun. + local CONF_LIBDIR LIBDIR_VAR="LIBDIR_${ABI}" + if [[ -n ${ABI} && -n ${!LIBDIR_VAR} ]] ; then + CONF_LIBDIR=${!LIBDIR_VAR} + fi + if [[ -n ${CONF_LIBDIR} ]] && ! _hasgq --libdir=\* "$@" ; then + export CONF_PREFIX=$(_hasg --exec-prefix=\* "$@") + [[ -z ${CONF_PREFIX} ]] && CONF_PREFIX=$(_hasg --prefix=\* "$@") + : ${CONF_PREFIX:=${EPREFIX}/usr} + CONF_PREFIX=${CONF_PREFIX#*=} + [[ ${CONF_PREFIX} != /* ]] && CONF_PREFIX="/${CONF_PREFIX}" + [[ ${CONF_LIBDIR} != /* ]] && CONF_LIBDIR="/${CONF_LIBDIR}" + set -- --libdir="$(strip_duplicate_slashes ${CONF_PREFIX}${CONF_LIBDIR})" "$@" + fi + + set -- \ + --prefix="${EPREFIX}"/usr \ + ${CBUILD:+--build=${CBUILD}} \ + --host=${CHOST} \ + ${CTARGET:+--target=${CTARGET}} \ + --mandir="${EPREFIX}"/usr/share/man \ + --infodir="${EPREFIX}"/usr/share/info \ + --datadir="${EPREFIX}"/usr/share \ + --sysconfdir="${EPREFIX}"/etc \ + --localstatedir="${EPREFIX}"/var/lib \ + "$@" \ + ${EXTRA_ECONF} + vecho "${ECONF_SOURCE}/configure" "$@" + + if ! "${ECONF_SOURCE}/configure" "$@" ; then + + if [ -s config.log ]; then + echo + echo "!!! Please attach the following file when seeking support:" + echo "!!! ${PWD}/config.log" + fi + die "econf failed" + fi + elif [ -f "${ECONF_SOURCE}/configure" ]; then + die "configure is not executable" + else + die "no configure script found" + fi +} + +einstall() { + # CONF_PREFIX is only set if they didn't pass in libdir above. + local LOCAL_EXTRA_EINSTALL="${EXTRA_EINSTALL}" + [[ " ${FEATURES} " == *" force-prefix "* ]] || \ + case "$EAPI" in 0|1|2) local ED=${D} ;; esac + LIBDIR_VAR="LIBDIR_${ABI}" + if [ -n "${ABI}" -a -n "${!LIBDIR_VAR}" ]; then + CONF_LIBDIR="${!LIBDIR_VAR}" + fi + unset LIBDIR_VAR + if [ -n "${CONF_LIBDIR}" ] && [ "${CONF_PREFIX:+set}" = set ]; then + EI_DESTLIBDIR="${D}/${CONF_PREFIX}/${CONF_LIBDIR}" + EI_DESTLIBDIR="$(strip_duplicate_slashes ${EI_DESTLIBDIR})" + LOCAL_EXTRA_EINSTALL="libdir=${EI_DESTLIBDIR} ${LOCAL_EXTRA_EINSTALL}" + unset EI_DESTLIBDIR + fi + + if [ -f ./[mM]akefile -o -f ./GNUmakefile ] ; then + if [ "${PORTAGE_DEBUG}" == "1" ]; then + ${MAKE:-make} -n prefix="${ED}usr" \ + datadir="${ED}usr/share" \ + infodir="${ED}usr/share/info" \ + localstatedir="${ED}var/lib" \ + mandir="${ED}usr/share/man" \ + sysconfdir="${ED}etc" \ + ${LOCAL_EXTRA_EINSTALL} \ + ${MAKEOPTS} ${EXTRA_EMAKE} -j1 \ + "$@" install + fi + ${MAKE:-make} prefix="${ED}usr" \ + datadir="${ED}usr/share" \ + infodir="${ED}usr/share/info" \ + localstatedir="${ED}var/lib" \ + mandir="${ED}usr/share/man" \ + sysconfdir="${ED}etc" \ + ${LOCAL_EXTRA_EINSTALL} \ + ${MAKEOPTS} ${EXTRA_EMAKE} -j1 \ + "$@" install || die "einstall failed" + else + die "no Makefile found" + fi +} + +_eapi0_pkg_nofetch() { + [ -z "${SRC_URI}" ] && return + + elog "The following are listed in SRC_URI for ${PN}:" + local x + for x in $(echo ${SRC_URI}); do + elog " ${x}" + done +} + +_eapi0_src_unpack() { + [[ -n ${A} ]] && unpack ${A} +} + +_eapi0_src_compile() { + if [ -x ./configure ] ; then + econf + fi + _eapi2_src_compile +} + +_eapi0_src_test() { + # Since we don't want emake's automatic die + # support (EAPI 4 and later), and we also don't + # want the warning messages that it produces if + # we call it in 'nonfatal' mode, we use emake_cmd + # to emulate the desired parts of emake behavior. + local emake_cmd="${MAKE:-make} ${MAKEOPTS} ${EXTRA_EMAKE}" + if $emake_cmd -j1 check -n &> /dev/null; then + vecho ">>> Test phase [check]: ${CATEGORY}/${PF}" + $emake_cmd -j1 check || \ + die "Make check failed. See above for details." + elif $emake_cmd -j1 test -n &> /dev/null; then + vecho ">>> Test phase [test]: ${CATEGORY}/${PF}" + $emake_cmd -j1 test || \ + die "Make test failed. See above for details." + else + vecho ">>> Test phase [none]: ${CATEGORY}/${PF}" + fi +} + +_eapi1_src_compile() { + _eapi2_src_configure + _eapi2_src_compile +} + +_eapi2_src_configure() { + if [[ -x ${ECONF_SOURCE:-.}/configure ]] ; then + econf + fi +} + +_eapi2_src_compile() { + if [ -f Makefile ] || [ -f GNUmakefile ] || [ -f makefile ]; then + emake || die "emake failed" + fi +} + +_eapi4_src_install() { + if [[ -f Makefile || -f GNUmakefile || -f makefile ]] ; then + emake DESTDIR="${D}" install + fi + + if ! declare -p DOCS &>/dev/null ; then + local d + for d in README* ChangeLog AUTHORS NEWS TODO CHANGES \ + THANKS BUGS FAQ CREDITS CHANGELOG ; do + [[ -s "${d}" ]] && dodoc "${d}" + done + elif [[ $(declare -p DOCS) == "declare -a "* ]] ; then + dodoc "${DOCS[@]}" + else + dodoc ${DOCS} + fi +} + +# @FUNCTION: has_version +# @USAGE: <DEPEND ATOM> +# @DESCRIPTION: +# Return true if given package is installed. Otherwise return false. +# Callers may override the ROOT variable in order to match packages from an +# alternative ROOT. +has_version() { + + local eroot + case "$EAPI" in + 0|1|2) + [[ " ${FEATURES} " == *" force-prefix "* ]] && \ + eroot=${ROOT%/}${EPREFIX}/ || eroot=${ROOT} + ;; + *) + eroot=${ROOT%/}${EPREFIX}/ + ;; + esac + if [[ -n $PORTAGE_IPC_DAEMON ]] ; then + "$PORTAGE_BIN_PATH"/ebuild-ipc has_version "${eroot}" "$1" + else + PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ + "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" has_version "${eroot}" "$1" + fi + local retval=$? + case "${retval}" in + 0|1) + return ${retval} + ;; + *) + die "unexpected portageq exit code: ${retval}" + ;; + esac +} + +# @FUNCTION: best_version +# @USAGE: <DEPEND ATOM> +# @DESCRIPTION: +# Returns the best/most-current match. +# Callers may override the ROOT variable in order to match packages from an +# alternative ROOT. +best_version() { + + local eroot + case "$EAPI" in + 0|1|2) + [[ " ${FEATURES} " == *" force-prefix "* ]] && \ + eroot=${ROOT%/}${EPREFIX}/ || eroot=${ROOT} + ;; + *) + eroot=${ROOT%/}${EPREFIX}/ + ;; + esac + if [[ -n $PORTAGE_IPC_DAEMON ]] ; then + "$PORTAGE_BIN_PATH"/ebuild-ipc best_version "${eroot}" "$1" + else + PYTHONPATH=${PORTAGE_PYM_PATH}${PYTHONPATH:+:}${PYTHONPATH} \ + "${PORTAGE_PYTHON:-/usr/bin/python}" "${PORTAGE_BIN_PATH}/portageq" best_version "${eroot}" "$1" + fi + local retval=$? + case "${retval}" in + 0|1) + return ${retval} + ;; + *) + die "unexpected portageq exit code: ${retval}" + ;; + esac +} diff --git a/portage_with_autodep/bin/portageq b/portage_with_autodep/bin/portageq index 57a7c39..280fe94 100755 --- a/portage_with_autodep/bin/portageq +++ b/portage_with_autodep/bin/portageq @@ -1,5 +1,5 @@ #!/usr/bin/python -O -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -42,12 +42,15 @@ except ImportError: del pym_path from portage import os +from portage.eapi import eapi_has_repo_deps from portage.util import writemsg, writemsg_stdout +from portage.output import colormap portage.proxy.lazyimport.lazyimport(globals(), 'subprocess', '_emerge.Package:Package', '_emerge.RootConfig:RootConfig', 'portage.dbapi._expand_new_virt:expand_new_virt', + 'portage._sets.base:InternalPackageSet', ) def eval_atom_use(atom): @@ -78,17 +81,18 @@ def eval_atom_use(atom): # def has_version(argv): - """<root> <category/package> + """<eroot> <category/package> Return code 0 if it's available, 1 otherwise. """ if (len(argv) < 2): print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 warnings = [] + allow_repo = atom_validate_strict is False or eapi_has_repo_deps(eapi) try: - atom = portage.dep.Atom(argv[1]) + atom = portage.dep.Atom(argv[1], allow_repo=allow_repo) except portage.exception.InvalidAtom: if atom_validate_strict: portage.writemsg("ERROR: Invalid atom: '%s'\n" % argv[1], @@ -99,7 +103,7 @@ def has_version(argv): else: if atom_validate_strict: try: - atom = portage.dep.Atom(argv[1], eapi=eapi) + atom = portage.dep.Atom(argv[1], allow_repo=allow_repo, eapi=eapi) except portage.exception.InvalidAtom as e: warnings.append( portage._unicode_decode("QA Notice: %s: %s") % \ @@ -112,11 +116,11 @@ def has_version(argv): try: mylist = portage.db[argv[0]]["vartree"].dbapi.match(atom) if mylist: - sys.exit(0) + return 0 else: - sys.exit(1) + return 1 except KeyError: - sys.exit(1) + return 1 except portage.exception.InvalidAtom: portage.writemsg("ERROR: Invalid atom: '%s'\n" % argv[1], noiselevel=-1) @@ -125,17 +129,18 @@ has_version.uses_root = True def best_version(argv): - """<root> <category/package> + """<eroot> <category/package> Returns category/package-version (without .ebuild). """ if (len(argv) < 2): print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 warnings = [] + allow_repo = atom_validate_strict is False or eapi_has_repo_deps(eapi) try: - atom = portage.dep.Atom(argv[1]) + atom = portage.dep.Atom(argv[1], allow_repo=allow_repo) except portage.exception.InvalidAtom: if atom_validate_strict: portage.writemsg("ERROR: Invalid atom: '%s'\n" % argv[1], @@ -146,7 +151,7 @@ def best_version(argv): else: if atom_validate_strict: try: - atom = portage.dep.Atom(argv[1], eapi=eapi) + atom = portage.dep.Atom(argv[1], allow_repo=allow_repo, eapi=eapi) except portage.exception.InvalidAtom as e: warnings.append( portage._unicode_decode("QA Notice: %s: %s") % \ @@ -160,31 +165,31 @@ def best_version(argv): mylist = portage.db[argv[0]]["vartree"].dbapi.match(atom) print(portage.best(mylist)) except KeyError: - sys.exit(1) + return 1 best_version.uses_root = True def mass_best_version(argv): - """<root> [<category/package>]+ + """<eroot> [<category/package>]+ Returns category/package-version (without .ebuild). """ if (len(argv) < 2): print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 try: for pack in argv[1:]: mylist=portage.db[argv[0]]["vartree"].dbapi.match(pack) print(pack+":"+portage.best(mylist)) except KeyError: - sys.exit(1) + return 1 mass_best_version.uses_root = True def metadata(argv): if (len(argv) < 4): print("ERROR: insufficient parameters!", file=sys.stderr) - sys.exit(2) + return 2 - root, pkgtype, pkgspec = argv[0:3] + eroot, pkgtype, pkgspec = argv[0:3] metakeys = argv[3:] type_map = { "ebuild":"porttree", @@ -192,20 +197,20 @@ def metadata(argv): "installed":"vartree"} if pkgtype not in type_map: print("Unrecognized package type: '%s'" % pkgtype, file=sys.stderr) - sys.exit(1) + return 1 trees = portage.db - if os.path.realpath(root) == os.path.realpath(portage.settings["ROOT"]): - root = portage.settings["ROOT"] # contains the normalized $ROOT + repo = portage.dep.dep_getrepo(pkgspec) + pkgspec = portage.dep.remove_slot(pkgspec) try: - values = trees[root][type_map[pkgtype]].dbapi.aux_get( - pkgspec, metakeys) + values = trees[eroot][type_map[pkgtype]].dbapi.aux_get( + pkgspec, metakeys, myrepo=repo) writemsg_stdout(''.join('%s\n' % x for x in values), noiselevel=-1) except KeyError: print("Package not found: '%s'" % pkgspec, file=sys.stderr) - sys.exit(1) + return 1 metadata.__doc__ = """ -<root> <pkgtype> <category/package> [<key>]+ +<eroot> <pkgtype> <category/package> [<key>]+ Returns metadata values for the specified package. Available keys: %s """ % ','.join(sorted(x for x in portage.auxdbkeys \ @@ -214,10 +219,10 @@ if not x.startswith('UNUSED_'))) metadata.uses_root = True def contents(argv): - """<root> <category/package> + """<eroot> <category/package> List the files that are installed for a given package, with one file listed on each line. All file names will begin with - <root>. + <eroot>. """ if len(argv) != 2: print("ERROR: expected 2 parameters, got %d!" % len(argv)) @@ -236,11 +241,11 @@ def contents(argv): contents.uses_root = True def owners(argv): - """<root> [<filename>]+ + """<eroot> [<filename>]+ Given a list of files, print the packages that own the files and which files belong to each package. Files owned by a package are listed on the lines below it, indented by a single tab character (\\t). All file - paths must either start with <root> or be a basename alone. + paths must either start with <eroot> or be a basename alone. Returns 1 if no owners could be found, and 0 otherwise. """ if len(argv) < 2: @@ -249,9 +254,9 @@ def owners(argv): return 2 from portage import catsplit, dblink - settings = portage.settings - root = settings["ROOT"] - vardb = portage.db[root]["vartree"].dbapi + eroot = argv[0] + vardb = portage.db[eroot]["vartree"].dbapi + root = portage.settings['ROOT'] cwd = None try: @@ -272,8 +277,8 @@ def owners(argv): return 2 f = os.path.join(cwd, f) f = portage.normalize_path(f) - if not is_basename and not f.startswith(root): - sys.stderr.write("ERROR: file paths must begin with <root>!\n") + if not is_basename and not f.startswith(eroot): + sys.stderr.write("ERROR: file paths must begin with <eroot>!\n") sys.stderr.flush() return 2 if is_basename: @@ -317,9 +322,9 @@ def owners(argv): owners.uses_root = True def is_protected(argv): - """<root> <filename> + """<eroot> <filename> Given a single filename, return code 0 if it's protected, 1 otherwise. - The filename must begin with <root>. + The filename must begin with <eroot>. """ if len(argv) != 2: sys.stderr.write("ERROR: expected 2 parameters, got %d!\n" % len(argv)) @@ -345,7 +350,7 @@ def is_protected(argv): f = portage.normalize_path(f) if not f.startswith(root): - err.write("ERROR: file paths must begin with <root>!\n") + err.write("ERROR: file paths must begin with <eroot>!\n") err.flush() return 2 @@ -364,9 +369,9 @@ def is_protected(argv): is_protected.uses_root = True def filter_protected(argv): - """<root> + """<eroot> Read filenames from stdin and write them to stdout if they are protected. - All filenames are delimited by \\n and must begin with <root>. + All filenames are delimited by \\n and must begin with <eroot>. """ if len(argv) != 1: sys.stderr.write("ERROR: expected 1 parameter, got %d!\n" % len(argv)) @@ -406,7 +411,7 @@ def filter_protected(argv): f = portage.normalize_path(f) if not f.startswith(root): - err.write("ERROR: file paths must begin with <root>!\n") + err.write("ERROR: file paths must begin with <eroot>!\n") err.flush() errors += 1 continue @@ -424,7 +429,7 @@ def filter_protected(argv): filter_protected.uses_root = True def best_visible(argv): - """<root> [pkgtype] <atom> + """<eroot> [pkgtype] <atom> Returns category/package-version (without .ebuild). The pkgtype argument defaults to "ebuild" if unspecified, otherwise it must be one of ebuild, binary, or installed. @@ -450,7 +455,8 @@ def best_visible(argv): noiselevel=-1) return 2 - db = portage.db[portage.settings["ROOT"]][type_map[pkgtype]].dbapi + eroot = argv[0] + db = portage.db[eroot][type_map[pkgtype]].dbapi try: atom = portage.dep_expand(atom, mydb=db, settings=portage.settings) @@ -460,43 +466,80 @@ def best_visible(argv): return 2 root_config = RootConfig(portage.settings, - portage.db[portage.settings["ROOT"]], None) + portage.db[eroot], None) - try: + if hasattr(db, "xmatch"): + cpv_list = db.xmatch("match-all-cpv-only", atom) + else: + cpv_list = db.match(atom) + + if cpv_list: # reversed, for descending order - for cpv in reversed(db.match(atom)): - metadata = dict(zip(Package.metadata_keys, - db.aux_get(cpv, Package.metadata_keys, myrepo=atom.repo))) - pkg = Package(built=(pkgtype != "ebuild"), cpv=cpv, - installed=(pkgtype=="installed"), metadata=metadata, - root_config=root_config, type_name=pkgtype) - if pkg.visible: - writemsg_stdout("%s\n" % (pkg.cpv,), noiselevel=-1) - return os.EX_OK - except KeyError: - pass + cpv_list.reverse() + # verify match, since the atom may match the package + # for a given cpv from one repo but not another, and + # we can use match-all-cpv-only to avoid redundant + # metadata access. + atom_set = InternalPackageSet(initial_atoms=(atom,)) + + if atom.repo is None and hasattr(db, "getRepositories"): + repo_list = db.getRepositories() + else: + repo_list = [atom.repo] + + for cpv in cpv_list: + for repo in repo_list: + try: + metadata = dict(zip(Package.metadata_keys, + db.aux_get(cpv, Package.metadata_keys, myrepo=repo))) + except KeyError: + continue + pkg = Package(built=(pkgtype != "ebuild"), cpv=cpv, + installed=(pkgtype=="installed"), metadata=metadata, + root_config=root_config, type_name=pkgtype) + if not atom_set.findAtomForPackage(pkg): + continue + + if pkg.visible: + writemsg_stdout("%s\n" % (pkg.cpv,), noiselevel=-1) + return os.EX_OK + + # No package found, write out an empty line. + writemsg_stdout("\n", noiselevel=-1) + return 1 best_visible.uses_root = True def mass_best_visible(argv): - """<root> [<category/package>]+ + """<root> [<type>] [<category/package>]+ Returns category/package-version (without .ebuild). + The pkgtype argument defaults to "ebuild" if unspecified, + otherwise it must be one of ebuild, binary, or installed. """ + type_map = { + "ebuild":"porttree", + "binary":"bintree", + "installed":"vartree"} + if (len(argv) < 2): print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 try: - for pack in argv[1:]: - mylist=portage.db[argv[0]]["porttree"].dbapi.match(pack) - print(pack+":"+portage.best(mylist)) + root = argv.pop(0) + pkgtype = "ebuild" + if argv[0] in type_map: + pkgtype = argv.pop(0) + for pack in argv: + writemsg_stdout("%s:" % pack, noiselevel=-1) + best_visible([root, pkgtype, pack]) except KeyError: - sys.exit(1) + return 1 mass_best_visible.uses_root = True def all_best_visible(argv): - """<root> + """<eroot> Returns all best_visible packages (without .ebuild). """ if len(argv) < 1: @@ -513,30 +556,57 @@ all_best_visible.uses_root = True def match(argv): - """<root> <atom> + """<eroot> <atom> Returns a \\n separated list of category/package-version. When given an empty string, all installed packages will be listed. """ if len(argv) != 2: print("ERROR: expected 2 parameters, got %d!" % len(argv)) - sys.exit(2) + return 2 root, atom = argv - if atom: - if atom_validate_strict and not portage.isvalidatom(atom): - portage.writemsg("ERROR: Invalid atom: '%s'\n" % atom, - noiselevel=-1) - return 2 - results = portage.db[root]["vartree"].dbapi.match(atom) - else: - results = portage.db[root]["vartree"].dbapi.cpv_all() + if not atom: + atom = "*/*" + + vardb = portage.db[root]["vartree"].dbapi + try: + atom = portage.dep.Atom(atom, allow_wildcard=True, allow_repo=True) + except portage.exception.InvalidAtom: + # maybe it's valid but missing category + atom = portage.dep_expand(atom, mydb=vardb, settings=vardb.settings) + + if atom.extended_syntax: + if atom == "*/*": + results = vardb.cpv_all() + else: + results = [] + require_metadata = atom.slot or atom.repo + for cpv in vardb.cpv_all(): + + if not portage.dep.extended_cp_match( + atom.cp, portage.cpv_getkey(cpv)): + continue + + if require_metadata: + slot, repo = vardb.aux_get(cpv, ["SLOT", "repository"]) + + if atom.slot is not None and atom.slot != slot: + continue + + if atom.repo is not None and atom.repo != repo: + continue + + results.append(cpv) + results.sort() + else: + results = vardb.match(atom) for cpv in results: print(cpv) match.uses_root = True def expand_virtual(argv): - """<root> <atom> + """<eroot> <atom> Returns a \\n separated list of atoms expanded from a given virtual atom (GLEP 37 virtuals only), excluding blocker atoms. Satisfied @@ -630,6 +700,13 @@ def distdir(argv): print(portage.settings["DISTDIR"]) +def colormap(argv): + """ + Display the color.map as environment variables. + """ + print(portage.output.colormap()) + + def envvar(argv): """<variable>+ Returns a specific environment variable as exists prior to ebuild.sh. @@ -641,7 +718,7 @@ def envvar(argv): if len(argv) == 0: print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 for arg in argv: if verbose: @@ -650,29 +727,33 @@ def envvar(argv): print(portage.settings[arg]) def get_repos(argv): - """<root> - Returns all repos with names (repo_name file) argv[0] = $ROOT + """<eroot> + Returns all repos with names (repo_name file) argv[0] = $EROOT """ if len(argv) < 1: print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 print(" ".join(portage.db[argv[0]]["porttree"].dbapi.getRepositories())) +get_repos.uses_root = True + def get_repo_path(argv): - """<root> <repo_id>+ - Returns the path to the repo named argv[1], argv[0] = $ROOT + """<eroot> <repo_id>+ + Returns the path to the repo named argv[1], argv[0] = $EROOT """ if len(argv) < 2: print("ERROR: insufficient parameters!") - sys.exit(2) + return 2 for arg in argv[1:]: path = portage.db[argv[0]]["porttree"].dbapi.getRepositoryPath(arg) if path is None: path = "" print(path) +get_repo_path.uses_root = True + def list_preserved_libs(argv): - """<root> + """<eroot> Print a list of libraries preserved during a package update in the form package: path. Returns 1 if no preserved libraries could be found, 0 otherwise. @@ -680,7 +761,7 @@ def list_preserved_libs(argv): if len(argv) != 1: print("ERROR: wrong number of arguments") - sys.exit(2) + return 2 mylibs = portage.db[argv[0]]["vartree"].dbapi._plib_registry.getPreservedLibs() rValue = 1 msg = [] @@ -788,16 +869,22 @@ def main(): sys.stderr.write("Run portageq with --help for info\n") sys.stderr.flush() sys.exit(os.EX_USAGE) - os.environ["ROOT"] = sys.argv[2] + eprefix = portage.const.EPREFIX + eroot = portage.util.normalize_path(sys.argv[2]) + if eprefix: + root = eroot[:1-len(eprefix)] + else: + root = eroot + os.environ["ROOT"] = root args = sys.argv[2:] - if args and sys.hexversion < 0x3000000 and not isinstance(args[0], unicode): + if args and isinstance(args[0], bytes): for i in range(len(args)): args[i] = portage._unicode_decode(args[i]) try: if uses_root: - args[0] = portage.settings["ROOT"] + args[0] = portage.settings['EROOT'] retval = function(args) if retval: sys.exit(retval) diff --git a/portage_with_autodep/bin/quickpkg b/portage_with_autodep/bin/quickpkg index 09723f5..d908c03 100755 --- a/portage_with_autodep/bin/quickpkg +++ b/portage_with_autodep/bin/quickpkg @@ -21,9 +21,9 @@ except ImportError: from portage import os from portage import xpak from portage.dbapi.dep_expand import dep_expand -from portage.dep import use_reduce -from portage.exception import InvalidAtom, InvalidData, InvalidDependString, \ - PackageSetNotFound, PermissionDenied +from portage.dep import Atom, extended_cp_match, use_reduce +from portage.exception import (AmbiguousPackageName, InvalidAtom, InvalidData, + InvalidDependString, PackageSetNotFound, PermissionDenied) from portage.util import ConfigProtect, ensure_dirs, shlex_split from portage.dbapi.vartree import dblink, tar_contents from portage.checksum import perform_md5 @@ -31,8 +31,9 @@ from portage._sets import load_default_config, SETPREFIX def quickpkg_atom(options, infos, arg, eout): settings = portage.settings - root = portage.settings["ROOT"] - trees = portage.db[root] + root = portage.settings['ROOT'] + eroot = portage.settings['EROOT'] + trees = portage.db[eroot] vartree = trees["vartree"] vardb = vartree.dbapi bintree = trees["bintree"] @@ -43,7 +44,7 @@ def quickpkg_atom(options, infos, arg, eout): try: atom = dep_expand(arg, mydb=vardb, settings=vartree.settings) - except ValueError as e: + except AmbiguousPackageName as e: # Multiple matches thrown from cpv_expand eout.eerror("Please use a more specific atom: %s" % \ " ".join(e.args[0])) @@ -65,10 +66,7 @@ def quickpkg_atom(options, infos, arg, eout): for cpv in matches: excluded_config_files = [] bintree.prevent_collision(cpv) - cat, pkg = portage.catsplit(cpv) - dblnk = dblink(cat, pkg, root, - vartree.settings, treetype="vartree", - vartree=vartree) + dblnk = vardb._dblink(cpv) have_lock = False try: dblnk.lockdb() @@ -101,7 +99,7 @@ def quickpkg_atom(options, infos, arg, eout): contents = dblnk.getcontents() protect = None if not include_config: - confprot = ConfigProtect(root, + confprot = ConfigProtect(eroot, shlex_split(settings.get("CONFIG_PROTECT", "")), shlex_split(settings.get("CONFIG_PROTECT_MASK", ""))) def protect(filename): @@ -161,8 +159,8 @@ def quickpkg_atom(options, infos, arg, eout): infos["missing"].append(arg) def quickpkg_set(options, infos, arg, eout): - root = portage.settings["ROOT"] - trees = portage.db[root] + eroot = portage.settings['EROOT'] + trees = portage.db[eroot] vartree = trees["vartree"] settings = vartree.settings @@ -187,9 +185,43 @@ def quickpkg_set(options, infos, arg, eout): for atom in atoms: quickpkg_atom(options, infos, atom, eout) + +def quickpkg_extended_atom(options, infos, atom, eout): + eroot = portage.settings['EROOT'] + trees = portage.db[eroot] + vartree = trees["vartree"] + vardb = vartree.dbapi + + require_metadata = atom.slot or atom.repo + atoms = [] + for cpv in vardb.cpv_all(): + cpv_atom = Atom("=%s" % cpv) + + if atom == "*/*": + atoms.append(cpv_atom) + continue + + if not extended_cp_match(atom.cp, cpv_atom.cp): + continue + + if require_metadata: + slot, repo = vardb.aux_get(cpv, ["SLOT", "repository"]) + + if atom.slot and atom.slot != slot: + continue + + if atom.repo and atom.repo != repo: + continue + + atoms.append(cpv_atom) + + for atom in atoms: + quickpkg_atom(options, infos, atom, eout) + + def quickpkg_main(options, args, eout): - root = portage.settings["ROOT"] - trees = portage.db[root] + eroot = portage.settings['EROOT'] + trees = portage.db[eroot] bintree = trees["bintree"] try: @@ -207,8 +239,17 @@ def quickpkg_main(options, args, eout): for arg in args: if arg[0] == SETPREFIX: quickpkg_set(options, infos, arg, eout) - else: + continue + try: + atom = Atom(arg, allow_wildcard=True, allow_repo=True) + except (InvalidAtom, InvalidData): + # maybe it's valid but missing category (requires dep_expand) quickpkg_atom(options, infos, arg, eout) + else: + if atom.extended_syntax: + quickpkg_extended_atom(options, infos, atom, eout) + else: + quickpkg_atom(options, infos, atom, eout) if not infos["successes"]: eout.eerror("No packages found") diff --git a/portage_with_autodep/bin/regenworld b/portage_with_autodep/bin/regenworld index 6b5af4c..3199fdf 100755 --- a/portage_with_autodep/bin/regenworld +++ b/portage_with_autodep/bin/regenworld @@ -47,7 +47,6 @@ def isunwanted(pkgline): __uniqlist__.append(pkgline) return True -root = portage.settings['ROOT'] eroot = portage.settings['EROOT'] world_file = os.path.join(eroot, portage.WORLD_FILE) @@ -78,7 +77,7 @@ for mykey in syslist: # drop the asterix mykey = mykey[1:] #print("candidate:",mykey) - mylist = portage.db[root]["vartree"].dbapi.match(mykey) + mylist = portage.db[eroot]["vartree"].dbapi.match(mykey) if mylist: mykey=portage.cpv_getkey(mylist[0]) if mykey not in realsyslist: @@ -87,7 +86,7 @@ for mykey in syslist: for mykey in biglist: #print("checking:",mykey) try: - mylist = portage.db[root]["vartree"].dbapi.match(mykey) + mylist = portage.db[eroot]["vartree"].dbapi.match(mykey) except (portage.exception.InvalidAtom, KeyError): if "--debug" in sys.argv: print("* ignoring broken log entry for %s (likely injected)" % mykey) diff --git a/portage_with_autodep/bin/repoman b/portage_with_autodep/bin/repoman index f1fbc24..fd87847 100755 --- a/portage_with_autodep/bin/repoman +++ b/portage_with_autodep/bin/repoman @@ -1,5 +1,5 @@ #!/usr/bin/python -O -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # Next to do: dep syntax checking in mask files @@ -18,8 +18,10 @@ import optparse import re import signal import stat +import subprocess import sys import tempfile +import textwrap import time import platform @@ -43,7 +45,7 @@ portage.dep._internal_warnings = True try: import xml.etree.ElementTree from xml.parsers.expat import ExpatError -except ImportError: +except (ImportError, SystemError): msg = ["Please enable python's \"xml\" USE flag in order to use repoman."] from portage.output import EOutput out = EOutput() @@ -67,16 +69,16 @@ from portage import cvstree, normalize_path from portage import util from portage.exception import (FileNotFound, MissingParameter, ParseError, PermissionDenied) -from portage.manifest import Manifest +from portage.manifest import _prohibited_filename_chars_re as \ + disallowed_filename_chars_re from portage.process import find_binary, spawn from portage.output import bold, create_color_func, \ green, nocolor, red from portage.output import ConsoleStyleFile, StyleWriter -from portage.util import cmp_sort_key, writemsg_level +from portage.util import writemsg_level +from portage.util._desktop_entry import validate_desktop_entry from portage.package.ebuild.digestgen import digestgen -from portage.eapi import eapi_has_slot_deps, \ - eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_iuse_defaults, \ - eapi_has_required_use, eapi_has_use_dep_defaults +from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use if sys.hexversion >= 0x3000000: basestring = str @@ -86,7 +88,6 @@ util.initialize_logger() # 14 is the length of DESCRIPTION="" max_desc_len = 100 allowed_filename_chars="a-zA-Z0-9._-+:" -disallowed_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]') pv_toolong_re = re.compile(r'[0-9]{19,}') bad = create_color_func("BAD") @@ -96,8 +97,8 @@ os.umask(0o22) # behave incrementally. repoman_incrementals = tuple(x for x in \ portage.const.INCREMENTALS if x != 'ACCEPT_KEYWORDS') -repoman_settings = portage.config(local_config=False) -repoman_settings.lock() +config_root = os.environ.get("PORTAGE_CONFIGROOT") +repoman_settings = portage.config(config_root=config_root, local_config=False) if repoman_settings.get("NOCOLOR", "").lower() in ("yes", "true") or \ repoman_settings.get('TERM') == 'dumb' or \ @@ -156,7 +157,7 @@ def ParseArgs(argv, qahelp): (opts, args), just like a call to parser.parse_args() """ - if argv and sys.hexversion < 0x3000000 and not isinstance(argv[0], unicode): + if argv and isinstance(argv[0], bytes): argv = [portage._unicode_decode(x) for x in argv] modes = { @@ -188,12 +189,21 @@ def ParseArgs(argv, qahelp): parser.add_option('-M', '--commitmsgfile', dest='commitmsgfile', help='specify a path to a file that contains a commit message') + parser.add_option('--digest', + type='choice', choices=('y', 'n'), metavar='<y|n>', + help='Automatically update Manifest digests for modified files') + parser.add_option('-p', '--pretend', dest='pretend', default=False, action='store_true', help='don\'t commit or fix anything; just show what would be done') parser.add_option('-q', '--quiet', dest="quiet", action="count", default=0, help='do not print unnecessary messages') + parser.add_option( + '--echangelog', type='choice', choices=('y', 'n', 'force'), metavar="<y|n|force>", + help='for commit mode, call echangelog if ChangeLog is unmodified (or ' + 'regardless of modification if \'force\' is specified)') + parser.add_option('-f', '--force', dest='force', default=False, action='store_true', help='Commit with QA violations') @@ -209,9 +219,18 @@ def ParseArgs(argv, qahelp): parser.add_option('-x', '--xmlparse', dest='xml_parse', action='store_true', default=False, help='forces the metadata.xml parse check to be carried out') + parser.add_option( + '--if-modified', type='choice', choices=('y', 'n'), default='n', + metavar="<y|n>", + help='only check packages that have uncommitted modifications') + parser.add_option('-i', '--ignore-arches', dest='ignore_arches', action='store_true', default=False, help='ignore arch-specific failures (where arch != host)') + parser.add_option("--ignore-default-opts", + action="store_true", + help="do not use the REPOMAN_DEFAULT_OPTS environment variable") + parser.add_option('-I', '--ignore-masked', dest='ignore_masked', action='store_true', default=False, help='ignore masked packages (not allowed with commit mode)') @@ -241,6 +260,11 @@ def ParseArgs(argv, qahelp): opts, args = parser.parse_args(argv[1:]) + if not opts.ignore_default_opts: + default_opts = repoman_settings.get("REPOMAN_DEFAULT_OPTS", "").split() + if default_opts: + opts, args = parser.parse_args(default_opts + sys.argv[1:]) + if opts.mode == 'help': parser.print_help(short=False) @@ -285,8 +309,8 @@ qahelp={ "ebuild.notadded":"Ebuilds that exist but have not been added to cvs", "ebuild.patches":"PATCHES variable should be a bash array to ensure white space safety", "changelog.notadded":"ChangeLogs that exist but have not been added to cvs", - "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be provided by an overlay)", - "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do note need the executable bit", + "dependency.unknown" : "Ebuild has a dependency that refers to an unknown package (which may be valid if it is a blocker for a renamed/removed package, or is an alternative choice provided by an overlay)", + "file.executable":"Ebuilds, digests, metadata.xml, Manifest, and ChangeLog do not need the executable bit", "file.size":"Files in the files directory must be under 20 KiB", "file.size.fatal":"Files in the files directory must be under 60 KiB", "file.name":"File/dir name must be composed of only the following chars: %s " % allowed_filename_chars, @@ -303,7 +327,7 @@ qahelp={ "LICENSE.virtual":"Virtuals that have a non-empty LICENSE variable", "DESCRIPTION.missing":"Ebuilds that have a missing or empty DESCRIPTION variable", "DESCRIPTION.toolong":"DESCRIPTION is over %d characters" % max_desc_len, - "EAPI.definition":"EAPI is defined after an inherit call (must be defined before)", + "EAPI.definition":"EAPI definition does not conform to PMS section 7.3.1 (first non-comment, non-blank line)", "EAPI.deprecated":"Ebuilds that use features that are deprecated in the current EAPI", "EAPI.incompatible":"Ebuilds that use features that are only available with a different EAPI", "EAPI.unsupported":"Ebuilds that have an unsupported EAPI version (you must upgrade portage)", @@ -355,8 +379,6 @@ qahelp={ "digest.assumed":"Existing digest must be assumed correct (Package level only)", "digest.missing":"Some files listed in SRC_URI aren't referenced in the Manifest", "digest.unused":"Some files listed in the Manifest aren't referenced in SRC_URI", - "ebuild.nostable":"There are no ebuilds that are marked as stable for your ARCH", - "ebuild.allmasked":"All ebuilds are masked for this package (Package level only)", "ebuild.majorsyn":"This ebuild has a major syntax error that may cause the ebuild to fail partially or fully", "ebuild.minorsyn":"This ebuild has a minor syntax error that contravenes gentoo coding style", "ebuild.badheader":"This ebuild has a malformed header", @@ -381,8 +403,6 @@ qawarnings = set(( "digest.assumed", "digest.unused", "ebuild.notadded", -"ebuild.nostable", -"ebuild.allmasked", "ebuild.nesteddie", "desktop.invalid", "DEPEND.badmasked","RDEPEND.badmasked","PDEPEND.badmasked", @@ -401,7 +421,6 @@ qawarnings = set(( "RDEPEND.implicit", "RDEPEND.suspect", "RESTRICT.invalid", -"SRC_URI.mirror", "ebuild.minorsyn", "ebuild.badheader", "ebuild.patches", @@ -414,7 +433,6 @@ qawarnings = set(( "portage.internal", "usage.obsolete", "upstream.workaround", -"virtual.oldstyle", "LIVEVCS.stable", "LIVEVCS.unmasked", )) @@ -524,14 +542,13 @@ else: else: vcs = None -# Note: We don't use ChangeLogs in distributed SCMs. -# It will be generated on server side from scm log, -# before package moves to the rsync server. -# This is needed because we try to avoid merge collisions. -check_changelog = vcs in ('cvs', 'svn') +if options.if_modified == "y" and vcs is None: + logging.info("Not in a version controlled repository; " + "disabling --if-modified.") + options.if_modified = "n" # Disable copyright/mtime check if vcs does not preserve mtime (bug #324075). -vcs_preserves_mtime = vcs not in ('git',) +vcs_preserves_mtime = vcs in ('cvs',) vcs_local_opts = repoman_settings.get("REPOMAN_VCS_LOCAL_OPTS", "").split() vcs_global_opts = repoman_settings.get("REPOMAN_VCS_GLOBAL_OPTS") @@ -542,50 +559,115 @@ if vcs_global_opts is None: vcs_global_opts = "" vcs_global_opts = vcs_global_opts.split() -if vcs == "cvs" and \ - "commit" == options.mode and \ - "RMD160" not in portage.checksum.hashorigin_map: - from portage.util import grablines - repo_lines = grablines("./CVS/Repository") - if repo_lines and \ - "gentoo-x86" == repo_lines[0].strip().split(os.path.sep)[0]: - msg = "Please install " \ - "pycrypto or enable python's ssl USE flag in order " \ - "to enable RMD160 hash support. See bug #198398 for " \ - "more information." - prefix = bad(" * ") - from textwrap import wrap - for line in wrap(msg, 70): - print(prefix + line) - sys.exit(1) - del repo_lines - if options.mode == 'commit' and not options.pretend and not vcs: logging.info("Not in a version controlled repository; enabling pretend mode.") options.pretend = True # Ensure that PORTDIR_OVERLAY contains the repository corresponding to $PWD. -repoman_settings = portage.config(local_config=False) repoman_settings['PORTDIR_OVERLAY'] = "%s %s" % \ - (repoman_settings.get('PORTDIR_OVERLAY', ''), portdir_overlay) + (repoman_settings.get('PORTDIR_OVERLAY', ''), + portage._shell_quote(portdir_overlay)) # We have to call the config constructor again so # that config.repositories is initialized correctly. -repoman_settings = portage.config(local_config=False, env=dict(os.environ, - PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY'])) +repoman_settings = portage.config(config_root=config_root, local_config=False, + env=dict(os.environ, PORTDIR_OVERLAY=repoman_settings['PORTDIR_OVERLAY'])) -root = '/' +root = repoman_settings['EROOT'] trees = { - root : {'porttree' : portage.portagetree(root, settings=repoman_settings)} + root : {'porttree' : portage.portagetree(settings=repoman_settings)} } portdb = trees[root]['porttree'].dbapi # Constrain dependency resolution to the master(s) # that are specified in layout.conf. -portdir_overlay = os.path.realpath(portdir_overlay) -repo_info = portdb._repo_info[portdir_overlay] -portdb.porttrees = list(repo_info.eclass_db.porttrees) +repodir = os.path.realpath(portdir_overlay) +repo_config = repoman_settings.repositories.get_repo_for_location(repodir) +portdb.porttrees = list(repo_config.eclass_db.porttrees) portdir = portdb.porttrees[0] +if repo_config.allow_provide_virtual: + qawarnings.add("virtual.oldstyle") + +if repo_config.sign_commit: + if vcs == 'git': + # NOTE: It's possible to use --gpg-sign=key_id to specify the key in + # the commit arguments. If key_id is unspecified, then it must be + # configured by `git config user.signingkey key_id`. + vcs_local_opts.append("--gpg-sign") + +# In order to disable manifest signatures, repos may set +# "sign-manifests = false" in metadata/layout.conf. This +# can be used to prevent merge conflicts like those that +# thin-manifests is designed to prevent. +sign_manifests = "sign" in repoman_settings.features and \ + repo_config.sign_manifest + +manifest_hashes = repo_config.manifest_hashes +if manifest_hashes is None: + manifest_hashes = portage.const.MANIFEST2_HASH_DEFAULTS + +if options.mode in ("commit", "fix", "manifest"): + if portage.const.MANIFEST2_REQUIRED_HASH not in manifest_hashes: + msg = ("The 'manifest-hashes' setting in the '%s' repository's " + "metadata/layout.conf does not contain the '%s' hash which " + "is required by this portage version. You will have to " + "upgrade portage if you want to generate valid manifests for " + "this repository.") % \ + (repo_config.name, portage.const.MANIFEST2_REQUIRED_HASH) + for line in textwrap.wrap(msg, 70): + logging.error(line) + sys.exit(1) + + unsupported_hashes = manifest_hashes.difference( + portage.const.MANIFEST2_HASH_FUNCTIONS) + if unsupported_hashes: + msg = ("The 'manifest-hashes' setting in the '%s' repository's " + "metadata/layout.conf contains one or more hash types '%s' " + "which are not supported by this portage version. You will " + "have to upgrade portage if you want to generate valid " + "manifests for this repository.") % \ + (repo_config.name, " ".join(sorted(unsupported_hashes))) + for line in textwrap.wrap(msg, 70): + logging.error(line) + sys.exit(1) + +if "commit" == options.mode and \ + repo_config.name == "gentoo" and \ + "RMD160" in manifest_hashes and \ + "RMD160" not in portage.checksum.hashorigin_map: + msg = "Please install " \ + "pycrypto or enable python's ssl USE flag in order " \ + "to enable RMD160 hash support. See bug #198398 for " \ + "more information." + prefix = bad(" * ") + for line in textwrap.wrap(msg, 70): + print(prefix + line) + sys.exit(1) + +if options.echangelog is None and repo_config.update_changelog: + options.echangelog = 'y' + +if vcs is None: + options.echangelog = 'n' + +# The --echangelog option causes automatic ChangeLog generation, +# which invalidates changelog.ebuildadded and changelog.missing +# checks. +# Note: Some don't use ChangeLogs in distributed SCMs. +# It will be generated on server side from scm log, +# before package moves to the rsync server. +# This is needed because they try to avoid merge collisions. +# Gentoo's Council decided to always use the ChangeLog file. +# TODO: shouldn't this just be switched on the repo, iso the VCS? +check_changelog = options.echangelog not in ('y', 'force') and vcs in ('cvs', 'svn') + +if 'digest' in repoman_settings.features and options.digest != 'n': + options.digest = 'y' + +logging.debug("vcs: %s" % (vcs,)) +logging.debug("repo config: %s" % (repo_config,)) +logging.debug("options: %s" % (options,)) + # Generate an appropriate PORTDIR_OVERLAY value for passing into the # profile-specific config constructor calls. env = os.environ.copy() @@ -601,19 +683,19 @@ logging.info('PORTDIR_OVERLAY = "%s"' % env['PORTDIR_OVERLAY']) env['FEATURES'] = env.get('FEATURES', '') + ' -unknown-features-warn' categories = [] -for path in set([portdir, portdir_overlay]): +for path in repo_config.eclass_db.porttrees: categories.extend(portage.util.grabfile( os.path.join(path, 'profiles', 'categories'))) -repoman_settings.categories = tuple(sorted( - portage.util.stack_lists([categories], incremental=1))) -del categories +repoman_settings.categories = frozenset( + portage.util.stack_lists([categories], incremental=1)) +categories = repoman_settings.categories portdb.settings = repoman_settings root_config = RootConfig(repoman_settings, trees[root], None) # We really only need to cache the metadata that's necessary for visibility # filtering. Anything else can be discarded to reduce memory consumption. portdb._aux_cache_keys.clear() -portdb._aux_cache_keys.update(["EAPI", "KEYWORDS", "SLOT"]) +portdb._aux_cache_keys.update(["EAPI", "IUSE", "KEYWORDS", "repository", "SLOT"]) reposplit = myreporoot.split(os.path.sep) repolevel = len(reposplit) @@ -628,11 +710,13 @@ if options.mode == 'commit' and repolevel not in [1,2,3]: print(red("***")) err("Unable to identify level we're commiting from for %s" % '/'.join(reposplit)) -startdir = normalize_path(mydir) -repodir = startdir -for x in range(0, repolevel - 1): - repodir = os.path.dirname(repodir) -repodir = os.path.realpath(repodir) +# Make startdir relative to the canonical repodir, so that we can pass +# it to digestgen and it won't have to be canonicalized again. +if repolevel == 1: + startdir = repodir +else: + startdir = normalize_path(mydir) + startdir = os.path.join(repodir, *startdir.split(os.sep)[-2-repolevel+3:]) def caterror(mycat): err(mycat+" is not an official category. Skipping QA checks in this directory.\nPlease ensure that you add "+catdir+" to "+repodir+"/profiles/categories\nif it is a new category.") @@ -792,7 +876,7 @@ scanlist=[] if repolevel==2: #we are inside a category directory catdir=reposplit[-1] - if catdir not in repoman_settings.categories: + if catdir not in categories: caterror(catdir) mydirlist=os.listdir(startdir) for x in mydirlist: @@ -802,7 +886,7 @@ if repolevel==2: scanlist.append(catdir+"/"+x) repo_subdir = catdir + os.sep elif repolevel==1: - for x in repoman_settings.categories: + for x in categories: if not os.path.isdir(startdir+"/"+x): continue for y in os.listdir(startdir+"/"+x): @@ -813,7 +897,7 @@ elif repolevel==1: repo_subdir = "" elif repolevel==3: catdir = reposplit[-2] - if catdir not in repoman_settings.categories: + if catdir not in categories: caterror(catdir) scanlist.append(catdir+"/"+reposplit[-1]) repo_subdir = scanlist[-1] + os.sep @@ -828,6 +912,53 @@ scanlist.sort() logging.debug("Found the following packages to scan:\n%s" % '\n'.join(scanlist)) +def vcs_files_to_cps(vcs_file_iter): + """ + Iterate over the given modified file paths returned from the vcs, + and return a frozenset containing category/pn strings for each + modified package. + """ + + modified_cps = [] + + if repolevel == 3: + if reposplit[-2] in categories and \ + next(vcs_file_iter, None) is not None: + modified_cps.append("/".join(reposplit[-2:])) + + elif repolevel == 2: + category = reposplit[-1] + if category in categories: + for filename in vcs_file_iter: + f_split = filename.split(os.sep) + # ['.', pn,...] + if len(f_split) > 2: + modified_cps.append(category + "/" + f_split[1]) + + else: + # repolevel == 1 + for filename in vcs_file_iter: + f_split = filename.split(os.sep) + # ['.', category, pn,...] + if len(f_split) > 3 and f_split[1] in categories: + modified_cps.append("/".join(f_split[1:3])) + + return frozenset(modified_cps) + +def git_supports_gpg_sign(): + status, cmd_output = \ + subprocess_getstatusoutput("git --version") + cmd_output = cmd_output.split() + if cmd_output: + version = re.match(r'^(\d+)\.(\d+)\.(\d+)', cmd_output[-1]) + if version is not None: + version = [int(x) for x in version.groups()[1:]] + if version[0] > 1 or \ + (version[0] == 1 and version[1] > 7) or \ + (version[0] == 1 and version[1] == 7 and version[2] >= 9): + return True + return False + def dev_keywords(profiles): """ Create a set of KEYWORDS values that exist in 'dev' @@ -896,7 +1027,7 @@ def fetch_metadata_dtd(): Fetch metadata.dtd if it doesn't exist or the ctime is older than metadata_dtd_ctime_interval. @rtype: bool - @returns: True if successful, otherwise False + @return: True if successful, otherwise False """ must_fetch = True @@ -995,25 +1126,51 @@ if vcs == "cvs": mycvstree = cvstree.getentries("./", recursive=1) mychanged = cvstree.findchanged(mycvstree, recursive=1, basedir="./") mynew = cvstree.findnew(mycvstree, recursive=1, basedir="./") -if vcs == "svn": - svnstatus = os.popen("svn status").readlines() + if options.if_modified == "y": + myremoved = cvstree.findremoved(mycvstree, recursive=1, basedir="./") + +elif vcs == "svn": + with os.popen("svn status") as f: + svnstatus = f.readlines() mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem and elem[:1] in "MR" ] mynew = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A") ] + if options.if_modified == "y": + myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")] + elif vcs == "git": - mychanged = os.popen("git diff-index --name-only --relative --diff-filter=M HEAD").readlines() + with os.popen("git diff-index --name-only " + "--relative --diff-filter=M HEAD") as f: + mychanged = f.readlines() mychanged = ["./" + elem[:-1] for elem in mychanged] - mynew = os.popen("git diff-index --name-only --relative --diff-filter=A HEAD").readlines() + with os.popen("git diff-index --name-only " + "--relative --diff-filter=A HEAD") as f: + mynew = f.readlines() mynew = ["./" + elem[:-1] for elem in mynew] + if options.if_modified == "y": + with os.popen("git diff-index --name-only " + "--relative --diff-filter=D HEAD") as f: + myremoved = f.readlines() + myremoved = ["./" + elem[:-1] for elem in myremoved] + elif vcs == "bzr": - bzrstatus = os.popen("bzr status -S .").readlines() + with os.popen("bzr status -S .") as f: + bzrstatus = f.readlines() mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ] mynew = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "NK" or elem[0:1] == "R" ) ] + if options.if_modified == "y": + myremoved = [ "./" + elem.split()[-3:-2][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] == "K" or elem[0:1] == "R" ) ] + elif vcs == "hg": - mychanged = os.popen("hg status --no-status --modified .").readlines() + with os.popen("hg status --no-status --modified .") as f: + mychanged = f.readlines() mychanged = ["./" + elem.rstrip() for elem in mychanged] mynew = os.popen("hg status --no-status --added .").readlines() mynew = ["./" + elem.rstrip() for elem in mynew] + if options.if_modified == "y": + with os.popen("hg status --no-status --removed .") as f: + myremoved = f.readlines() + myremoved = ["./" + elem.rstrip() for elem in myremoved] if vcs: new_ebuilds.update(x for x in mynew if x.endswith(".ebuild")) @@ -1021,9 +1178,18 @@ if vcs: modified_changelogs.update(x for x in chain(mychanged, mynew) \ if os.path.basename(x) == "ChangeLog") +def vcs_new_changed(relative_path): + for x in chain(mychanged, mynew): + if x == relative_path: + return True + return False + have_pmasked = False have_dev_keywords = False dofail = 0 + +# NOTE: match-all caches are not shared due to potential +# differences between profiles in _get_implicit_iuse. arch_caches={} arch_xmatch_caches = {} shared_xmatch_caches = {"cp-list":{}} @@ -1037,7 +1203,10 @@ check_ebuild_notadded = not \ # Build a regex from thirdpartymirrors for the SRC_URI.mirror check. thirdpartymirrors = [] for v in repoman_settings.thirdpartymirrors().values(): - thirdpartymirrors.extend(v) + for v in v: + if not v.endswith("/"): + v += "/" + thirdpartymirrors.append(v) class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): """ @@ -1056,9 +1225,16 @@ except FileNotFound: # disable for non-gentoo repoman users who may not have herds. herd_base = None -for x in scanlist: +effective_scanlist = scanlist +if options.if_modified == "y": + effective_scanlist = sorted(vcs_files_to_cps( + chain(mychanged, mynew, myremoved))) + +for x in effective_scanlist: #ebuilds and digests added to cvs respectively. logging.info("checking package %s" % x) + # save memory by discarding xmatch caches from previous package(s) + arch_xmatch_caches.clear() eadded=[] catdir,pkgdir=x.split("/") checkdir=repodir+"/"+x @@ -1071,8 +1247,7 @@ for x in scanlist: generated_manifest = False if options.mode == "manifest" or \ - (options.mode != 'manifest-check' and \ - 'digest' in repoman_settings.features) or \ + (options.mode != 'manifest-check' and options.digest == 'y') or \ options.mode in ('commit', 'fix') and not options.pretend: auto_assumed = set() fetchlist_dict = portage.FetchlistDict(checkdir, @@ -1081,7 +1256,9 @@ for x in scanlist: portage._doebuild_manifest_exempt_depend += 1 try: distdir = repoman_settings['DISTDIR'] - mf = portage.manifest.Manifest(checkdir, distdir, + mf = repoman_settings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(checkdir))) + mf = mf.load_manifest(checkdir, distdir, fetchlist_dict=fetchlist_dict) mf.create(requiredDistfiles=None, assumeDistHashesAlways=True) @@ -1175,17 +1352,6 @@ for x in scanlist: pkgs[pf] = Package(cpv=cpv, metadata=myaux, root_config=root_config, type_name="ebuild") - # Sort ebuilds in ascending order for the KEYWORDS.dropped check. - pkgsplits = {} - for i in range(len(ebuildlist)): - ebuild_split = portage.pkgsplit(ebuildlist[i]) - pkgsplits[ebuild_split] = ebuildlist[i] - ebuildlist[i] = ebuild_split - ebuildlist.sort(key=cmp_sort_key(portage.pkgcmp)) - for i in range(len(ebuildlist)): - ebuildlist[i] = pkgsplits[ebuildlist[i]] - del pkgsplits - slot_keywords = {} if len(pkgs) != len(ebuildlist): @@ -1197,20 +1363,34 @@ for x in scanlist: can_force = False continue + # Sort ebuilds in ascending order for the KEYWORDS.dropped check. + ebuildlist = sorted(pkgs.values()) + ebuildlist = [pkg.pf for pkg in ebuildlist] + for y in checkdirlist: m = disallowed_filename_chars_re.search(y.strip(os.sep)) if m is not None: + y_relative = os.path.join(checkdir_relative, y) + if vcs is not None and not vcs_new_changed(y_relative): + # If the file isn't in the VCS new or changed set, then + # assume that it's an irrelevant temporary file (Manifest + # entries are not generated for file names containing + # prohibited characters). See bug #406877. + m = None + if m is not None: stats["file.name"] += 1 fails["file.name"].append("%s/%s: char '%s'" % \ (checkdir, y, m.group(0))) if not (y in ("ChangeLog", "metadata.xml") or y.endswith(".ebuild")): continue + f = None try: line = 1 - for l in io.open(_unicode_encode(os.path.join(checkdir, y), + f = io.open(_unicode_encode(os.path.join(checkdir, y), encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content']): + mode='r', encoding=_encodings['repo.content']) + for l in f: line +=1 except UnicodeDecodeError as ue: stats["file.UTF8"] += 1 @@ -1220,6 +1400,9 @@ for x in scanlist: if l2 != 0: s = s[s.rfind("\n") + 1:] fails["file.UTF8"].append("%s/%s: line %i, just after: '%s'" % (checkdir, y, line, s)) + finally: + if f is not None: + f.close() if vcs in ("git", "hg") and check_ebuild_notadded: if vcs == "git": @@ -1286,7 +1469,9 @@ for x in scanlist: raise continue - mf = Manifest(checkdir, repoman_settings["DISTDIR"]) + mf = repoman_settings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(checkdir))) + mf = mf.load_manifest(checkdir, repoman_settings["DISTDIR"]) mydigests=mf.getTypeDigests("DIST") fetchlist_dict = portage.FetchlistDict(checkdir, repoman_settings, portdb) @@ -1361,20 +1546,27 @@ for x in scanlist: m = disallowed_filename_chars_re.search( os.path.basename(y.rstrip(os.sep))) if m is not None: + y_relative = os.path.join(checkdir_relative, "files", y) + if vcs is not None and not vcs_new_changed(y_relative): + # If the file isn't in the VCS new or changed set, then + # assume that it's an irrelevant temporary file (Manifest + # entries are not generated for file names containing + # prohibited characters). See bug #406877. + m = None + if m is not None: stats["file.name"] += 1 fails["file.name"].append("%s/files/%s: char '%s'" % \ (checkdir, y, m.group(0))) if desktop_file_validate and desktop_pattern.match(y): - status, cmd_output = subprocess_getstatusoutput( - "'%s' '%s'" % (desktop_file_validate, full_path)) - if os.WIFEXITED(status) and os.WEXITSTATUS(status) != os.EX_OK: + cmd_output = validate_desktop_entry(full_path) + if cmd_output: # Note: in the future we may want to grab the # warnings in addition to the errors. We're # just doing errors now since we don't want # to generate too much noise at first. error_re = re.compile(r'.*\s*error:\s*(.*)') - for line in cmd_output.splitlines(): + for line in cmd_output: error_match = error_re.match(line) if error_match is None: continue @@ -1446,7 +1638,6 @@ for x in scanlist: changelog_path = os.path.join(checkdir_relative, "ChangeLog") changelog_modified = changelog_path in modified_changelogs - allmasked = True # detect unused local USE-descriptions used_useflags = set() @@ -1599,7 +1790,7 @@ for x in scanlist: Ebuilds that inherit a "Live" eclass (darcs,subversion,git,cvs,etc..) should not be allowed to be marked stable """ - if live_ebuild: + if live_ebuild and repo_config.name == "gentoo": bad_stable_keywords = [] for keyword in keywords: if not keyword.startswith("~") and \ @@ -1629,13 +1820,12 @@ for x in scanlist: arches.append([keyword, keyword[1:], [keyword[1:], keyword]]) else: arches.append([keyword, keyword, [keyword]]) - allmasked = False if not arches: # Use an empty profile for checking dependencies of # packages that have empty KEYWORDS. arches.append(['**', '**', ['**']]) - unknown_pkgs = {} + unknown_pkgs = set() baddepsyntax = False badlicsyntax = False badprovsyntax = False @@ -1672,11 +1862,13 @@ for x in scanlist: if atom == "||": continue - if not atom.blocker and \ - not portdb.cp_list(atom.cp) and \ + # Skip dependency.unknown for blockers, so that we + # don't encourage people to remove necessary blockers, + # as discussed in bug #382407. + if atom.blocker is None and \ + not portdb.xmatch("match-all", atom) and \ not atom.cp.startswith("virtual/"): - unknown_pkgs.setdefault(atom.cp, set()).add( - (mytype, atom.unevaluated_atom)) + unknown_pkgs.add((mytype, atom.unevaluated_atom)) is_blocker = atom.blocker @@ -1766,12 +1958,12 @@ for x in scanlist: #keyword checks myuse = myaux["KEYWORDS"].split() for mykey in myuse: - myskey=mykey[:] - if myskey[0]=="-": - myskey=myskey[1:] - if myskey[0]=="~": - myskey=myskey[1:] - if mykey!="-*": + if mykey not in ("-*", "*", "~*"): + myskey = mykey + if myskey[:1] == "-": + myskey = myskey[1:] + if myskey[:1] == "~": + myskey = myskey[1:] if myskey not in kwlist: stats["KEYWORDS.invalid"] += 1 fails["KEYWORDS.invalid"].append(x+"/"+y+".ebuild: %s" % mykey) @@ -1858,9 +2050,11 @@ for x in scanlist: dep_settings = portage.config( config_profile_path=prof.abs_path, config_incrementals=repoman_incrementals, + config_root=config_root, local_config=False, _unmatched_removal=options.unmatched_removal, env=env) + dep_settings.categories = repoman_settings.categories if options.without_mask: dep_settings._mask_manager = \ copy.deepcopy(dep_settings._mask_manager) @@ -1876,7 +2070,7 @@ for x in scanlist: xcache.update(shared_xmatch_caches) arch_xmatch_caches[xmatch_cache_key] = xcache - trees["/"]["porttree"].settings = dep_settings + trees[root]["porttree"].settings = dep_settings portdb.settings = dep_settings portdb.xcache = xcache # for package.use.mask support inside dep_check @@ -1887,7 +2081,7 @@ for x in scanlist: if not baddepsyntax: ismasked = not ebuild_archs or \ - pkg.cpv not in portdb.xmatch("list-visible", pkg.cp) + pkg.cpv not in portdb.xmatch("match-visible", pkg.cp) if ismasked: if not have_pmasked: have_pmasked = bool(dep_settings._getMaskAtom( @@ -1921,12 +2115,16 @@ for x in scanlist: if success: if atoms: + + # Don't bother with dependency.unknown for + # cases in which *DEPEND.bad is triggered. for atom in atoms: + # dep_check returns all blockers and they + # aren't counted for *DEPEND.bad, so we + # ignore them here. if not atom.blocker: - # Don't bother with dependency.unknown - # for cases in which *DEPEND.bad is - # triggered. - unknown_pkgs.pop(atom.cp, None) + unknown_pkgs.discard( + (mytype, atom.unevaluated_atom)) if not prof.sub_path: # old-style virtuals currently aren't @@ -1957,25 +2155,14 @@ for x in scanlist: prof, repr(atoms))) if not baddepsyntax and unknown_pkgs: - all_unknown = set() - all_unknown.update(*unknown_pkgs.values()) type_map = {} - for mytype, atom in all_unknown: + for mytype, atom in unknown_pkgs: type_map.setdefault(mytype, set()).add(atom) for mytype, atoms in type_map.items(): stats["dependency.unknown"] += 1 fails["dependency.unknown"].append("%s: %s: %s" % (relative_path, mytype, ", ".join(sorted(atoms)))) - # Check for 'all unstable' or 'all masked' -- ACCEPT_KEYWORDS is stripped - # XXX -- Needs to be implemented in dep code. Can't determine ~arch nicely. - #if not portage.portdb.xmatch("bestmatch-visible",x): - # stats["ebuild.nostable"]+=1 - # fails["ebuild.nostable"].append(x) - if ebuildlist and allmasked and repolevel == 3: - stats["ebuild.allmasked"]+=1 - fails["ebuild.allmasked"].append(x) - # check if there are unused local USE-descriptions in metadata.xml # (unless there are any invalids, to avoid noise) if allvalid: @@ -1985,6 +2172,9 @@ for x in scanlist: "%s/metadata.xml: unused local USE-description: '%s'" % \ (x, myflag)) +if options.if_modified == "y" and len(effective_scanlist) < 1: + logging.warn("--if-modified is enabled, but no modified packages were found!") + if options.mode == "manifest": sys.exit(dofail) @@ -2099,7 +2289,8 @@ else: err("Error retrieving CVS tree; exiting.") if vcs == "svn": try: - svnstatus=os.popen("svn status --no-ignore").readlines() + with os.popen("svn status --no-ignore") as f: + svnstatus = f.readlines() myunadded = [ "./"+elem.rstrip().split()[1] for elem in svnstatus if elem.startswith("?") or elem.startswith("I") ] except SystemExit as e: raise # TODO propagate this @@ -2112,20 +2303,23 @@ else: myf.close() if vcs == "bzr": try: - bzrstatus=os.popen("bzr status -S .").readlines() + with os.popen("bzr status -S .") as f: + bzrstatus = f.readlines() myunadded = [ "./"+elem.rstrip().split()[1].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("?") or elem[0:2] == " D" ] except SystemExit as e: raise # TODO propagate this except: err("Error retrieving bzr info; exiting.") if vcs == "hg": - myunadded = os.popen("hg status --no-status --unknown .").readlines() + with os.popen("hg status --no-status --unknown .") as f: + myunadded = f.readlines() myunadded = ["./" + elem.rstrip() for elem in myunadded] # Mercurial doesn't handle manually deleted files as removed from # the repository, so the user need to remove them before commit, # using "hg remove [FILES]" - mydeleted = os.popen("hg status --no-status --deleted .").readlines() + with os.popen("hg status --no-status --deleted .") as f: + mydeleted = f.readlines() mydeleted = ["./" + elem.rstrip() for elem in mydeleted] @@ -2141,36 +2335,6 @@ else: myautoadd+=[myunadded[x]] del myunadded[x] - if myautoadd: - print(">>> Auto-Adding missing Manifest(s)...") - if options.pretend: - if vcs == "cvs": - print("(cvs add "+" ".join(myautoadd)+")") - elif vcs == "svn": - print("(svn add "+" ".join(myautoadd)+")") - elif vcs == "git": - print("(git add "+" ".join(myautoadd)+")") - elif vcs == "bzr": - print("(bzr add "+" ".join(myautoadd)+")") - elif vcs == "hg": - print("(hg add "+" ".join(myautoadd)+")") - retval=0 - else: - if vcs == "cvs": - retval=os.system("cvs add "+" ".join(myautoadd)) - elif vcs == "svn": - retval=os.system("svn add "+" ".join(myautoadd)) - elif vcs == "git": - retval=os.system("git add "+" ".join(myautoadd)) - elif vcs == "bzr": - retval=os.system("bzr add "+" ".join(myautoadd)) - elif vcs == "hg": - retval=os.system("hg add "+" ".join(myautoadd)) - if retval: - writemsg_level("!!! Exiting on %s (shell) error code: %s\n" % \ - (vcs, retval), level=logging.ERROR, noiselevel=-1) - sys.exit(retval) - if myunadded: print(red("!!! The following files are in your local tree but are not added to the master")) print(red("!!! tree. Please remove them from the local tree or add them to the master tree.")) @@ -2200,28 +2364,37 @@ else: if vcs == "svn": - svnstatus = os.popen("svn status").readlines() + with os.popen("svn status") as f: + svnstatus = f.readlines() mychanged = [ "./" + elem.split()[-1:][0] for elem in svnstatus if (elem[:1] in "MR" or elem[1:2] in "M")] mynew = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("A")] myremoved = [ "./" + elem.split()[-1:][0] for elem in svnstatus if elem.startswith("D")] # Subversion expands keywords specified in svn:keywords properties. - props = os.popen("svn propget -R svn:keywords").readlines() + with os.popen("svn propget -R svn:keywords") as f: + props = f.readlines() expansion = dict(("./" + prop.split(" - ")[0], prop.split(" - ")[1].split()) \ for prop in props if " - " in prop) elif vcs == "git": - mychanged = os.popen("git diff-index --name-only --relative --diff-filter=M HEAD").readlines() + with os.popen("git diff-index --name-only " + "--relative --diff-filter=M HEAD") as f: + mychanged = f.readlines() mychanged = ["./" + elem[:-1] for elem in mychanged] - mynew = os.popen("git diff-index --name-only --relative --diff-filter=A HEAD").readlines() + with os.popen("git diff-index --name-only " + "--relative --diff-filter=A HEAD") as f: + mynew = f.readlines() mynew = ["./" + elem[:-1] for elem in mynew] - myremoved = os.popen("git diff-index --name-only --relative --diff-filter=D HEAD").readlines() + with os.popen("git diff-index --name-only " + "--relative --diff-filter=D HEAD") as f: + myremoved = f.readlines() myremoved = ["./" + elem[:-1] for elem in myremoved] if vcs == "bzr": - bzrstatus = os.popen("bzr status -S .").readlines() + with os.popen("bzr status -S .") as f: + bzrstatus = f.readlines() mychanged = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and elem[1:2] == "M" ] mynew = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem and ( elem[1:2] in "NK" or elem[0:1] == "R" ) ] myremoved = [ "./" + elem.split()[-1:][0].split('/')[-1:][0] for elem in bzrstatus if elem.startswith("-") ] @@ -2229,11 +2402,16 @@ else: # Bazaar expands nothing. if vcs == "hg": - mychanged = os.popen("hg status --no-status --modified .").readlines() + with os.popen("hg status --no-status --modified .") as f: + mychanged = f.readlines() mychanged = ["./" + elem.rstrip() for elem in mychanged] - mynew = os.popen("hg status --no-status --added .").readlines() + + with os.popen("hg status --no-status --added .") as f: + mynew = f.readlines() mynew = ["./" + elem.rstrip() for elem in mynew] - myremoved = os.popen("hg status --no-status --removed .").readlines() + + with os.popen("hg status --no-status --removed .") as f: + myremoved = f.readlines() myremoved = ["./" + elem.rstrip() for elem in myremoved] if vcs: @@ -2253,20 +2431,143 @@ else: mymanifests.add(f) else: myupdates.add(f) - if vcs in ('git', 'hg'): - myupdates.difference_update(myremoved) + myupdates.difference_update(myremoved) myupdates = list(myupdates) mymanifests = list(mymanifests) myheaders = [] mydirty = [] + commitmessage = options.commitmsg + if options.commitmsgfile: + try: + f = io.open(_unicode_encode(options.commitmsgfile, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['content'], errors='replace') + commitmessage = f.read() + f.close() + del f + except (IOError, OSError) as e: + if e.errno == errno.ENOENT: + portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile) + else: + raise + # We've read the content so the file is no longer needed. + commitmessagefile = None + if not commitmessage or not commitmessage.strip(): + try: + editor = os.environ.get("EDITOR") + if editor and utilities.editor_is_executable(editor): + commitmessage = utilities.get_commit_message_with_editor( + editor, message=qa_output) + else: + commitmessage = utilities.get_commit_message_with_stdin() + except KeyboardInterrupt: + exithandler() + if not commitmessage or not commitmessage.strip(): + print("* no commit message? aborting commit.") + sys.exit(1) + commitmessage = commitmessage.rstrip() + changelog_msg = commitmessage + portage_version = getattr(portage, "VERSION", None) + if portage_version is None: + sys.stderr.write("Failed to insert portage version in message!\n") + sys.stderr.flush() + portage_version = "Unknown" + unameout = platform.system() + " " + if platform.system() in ["Darwin", "SunOS"]: + unameout += platform.processor() + else: + unameout += platform.machine() + commitmessage += "\n\n(Portage version: %s/%s/%s" % \ + (portage_version, vcs, unameout) + if options.force: + commitmessage += ", RepoMan options: --force" + commitmessage += ")" + + if options.echangelog in ('y', 'force'): + logging.info("checking for unmodified ChangeLog files") + committer_name = utilities.get_committer_name(env=repoman_settings) + for x in sorted(vcs_files_to_cps( + chain(myupdates, mymanifests, myremoved))): + catdir, pkgdir = x.split("/") + checkdir = repodir + "/" + x + checkdir_relative = "" + if repolevel < 3: + checkdir_relative = os.path.join(pkgdir, checkdir_relative) + if repolevel < 2: + checkdir_relative = os.path.join(catdir, checkdir_relative) + checkdir_relative = os.path.join(".", checkdir_relative) + + changelog_path = os.path.join(checkdir_relative, "ChangeLog") + changelog_modified = changelog_path in modified_changelogs + if changelog_modified and options.echangelog != 'force': + continue + + # get changes for this package + cdrlen = len(checkdir_relative) + clnew = [elem[cdrlen:] for elem in mynew if elem.startswith(checkdir_relative)] + clremoved = [elem[cdrlen:] for elem in myremoved if elem.startswith(checkdir_relative)] + clchanged = [elem[cdrlen:] for elem in mychanged if elem.startswith(checkdir_relative)] + + # Skip ChangeLog generation if only the Manifest was modified, + # as discussed in bug #398009. + nontrivial_cl_files = set() + nontrivial_cl_files.update(clnew, clremoved, clchanged) + nontrivial_cl_files.difference_update(['Manifest']) + if not nontrivial_cl_files and options.echangelog != 'force': + continue + + new_changelog = utilities.UpdateChangeLog(checkdir_relative, + committer_name, changelog_msg, + os.path.join(repodir, 'skel.ChangeLog'), + catdir, pkgdir, + new=clnew, removed=clremoved, changed=clchanged, + pretend=options.pretend) + if new_changelog is None: + writemsg_level("!!! Updating the ChangeLog failed\n", \ + level=logging.ERROR, noiselevel=-1) + sys.exit(1) + + # if the ChangeLog was just created, add it to vcs + if new_changelog: + myautoadd.append(changelog_path) + # myautoadd is appended to myupdates below + else: + myupdates.append(changelog_path) + + if myautoadd: + print(">>> Auto-Adding missing Manifest/ChangeLog file(s)...") + add_cmd = [vcs, "add"] + add_cmd += myautoadd + if options.pretend: + portage.writemsg_stdout("(%s)\n" % " ".join(add_cmd), + noiselevel=-1) + else: + if not (sys.hexversion >= 0x3000000 and sys.hexversion < 0x3020000): + # Python 3.1 produces the following TypeError if raw bytes are + # passed to subprocess.call(): + # File "/usr/lib/python3.1/subprocess.py", line 646, in __init__ + # errread, errwrite) + # File "/usr/lib/python3.1/subprocess.py", line 1157, in _execute_child + # raise child_exception + # TypeError: expected an object with the buffer interface + add_cmd = [_unicode_encode(arg) for arg in add_cmd] + retcode = subprocess.call(add_cmd) + if retcode != os.EX_OK: + logging.error( + "Exiting on %s error code: %s\n" % (vcs, retcode)) + sys.exit(retcode) + + myupdates += myautoadd + print("* %s files being committed..." % green(str(len(myupdates))), end=' ') + if vcs not in ('cvs', 'svn'): # With git, bzr and hg, there's never any keyword expansion, so # there's no need to regenerate manifests and all files will be # committed in one big commit at the end. print() - else: + elif not repo_config.thin_manifest: if vcs == 'cvs': headerstring = "'\$(Header|Id).*\$'" elif vcs == "svn": @@ -2315,60 +2616,15 @@ else: logging.info("myupdates: %s", myupdates) logging.info("myheaders: %s", myheaders) - commitmessage = options.commitmsg - if options.commitmsgfile: - try: - f = io.open(_unicode_encode(options.commitmsgfile, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace') - commitmessage = f.read() - f.close() - del f - except (IOError, OSError) as e: - if e.errno == errno.ENOENT: - portage.writemsg("!!! File Not Found: --commitmsgfile='%s'\n" % options.commitmsgfile) - else: - raise - # We've read the content so the file is no longer needed. - commitmessagefile = None - if not commitmessage or not commitmessage.strip(): - try: - editor = os.environ.get("EDITOR") - if editor and utilities.editor_is_executable(editor): - commitmessage = utilities.get_commit_message_with_editor( - editor, message=qa_output) - else: - commitmessage = utilities.get_commit_message_with_stdin() - except KeyboardInterrupt: - exithandler() - if not commitmessage or not commitmessage.strip(): - print("* no commit message? aborting commit.") - sys.exit(1) - commitmessage = commitmessage.rstrip() - portage_version = getattr(portage, "VERSION", None) - if portage_version is None: - sys.stderr.write("Failed to insert portage version in message!\n") - sys.stderr.flush() - portage_version = "Unknown" - unameout = platform.system() + " " - if platform.system() in ["Darwin", "SunOS"]: - unameout += platform.processor() - else: - unameout += platform.machine() - commitmessage += "\n\n(Portage version: %s/%s/%s" % \ - (portage_version, vcs, unameout) - if options.force: - commitmessage += ", RepoMan options: --force" - commitmessage += ")" - if options.ask and userquery('Commit changes?', True) != 'Yes': print("* aborting commit.") - sys.exit(1) + sys.exit(128 + signal.SIGINT) - if vcs in ('cvs', 'svn') and (myupdates or myremoved): + # Handle the case where committed files have keywords which + # will change and need a priming commit before the Manifest + # can be committed. + if (myupdates or myremoved) and myheaders: myfiles = myupdates + myremoved - if not myheaders and "sign" not in repoman_settings.features: - myfiles += mymanifests fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") mymsg = os.fdopen(fd, "wb") mymsg.write(_unicode_encode(commitmessage)) @@ -2469,121 +2725,25 @@ else: portage.util.write_atomic(x, b''.join(mylines), mode='wb') - manifest_commit_required = True - if vcs in ('cvs', 'svn') and (myupdates or myremoved): - myfiles = myupdates + myremoved - for x in range(len(myfiles)-1, -1, -1): - if myfiles[x].count("/") < 4-repolevel: - del myfiles[x] - mydone=[] - if repolevel==3: # In a package dir - repoman_settings["O"] = startdir - digestgen(mysettings=repoman_settings, myportdb=portdb) - elif repolevel==2: # In a category dir - for x in myfiles: - xs=x.split("/") - if len(xs) < 4-repolevel: - continue - if xs[0]==".": - xs=xs[1:] - if xs[0] in mydone: - continue - mydone.append(xs[0]) - repoman_settings["O"] = os.path.join(startdir, xs[0]) - if not os.path.isdir(repoman_settings["O"]): - continue - digestgen(mysettings=repoman_settings, myportdb=portdb) - elif repolevel==1: # repo-cvsroot - print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n") - for x in myfiles: - xs=x.split("/") - if len(xs) < 4-repolevel: - continue - if xs[0]==".": - xs=xs[1:] - if "/".join(xs[:2]) in mydone: - continue - mydone.append("/".join(xs[:2])) - repoman_settings["O"] = os.path.join(startdir, xs[0], xs[1]) - if not os.path.isdir(repoman_settings["O"]): - continue - digestgen(mysettings=repoman_settings, myportdb=portdb) - else: - print(red("I'm confused... I don't know where I am!")) - sys.exit(1) - - # Force an unsigned commit when more than one Manifest needs to be signed. - if repolevel < 3 and "sign" in repoman_settings.features: - - fd, commitmessagefile = tempfile.mkstemp(".repoman.msg") - mymsg = os.fdopen(fd, "wb") - mymsg.write(_unicode_encode(commitmessage)) - mymsg.write(b"\n (Unsigned Manifest commit)") - mymsg.close() + if repolevel == 1: + print(green("RepoMan sez:"), "\"You're rather crazy... " + "doing the entire repository.\"\n") - commit_cmd = [vcs] - commit_cmd.extend(vcs_global_opts) - commit_cmd.append("commit") - commit_cmd.extend(vcs_local_opts) - commit_cmd.extend(["-F", commitmessagefile]) - commit_cmd.extend(f.lstrip("./") for f in mymanifests) + if vcs in ('cvs', 'svn') and (myupdates or myremoved): - try: - if options.pretend: - print("(%s)" % (" ".join(commit_cmd),)) - else: - retval = spawn(commit_cmd, env=os.environ) - if retval: - writemsg_level(("!!! Exiting on %s (shell) " + \ - "error code: %s\n") % (vcs, retval), - level=logging.ERROR, noiselevel=-1) - sys.exit(retval) - finally: - try: - os.unlink(commitmessagefile) - except OSError: - pass - manifest_commit_required = False + for x in sorted(vcs_files_to_cps( + chain(myupdates, myremoved, mymanifests))): + repoman_settings["O"] = os.path.join(repodir, x) + digestgen(mysettings=repoman_settings, myportdb=portdb) signed = False - if "sign" in repoman_settings.features: + if sign_manifests: signed = True - myfiles = myupdates + myremoved + mymanifests try: - if repolevel==3: # In a package dir - repoman_settings["O"] = "." + for x in sorted(vcs_files_to_cps( + chain(myupdates, myremoved, mymanifests))): + repoman_settings["O"] = os.path.join(repodir, x) gpgsign(os.path.join(repoman_settings["O"], "Manifest")) - elif repolevel==2: # In a category dir - mydone=[] - for x in myfiles: - xs=x.split("/") - if len(xs) < 4-repolevel: - continue - if xs[0]==".": - xs=xs[1:] - if xs[0] in mydone: - continue - mydone.append(xs[0]) - repoman_settings["O"] = os.path.join(".", xs[0]) - if not os.path.isdir(repoman_settings["O"]): - continue - gpgsign(os.path.join(repoman_settings["O"], "Manifest")) - elif repolevel==1: # repo-cvsroot - print(green("RepoMan sez:"), "\"You're rather crazy... doing the entire repository.\"\n") - mydone=[] - for x in myfiles: - xs=x.split("/") - if len(xs) < 4-repolevel: - continue - if xs[0]==".": - xs=xs[1:] - if "/".join(xs[:2]) in mydone: - continue - mydone.append("/".join(xs[:2])) - repoman_settings["O"] = os.path.join(".", xs[0], xs[1]) - if not os.path.isdir(repoman_settings["O"]): - continue - gpgsign(os.path.join(repoman_settings["O"], "Manifest")) except portage.exception.PortageException as e: portage.writemsg("!!! %s\n" % str(e)) portage.writemsg("!!! Disabled FEATURES='sign'\n") @@ -2610,10 +2770,13 @@ else: level=logging.ERROR, noiselevel=-1) sys.exit(retval) - if vcs in ['git', 'bzr', 'hg'] or manifest_commit_required or signed: + if True: myfiles = mymanifests[:] - if vcs in ['git', 'bzr', 'hg']: + # If there are no header (SVN/CVS keywords) changes in + # the files, this Manifest commit must include the + # other (yet uncommitted) files. + if not myheaders: myfiles += myupdates myfiles += myremoved myfiles.sort() @@ -2652,6 +2815,13 @@ else: else: retval = spawn(commit_cmd, env=os.environ) if retval != os.EX_OK: + + if repo_config.sign_commit and vcs == 'git' and \ + not git_supports_gpg_sign(): + # Inform user that newer git is needed (bug #403323). + logging.error( + "Git >=1.7.9 is required for signed commits!") + writemsg_level(("!!! Exiting on %s (shell) " + \ "error code: %s\n") % (vcs, retval), level=logging.ERROR, noiselevel=-1) diff --git a/portage_with_autodep/bin/save-ebuild-env.sh b/portage_with_autodep/bin/save-ebuild-env.sh new file mode 100755 index 0000000..23e7aaa --- /dev/null +++ b/portage_with_autodep/bin/save-ebuild-env.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# Copyright 1999-2011 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# @FUNCTION: save_ebuild_env +# @DESCRIPTION: +# echo the current environment to stdout, filtering out redundant info. +# +# --exclude-init-phases causes pkg_nofetch and src_* phase functions to +# be excluded from the output. These function are not needed for installation +# or removal of the packages, and can therefore be safely excluded. +# +save_ebuild_env() { + ( + if has --exclude-init-phases $* ; then + unset S _E_DOCDESTTREE_ _E_EXEDESTTREE_ \ + PORTAGE_DOCOMPRESS PORTAGE_DOCOMPRESS_SKIP + if [[ -n $PYTHONPATH && + ${PYTHONPATH%%:*} -ef $PORTAGE_PYM_PATH ]] ; then + if [[ $PYTHONPATH == *:* ]] ; then + export PYTHONPATH=${PYTHONPATH#*:} + else + unset PYTHONPATH + fi + fi + fi + + # misc variables inherited from the calling environment + unset COLORTERM DISPLAY EDITOR LESS LESSOPEN LOGNAME LS_COLORS PAGER \ + TERM TERMCAP USER ftp_proxy http_proxy no_proxy + + # other variables inherited from the calling environment + unset CVS_RSH ECHANGELOG_USER GPG_AGENT_INFO \ + SSH_AGENT_PID SSH_AUTH_SOCK STY WINDOW XAUTHORITY + + # CCACHE and DISTCC config + unset ${!CCACHE_*} ${!DISTCC_*} + + # There's no need to bloat environment.bz2 with internally defined + # functions and variables, so filter them out if possible. + + for x in pkg_setup pkg_nofetch src_unpack src_prepare src_configure \ + src_compile src_test src_install pkg_preinst pkg_postinst \ + pkg_prerm pkg_postrm ; do + unset -f default_$x _eapi{0,1,2,3,4}_$x + done + unset x + + unset -f assert assert_sigpipe_ok dump_trace die diefunc \ + quiet_mode vecho elog_base eqawarn elog \ + esyslog einfo einfon ewarn eerror ebegin _eend eend KV_major \ + KV_minor KV_micro KV_to_int get_KV unset_colors set_colors has \ + has_phase_defined_up_to \ + hasv hasq qa_source qa_call \ + addread addwrite adddeny addpredict _sb_append_var \ + use usev useq has_version portageq \ + best_version use_with use_enable register_die_hook \ + keepdir unpack strip_duplicate_slashes econf einstall \ + dyn_setup dyn_unpack dyn_clean into insinto exeinto docinto \ + insopts diropts exeopts libopts docompress \ + abort_handler abort_prepare abort_configure abort_compile \ + abort_test abort_install dyn_prepare dyn_configure \ + dyn_compile dyn_test dyn_install \ + dyn_preinst dyn_pretend dyn_help debug-print debug-print-function \ + debug-print-section helpers_die inherit EXPORT_FUNCTIONS \ + nonfatal register_success_hook remove_path_entry \ + save_ebuild_env filter_readonly_variables preprocess_ebuild_env \ + set_unless_changed unset_unless_changed source_all_bashrcs \ + ebuild_main ebuild_phase ebuild_phase_with_hooks \ + _ebuild_arg_to_phase _ebuild_phase_funcs default \ + _hasg _hasgq _unpack_tar \ + ${QA_INTERCEPTORS} + + # portage config variables and variables set directly by portage + unset ACCEPT_LICENSE BAD BRACKET BUILD_PREFIX COLS \ + DISTCC_DIR DISTDIR DOC_SYMLINKS_DIR \ + EBUILD_FORCE_TEST EBUILD_MASTER_PID \ + ECLASS_DEPTH ENDCOL FAKEROOTKEY \ + GOOD HILITE HOME \ + LAST_E_CMD LAST_E_LEN LD_PRELOAD MISC_FUNCTIONS_ARGS MOPREFIX \ + NOCOLOR NORMAL PKGDIR PKGUSE PKG_LOGDIR PKG_TMPDIR \ + PORTAGE_BASHRCS_SOURCED PORTAGE_COMPRESS_EXCLUDE_SUFFIXES \ + PORTAGE_NONFATAL PORTAGE_QUIET \ + PORTAGE_SANDBOX_DENY PORTAGE_SANDBOX_PREDICT \ + PORTAGE_SANDBOX_READ PORTAGE_SANDBOX_WRITE PREROOTPATH \ + QA_INTERCEPTORS \ + RC_DEFAULT_INDENT RC_DOT_PATTERN RC_ENDCOL RC_INDENTATION \ + ROOT ROOTPATH RPMDIR TEMP TMP TMPDIR USE_EXPAND \ + WARN XARGS _RC_GET_KV_CACHE + + # user config variables + unset DOC_SYMLINKS_DIR INSTALL_MASK PKG_INSTALL_MASK + + declare -p + declare -fp + if [[ ${BASH_VERSINFO[0]} == 3 ]]; then + export + fi + ) +} diff --git a/portage_with_autodep/bin/xattr-helper.py b/portage_with_autodep/bin/xattr-helper.py new file mode 100755 index 0000000..6d99521 --- /dev/null +++ b/portage_with_autodep/bin/xattr-helper.py @@ -0,0 +1,190 @@ +#!/usr/bin/python +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""Dump and restore extended attributes. + +We use formats like that used by getfattr --dump. This is meant for shell +helpers to save/restore. If you're looking for a python/portage API, see +portage.util.movefile._copyxattr instead. + +https://en.wikipedia.org/wiki/Extended_file_attributes +""" + +import array +import os +import re +import sys + +from portage.util._argparse import ArgumentParser + +if hasattr(os, "getxattr"): + + class xattr(object): + get = os.getxattr + set = os.setxattr + list = os.listxattr + +else: + import xattr + + +_UNQUOTE_RE = re.compile(br'\\[0-7]{3}') +_FS_ENCODING = sys.getfilesystemencoding() + + +if sys.hexversion < 0x3000000: + + def octal_quote_byte(b): + return b'\\%03o' % ord(b) + + def unicode_encode(s): + if isinstance(s, unicode): + s = s.encode(_FS_ENCODING) + return s +else: + + def octal_quote_byte(b): + return ('\\%03o' % ord(b)).encode('ascii') + + def unicode_encode(s): + if isinstance(s, str): + s = s.encode(_FS_ENCODING) + return s + + +def quote(s, quote_chars): + """Convert all |quote_chars| in |s| to escape sequences + + This is normally used to escape any embedded quotation marks. + """ + quote_re = re.compile(b'[' + quote_chars + b']') + result = [] + pos = 0 + s_len = len(s) + + while pos < s_len: + m = quote_re.search(s, pos=pos) + if m is None: + result.append(s[pos:]) + pos = s_len + else: + start = m.start() + result.append(s[pos:start]) + result.append(octal_quote_byte(s[start:start+1])) + pos = start + 1 + + return b''.join(result) + + +def unquote(s): + """Process all escape sequences in |s|""" + result = [] + pos = 0 + s_len = len(s) + + while pos < s_len: + m = _UNQUOTE_RE.search(s, pos=pos) + if m is None: + result.append(s[pos:]) + pos = s_len + else: + start = m.start() + result.append(s[pos:start]) + pos = start + 4 + a = array.array('B') + a.append(int(s[start + 1:pos], 8)) + try: + # Python >= 3.2 + result.append(a.tobytes()) + except AttributeError: + result.append(a.tostring()) + + return b''.join(result) + + +def dump_xattrs(pathnames, file_out): + """Dump the xattr data for |pathnames| to |file_out|""" + # NOTE: Always quote backslashes, in order to ensure that they are + # not interpreted as quotes when they are processed by unquote. + quote_chars = b'\n\r\\\\' + + for pathname in pathnames: + attrs = xattr.list(pathname) + if not attrs: + continue + + file_out.write(b'# file: %s\n' % quote(pathname, quote_chars)) + for attr in attrs: + attr = unicode_encode(attr) + value = xattr.get(pathname, attr) + file_out.write(b'%s="%s"\n' % ( + quote(attr, b'=' + quote_chars), + quote(value, b'\0"' + quote_chars))) + + +def restore_xattrs(file_in): + """Read |file_in| and restore xattrs content from it + + This expects textual data in the format written by dump_xattrs. + """ + pathname = None + for i, line in enumerate(file_in): + if line.startswith(b'# file: '): + pathname = unquote(line.rstrip(b'\n')[8:]) + else: + parts = line.split(b'=', 1) + if len(parts) == 2: + if pathname is None: + raise ValueError('line %d: missing pathname' % (i + 1,)) + attr = unquote(parts[0]) + # strip trailing newline and quotes + value = unquote(parts[1].rstrip(b'\n')[1:-1]) + xattr.set(pathname, attr, value) + elif line.strip(): + raise ValueError('line %d: malformed entry' % (i + 1,)) + + +def main(argv): + + parser = ArgumentParser(description=__doc__) + parser.add_argument('paths', nargs='*', default=[]) + + actions = parser.add_argument_group('Actions') + actions.add_argument('--dump', + action='store_true', + help='Dump the values of all extended ' + 'attributes associated with null-separated' + ' paths read from stdin.') + actions.add_argument('--restore', + action='store_true', + help='Restore extended attributes using' + ' a dump read from stdin.') + + options = parser.parse_args(argv) + + if sys.hexversion >= 0x3000000: + file_in = sys.stdin.buffer.raw + else: + file_in = sys.stdin + if not options.paths: + options.paths += [x for x in file_in.read().split(b'\0') if x] + + if options.dump: + if sys.hexversion >= 0x3000000: + file_out = sys.stdout.buffer + else: + file_out = sys.stdout + dump_xattrs(options.paths, file_out) + + elif options.restore: + restore_xattrs(file_in) + + else: + parser.error('missing action!') + + return os.EX_OK + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/portage_with_autodep/bin/xpak-helper.py b/portage_with_autodep/bin/xpak-helper.py index 4766d99..ef74920 100755 --- a/portage_with_autodep/bin/xpak-helper.py +++ b/portage_with_autodep/bin/xpak-helper.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2009 Gentoo Foundation +# Copyright 2009-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import optparse @@ -36,7 +36,7 @@ def command_recompose(args): def main(argv): - if argv and sys.hexversion < 0x3000000 and not isinstance(argv[0], unicode): + if argv and isinstance(argv[0], bytes): for i, x in enumerate(argv): argv[i] = portage._unicode_decode(x, errors='strict') diff --git a/portage_with_autodep/integration_with_portage.patch b/portage_with_autodep/integration_with_portage.patch new file mode 100644 index 0000000..794c156 --- /dev/null +++ b/portage_with_autodep/integration_with_portage.patch @@ -0,0 +1,920 @@ +diff -urN /usr/lib/portage/pym/_emerge/EbuildBuild.py ./pym/_emerge/EbuildBuild.py +--- /usr/lib/portage/pym/_emerge/EbuildBuild.py 2012-05-28 16:20:40.737712559 +0600 ++++ ./pym/_emerge/EbuildBuild.py 2012-06-01 21:37:22.799844102 +0600 +@@ -9,6 +9,8 @@ + from _emerge.EbuildMerge import EbuildMerge + from _emerge.EbuildFetchonly import EbuildFetchonly + from _emerge.EbuildBuildDir import EbuildBuildDir ++from _emerge.EventsAnalyser import EventsAnalyser, FilterProcGenerator ++from _emerge.EventsLogger import EventsLogger + from _emerge.MiscFunctionsProcess import MiscFunctionsProcess + from portage.util import writemsg + import portage +@@ -21,7 +23,7 @@ + class EbuildBuild(CompositeTask): + + __slots__ = ("args_set", "config_pool", "find_blockers", +- "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", ++ "ldpath_mtimes", "logger", "logserver", "opts", "pkg", "pkg_count", + "prefetcher", "settings", "world_atom") + \ + ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") + +@@ -251,8 +253,54 @@ + + build = EbuildExecuter(background=self.background, pkg=pkg, + scheduler=scheduler, settings=settings) ++ ++ build.addStartListener(self._build_start) ++ build.addExitListener(self._build_stop) ++ + self._start_task(build, self._build_exit) + ++ def _build_start(self,phase): ++ if "depcheck" in self.settings["FEATURES"] or \ ++ "depcheckstrict" in self.settings["FEATURES"]: ++ # Lets start a log listening server ++ temp_path=self.settings.get("T",self.settings["PORTAGE_TMPDIR"]) ++ ++ if "depcheckstrict" not in self.settings["FEATURES"]: ++ # use default filter_proc ++ self.logserver=EventsLogger(socket_dir=temp_path) ++ else: ++ portage.util.writemsg("Getting list of allowed files..." + \ ++ "This may take some time\n") ++ filter_gen=FilterProcGenerator(self.pkg.cpv, self.settings) ++ filter_proc=filter_gen.get_filter_proc() ++ self.logserver=EventsLogger(socket_dir=temp_path, ++ filter_proc=filter_proc) ++ ++ self.logserver.start() ++ ++ # Copy socket path to LOG_SOCKET environment variable ++ env=self.settings.configdict["pkg"] ++ env['LOG_SOCKET'] = self.logserver.socket_name ++ ++ #import pdb; pdb.set_trace() ++ ++ def _build_stop(self,phase): ++ if "depcheck" in self.settings["FEATURES"] or \ ++ "depcheckstrict" in self.settings["FEATURES"]: ++ # Delete LOG_SOCKET from environment ++ env=self.settings.configdict["pkg"] ++ if 'LOG_SOCKET' in env: ++ del env['LOG_SOCKET'] ++ ++ events=self.logserver.stop() ++ self.logserver=None ++ analyser=EventsAnalyser(self.pkg.cpv, events, self.settings) ++ analyser.display() # show the analyse ++ ++ #import pdb; pdb.set_trace() ++ ++ ++ + def _fetch_failed(self): + # We only call the pkg_nofetch phase if either RESTRICT=fetch + # is set or the package has explicitly overridden the default +diff -urN /usr/lib/portage/pym/_emerge/EbuildPhase.py ./pym/_emerge/EbuildPhase.py +--- /usr/lib/portage/pym/_emerge/EbuildPhase.py 2012-05-28 16:20:40.738712559 +0600 ++++ ./pym/_emerge/EbuildPhase.py 2012-06-01 21:38:11.935842036 +0600 +@@ -34,7 +34,8 @@ + + # FEATURES displayed prior to setup phase + _features_display = ( +- "ccache", "compressdebug", "distcc", "distcc-pump", "fakeroot", ++ "ccache", "compressdebug", "depcheck", "depcheckstrict", ++ "distcc", "distcc-pump", "fakeroot", + "installsources", "keeptemp", "keepwork", "nostrip", + "preserve-libs", "sandbox", "selinux", "sesandbox", + "splitdebug", "suidctl", "test", "userpriv", +diff -urN /usr/lib/portage/pym/_emerge/EbuildPhase.py.rej ./pym/_emerge/EbuildPhase.py.rej +--- /usr/lib/portage/pym/_emerge/EbuildPhase.py.rej 1970-01-01 05:00:00.000000000 +0500 ++++ ./pym/_emerge/EbuildPhase.py.rej 2012-06-01 21:37:22.800844102 +0600 +@@ -0,0 +1,12 @@ ++--- pym/_emerge/EbuildPhase.py +++++ pym/_emerge/EbuildPhase.py ++@@ -33,7 +33,8 @@ ++ ("_ebuild_lock",) ++ ++ # FEATURES displayed prior to setup phase ++- _features_display = ("ccache", "distcc", "distcc-pump", "fakeroot", +++ _features_display = ("ccache", "depcheck", "depcheckstrict" "distcc", +++ "distcc-pump", "fakeroot", ++ "installsources", "keeptemp", "keepwork", "nostrip", ++ "preserve-libs", "sandbox", "selinux", "sesandbox", ++ "splitdebug", "suidctl", "test", "userpriv", +diff -urN /usr/lib/portage/pym/_emerge/EventsAnalyser.py ./pym/_emerge/EventsAnalyser.py +--- /usr/lib/portage/pym/_emerge/EventsAnalyser.py 1970-01-01 05:00:00.000000000 +0500 ++++ ./pym/_emerge/EventsAnalyser.py 2012-06-01 21:37:22.802844102 +0600 +@@ -0,0 +1,511 @@ ++# Distributed under the terms of the GNU General Public License v2 ++ ++import portage ++from portage.dbapi._expand_new_virt import expand_new_virt ++from portage import os ++ ++import subprocess ++import re ++ ++class PortageUtils: ++ """ class for accessing the portage api """ ++ def __init__(self, settings): ++ """ test """ ++ self.settings=settings ++ self.vartree=portage.vartree(settings=settings) ++ self.vardbapi=portage.vardbapi(settings=settings, vartree=self.vartree) ++ self.portdbapi=portage.portdbapi(mysettings=settings) ++ self.metadata_keys = [k for k in portage.auxdbkeys if not k.startswith("UNUSED_")] ++ self.use=self.settings["USE"] ++ ++ def get_best_visible_pkg(self,pkg): ++ """ ++ Gets best candidate on installing. Returns empty string if no found ++ ++ :param pkg: package name ++ ++ """ ++ try: ++ return self.portdbapi.xmatch("bestmatch-visible", pkg) ++ except: ++ return '' ++ ++ # non-recursive dependency getter ++ def get_dep(self,pkg,dep_type=["RDEPEND","DEPEND"]): ++ """ ++ Gets current dependencies of a package. Looks in portage db ++ ++ :param pkg: name of package ++ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ++ ["RDEPEND", "DEPEND"] ++ :returns: **set** of packages names ++ """ ++ ret=set() ++ ++ pkg = self.get_best_visible_pkg(pkg) ++ if not pkg: ++ return ret ++ ++ # we found the best visible match in common tree ++ ++ ++ metadata = dict(zip(self.metadata_keys, ++ self.portdbapi.aux_get(pkg, self.metadata_keys))) ++ dep_str = " ".join(metadata[k] for k in dep_type) ++ ++ # the IUSE default are very important for us ++ iuse_defaults=[ ++ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] ++ ++ use=self.use.split() ++ ++ for u in iuse_defaults: ++ if u not in use: ++ use.append(u) ++ ++ success, atoms = portage.dep_check(dep_str, None, self.settings, ++ myuse=use, myroot=self.settings["ROOT"], ++ trees={self.settings["ROOT"]:{"vartree":self.vartree, "porttree": self.vartree}}) ++ if not success: ++ return ret ++ ++ for atom in atoms: ++ atomname = self.vartree.dep_bestmatch(atom) ++ ++ if not atomname: ++ continue ++ ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.add(pkg) ++ ++ return ret ++ ++ # recursive dependency getter ++ def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]): ++ """ ++ Gets current dependencies of a package on any depth ++ All dependencies **must** be installed ++ ++ :param pkg: name of package ++ :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ++ ["RDEPEND", "DEPEND"] ++ :returns: **set** of packages names ++ """ ++ ret=set() ++ ++ ++ # get porttree dependencies on the first package ++ ++ pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) ++ if not pkg: ++ return ret ++ ++ known_packages=set() ++ unknown_packages=self.get_dep(pkg,dep_type) ++ ret=ret.union(unknown_packages) ++ ++ while unknown_packages: ++ p=unknown_packages.pop() ++ if p in known_packages: ++ continue ++ known_packages.add(p) ++ ++ metadata = dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, self.metadata_keys))) ++ ++ dep_str = " ".join(metadata[k] for k in dep_type) ++ ++ # the IUSE default are very important for us ++ iuse_defaults=[ ++ u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] ++ ++ use=self.use.split() ++ ++ for u in iuse_defaults: ++ if u not in use: ++ use.append(u) ++ ++ success, atoms = portage.dep_check(dep_str, None, self.settings, ++ myuse=use, myroot=self.settings["ROOT"], ++ trees={self.settings["ROOT"]:{"vartree":self.vartree,"porttree": self.vartree}}) ++ ++ if not success: ++ continue ++ ++ for atom in atoms: ++ atomname = self.vartree.dep_bestmatch(atom) ++ if not atomname: ++ continue ++ ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.add(pkg) ++ unknown_packages.add(pkg) ++ return ret ++ ++ def get_deps_for_package_building(self, pkg): ++ """ ++ returns buildtime dependencies of current package and ++ all runtime dependencies of that buildtime dependencies ++ """ ++ buildtime_deps=self.get_dep(pkg, ["DEPEND"]) ++ runtime_deps=set() ++ for dep in buildtime_deps: ++ runtime_deps=runtime_deps.union(self.get_deps(dep,["RDEPEND"])) ++ ++ ret=buildtime_deps.union(runtime_deps) ++ return ret ++ ++ def get_system_packages_list(self): ++ """ ++ returns all packages from system set. They are always implicit dependencies ++ ++ :returns: **list** of package names ++ """ ++ ret=[] ++ for atom in self.settings.packages: ++ for pre_pkg in self.vartree.dep_match(atom): ++ for unvirt_pkg in expand_new_virt(self.vardbapi,'='+pre_pkg): ++ for pkg in self.vartree.dep_match(unvirt_pkg): ++ ret.append(pkg) ++ return ret ++ ++ ++class GentoolkitUtils: ++ """ ++ Interface with qfile and qlist utils. They are much faster than ++ internals. ++ """ ++ ++ def getpackagesbyfiles(files): ++ """ ++ :param files: list of filenames ++ :returns: **dictionary** file->package, if file doesn't belong to any ++ package it not returned as key of this dictionary ++ """ ++ ret={} ++ listtocheck=[] ++ for f in files: ++ if os.path.isdir(f): ++ ret[f]="directory" ++ else: ++ listtocheck.append(f) ++ ++ try: ++ proc=subprocess.Popen(['qfile']+['--nocolor','--exact','','--from','-'], ++ stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ bufsize=4096) ++ ++ out,err=proc.communicate("\n".join(listtocheck).encode("utf8")) ++ ++ lines=out.decode("utf8").split("\n") ++ #print lines ++ line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$") ++ for line in lines: ++ if len(line)==0: ++ continue ++ match=line_re.match(line) ++ if match: ++ ret[match.group(2)]=match.group(1) ++ else: ++ portage.util.writemsg("Util qfile returned unparsable string: %s\n" % line) ++ ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ ++ ++ return ret ++ ++ def getfilesbypackages(packagenames): ++ """ ++ ++ :param packagename: name of package ++ :returns: **list** of files in package with name *packagename* ++ """ ++ ret=[] ++ try: ++ proc=subprocess.Popen(['qlist']+['--nocolor',"--obj"]+packagenames, ++ stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ bufsize=4096) ++ ++ out,err=proc.communicate() ++ ++ ret=out.decode("utf8").split("\n") ++ if ret==['']: ++ ret=[] ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ ++ return ret ++ ++ def get_all_packages_files(): ++ """ ++ Memory-hungry operation ++ ++ :returns: **set** of all files that belongs to package ++ """ ++ ret=[] ++ try: ++ proc=subprocess.Popen(['qlist']+['--all',"--obj"], ++ stdout=subprocess.PIPE,stderr=subprocess.PIPE, ++ bufsize=4096) ++ ++ out,err=proc.communicate() ++ ++ ret=out.decode("utf8").split("\n") ++ except OSError as e: ++ portage.util.writemsg("Error while launching qfile: %s\n" % e) ++ ++ return set(ret) ++ ++class FilterProcGenerator: ++ def __init__(self, pkgname, settings): ++ portageutils=PortageUtils(settings=settings) ++ ++ deps_all=portageutils.get_deps_for_package_building(pkgname) ++ deps_portage=portageutils.get_dep('portage',["RDEPEND"]) ++ ++ system_packages=portageutils.get_system_packages_list() ++ ++ allfiles=GentoolkitUtils.get_all_packages_files() ++ portage.util.writemsg("All files list recieved, waiting for " \ ++ "a list of allowed files\n") ++ ++ ++ allowedpkgs=system_packages+list(deps_portage)+list(deps_all) ++ ++ allowedfiles=GentoolkitUtils.getfilesbypackages(allowedpkgs) ++ #for pkg in allowedpkgs: ++ # allowedfiles+=GentoolkitUtils.getfilesbypackage(pkg) ++ ++ #import pdb; pdb.set_trace() ++ ++ # manually add all python interpreters to this list ++ allowedfiles+=GentoolkitUtils.getfilesbypackages(['python']) ++ allowedfiles=set(allowedfiles) ++ ++ deniedfiles=allfiles-allowedfiles ++ ++ def filter_proc(eventname,filename,stage): ++ if filename in deniedfiles: ++ return False ++ return True ++ ++ self.filter_proc=filter_proc ++ def get_filter_proc(self): ++ return self.filter_proc ++ ++class EventsAnalyser: ++ def __init__(self, pkgname, events, settings): ++ self.pkgname=pkgname ++ self.events=events ++ self.settings=settings ++ self.portageutils=PortageUtils(settings=settings) ++ ++ self.deps_all=self.portageutils.get_deps_for_package_building(pkgname) ++ self.deps_direct=self.portageutils.get_dep(pkgname,["DEPEND"]) ++ self.deps_portage=self.portageutils.get_dep('portage',["RDEPEND"]) ++ ++ self.system_packages=self.portageutils.get_system_packages_list() ++ # All analyse work is here ++ ++ # get unique filenames ++ filenames=set() ++ for stage in events: ++ succ_events=set(events[stage][0]) ++ fail_events=set(events[stage][1]) ++ filenames=filenames.union(succ_events) ++ filenames=filenames.union(fail_events) ++ filenames=list(filenames) ++ ++ file_to_package=GentoolkitUtils.getpackagesbyfiles(filenames) ++ # This part is completly unreadable. ++ # It converting one complex struct(returned by getfsevents) to another complex ++ # struct which good for generating output. ++ # ++ # Old struct is also used during output ++ ++ packagesinfo={} ++ ++ for stage in sorted(events): ++ succ_events=events[stage][0] ++ fail_events=events[stage][1] ++ ++ for filename in succ_events: ++ if filename in file_to_package: ++ package=file_to_package[filename] ++ else: ++ package="unknown" ++ ++ if not package in packagesinfo: ++ packagesinfo[package]={} ++ stageinfo=packagesinfo[package] ++ if not stage in stageinfo: ++ stageinfo[stage]={} ++ ++ filesinfo=stageinfo[stage] ++ if not filename in filesinfo: ++ filesinfo[filename]={"found":[],"notfound":[]} ++ filesinfo[filename]["found"]=succ_events[filename] ++ ++ for filename in fail_events: ++ if filename in file_to_package: ++ package=file_to_package[filename] ++ else: ++ package="unknown" ++ if not package in packagesinfo: ++ packagesinfo[package]={} ++ stageinfo=packagesinfo[package] ++ if not stage in stageinfo: ++ stageinfo[stage]={} ++ ++ filesinfo=stageinfo[stage] ++ if not filename in filesinfo: ++ filesinfo[filename]={"found":[],"notfound":[]} ++ filesinfo[filename]["notfound"]=fail_events[filename] ++ self.packagesinfo=packagesinfo ++ ++ def display(self): ++ portage.util.writemsg( ++ portage.output.colorize( ++ "WARN", "\nFile access report for %s:\n" % self.pkgname)) ++ ++ stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7, ++ "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13} ++ packagesinfo=self.packagesinfo ++ # print information grouped by package ++ for package in sorted(packagesinfo): ++ # not showing special directory package ++ if package=="directory": ++ continue ++ ++ if package=="unknown": ++ continue ++ ++ ++ is_pkg_in_dep=package in self.deps_all ++ is_pkg_in_portage_dep=package in self.deps_portage ++ is_pkg_in_system=package in self.system_packages ++ is_pkg_python="dev-lang/python" in package ++ ++ stages=[] ++ for stage in sorted(packagesinfo[package].keys(), key=stagesorder.get): ++ if stage!="unknown": ++ stages.append(stage) ++ ++ if len(stages)==0: ++ continue ++ ++ filenames={} ++ for stage in stages: ++ for filename in packagesinfo[package][stage]: ++ if len(packagesinfo[package][stage][filename]["found"])!=0: ++ was_readed,was_writed=packagesinfo[package][stage][filename]["found"] ++ if not filename in filenames: ++ filenames[filename]=['ok',was_readed,was_writed] ++ else: ++ status, old_was_readed, old_was_writed=filenames[filename] ++ filenames[filename]=[ ++ 'ok',old_was_readed | was_readed, old_was_writed | was_writed ++ ] ++ if len(packagesinfo[package][stage][filename]["notfound"])!=0: ++ was_notfound,was_blocked=packagesinfo[package][stage][filename]["notfound"] ++ if not filename in filenames: ++ filenames[filename]=['err',was_notfound,was_blocked] ++ else: ++ status, old_was_notfound, old_was_blocked=filenames[filename] ++ filenames[filename]=[ ++ 'err',old_was_notfound | was_notfound, old_was_blocked | was_blocked ++ ] ++ ++ ++ if is_pkg_in_dep: ++ portage.util.writemsg("[OK]") ++ elif is_pkg_in_system: ++ portage.util.writemsg("[SYSTEM]") ++ elif is_pkg_in_portage_dep: ++ portage.util.writemsg("[PORTAGE DEP]") ++ elif is_pkg_python: ++ portage.util.writemsg("[INTERPRETER]") ++ elif not self.is_package_useful(package,stages,filenames.keys()): ++ portage.util.writemsg("[LIKELY OK]") ++ else: ++ portage.util.writemsg(portage.output.colorize("BAD", "[NOT IN DEPS]")) ++ # show information about accessed files ++ ++ portage.util.writemsg(" %-40s: %s\n" % (package,stages)) ++ ++ # this is here for readability ++ action={ ++ ('ok',False,False):"accessed", ++ ('ok',True,False):"readed", ++ ('ok',False,True):"writed", ++ ('ok',True,True):"readed and writed", ++ ('err',False,False):"other error", ++ ('err',True,False):"not found", ++ ('err',False,True):"blocked", ++ ('err',True,True):"not found and blocked" ++ } ++ ++ filescounter=0 ++ ++ for filename in filenames: ++ event_info=tuple(filenames[filename]) ++ portage.util.writemsg(" %-56s %-21s\n" % (filename,action[event_info])) ++ filescounter+=1 ++ if filescounter>10: ++ portage.util.writemsg(" ... and %d more ...\n" % (len(filenames)-10)) ++ break ++ # ... and one more check. Making sure that direct build time ++ # dependencies were accessed ++ #import pdb; pdb.set_trace() ++ not_accessed_deps=set(self.deps_direct)-set(self.packagesinfo.keys()) ++ if not_accessed_deps: ++ portage.util.writemsg(portage.output.colorize("WARN", "!!! ")) ++ portage.util.writemsg("Warning! Some build time dependencies " + \ ++ "of packages were not accessed: " + \ ++ " ".join(not_accessed_deps) + "\n") ++ ++ def is_package_useful(self,pkg,stages,files): ++ """ some basic heuristics here to cut part of packages """ ++ ++ excluded_paths=set( ++ ['/etc/sandbox.d/'] ++ ) ++ ++ excluded_packages=set( ++ # autodep shows these two packages every time ++ ['net-zope/zope-fixers', 'net-zope/zope-interface'] ++ ) ++ ++ ++ def is_pkg_excluded(p): ++ for pkg in excluded_packages: ++ if p.startswith(pkg): # if package is excluded ++ return True ++ return False ++ ++ ++ def is_file_excluded(f): ++ for path in excluded_paths: ++ if f.startswith(path): # if path is excluded ++ return True ++ return False ++ ++ ++ if is_pkg_excluded(pkg): ++ return False ++ ++ for f in files: ++ if is_file_excluded(f): ++ continue ++ ++ # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 ++ if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith(".m4") or f.endswith(".pc")): ++ break ++ else: ++ return False # we get here if cycle ends not with break ++ ++ return True ++ ++ +\ No newline at end of file +diff -urN /usr/lib/portage/pym/_emerge/EventsLogger.py ./pym/_emerge/EventsLogger.py +--- /usr/lib/portage/pym/_emerge/EventsLogger.py 1970-01-01 05:00:00.000000000 +0500 ++++ ./pym/_emerge/EventsLogger.py 2012-06-01 21:37:22.803844102 +0600 +@@ -0,0 +1,180 @@ ++# Distributed under the terms of the GNU General Public License v2 ++ ++import io ++import sys ++import stat ++import socket ++import select ++import tempfile ++ ++import threading ++ ++from portage import os ++ ++class EventsLogger(threading.Thread): ++ def default_filter(eventname, filename, stage): ++ return True ++ ++ def __init__(self, socket_dir="/tmp/", filter_proc=default_filter): ++ threading.Thread.__init__(self) # init the Thread ++ ++ self.alive=False ++ ++ self.main_thread=threading.currentThread() ++ ++ self.socket_dir=socket_dir ++ self.filter_proc=filter_proc ++ ++ self.socket_name=None ++ self.socket_logger=None ++ ++ self.events={} ++ ++ try: ++ socket_dir_name = tempfile.mkdtemp(dir=self.socket_dir, ++ prefix="log_socket_") ++ ++ socket_name = os.path.join(socket_dir_name, 'socket') ++ ++ except OSError as e: ++ return ++ ++ self.socket_name=socket_name ++ ++ #print(self.socket_name) ++ ++ try: ++ socket_logger=socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) ++ socket_logger.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) ++ ++ socket_logger.bind(self.socket_name) ++ socket_logger.listen(64) ++ ++ except socket.error as e: ++ return ++ ++ self.socket_logger=socket_logger ++ ++ try: ++ # Allow connecting to socket for anyone ++ os.chmod(socket_dir_name, ++ stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR| ++ stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) ++ os.chmod(socket_name, ++ stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR| ++ stat.S_IROTH|stat.S_IWOTH|stat.S_IXOTH) ++ except OSError as e: ++ return ++ ++ def run(self): ++ """ Starts the log server """ ++ ++ self.alive=True ++ self.listen_thread=threading.currentThread() ++ clients={} ++ ++ epoll=select.epoll() ++ epoll.register(self.socket_logger.fileno(), select.EPOLLIN) ++ ++ while self.alive: ++ try: ++ sock_events = epoll.poll(3) ++ ++ for fileno, sock_event in sock_events: ++ if fileno == self.socket_logger.fileno(): ++ ret = self.socket_logger.accept() ++ if ret is None: ++ pass ++ else: ++ (client,addr)=ret ++ epoll.register(client.fileno(), select.EPOLLIN) ++ clients[client.fileno()]=client ++ elif sock_event & select.EPOLLIN: ++ s=clients[fileno] ++ record=s.recv(8192) ++ ++ if not record: # if connection was closed ++ epoll.unregister(fileno) ++ clients[fileno].close() ++ del clients[fileno] ++ continue ++ ++ #import pdb; pdb.set_trace() ++ try: ++ message=record.decode("utf8").split("\0") ++ except UnicodeDecodeError: ++ print("Bad message %s" % record) ++ continue ++ ++ # continue ++ ++ #print(message) ++ ++ try: ++ if message[4]=="ASKING": ++ if self.filter_proc(message[1],message[2],message[3]): ++ s.sendall(b"ALLOW\0") ++ else: ++ # TODO: log through portage infrastructure ++ #print("Blocking an access to %s" % message[2]) ++ s.sendall(b"DENY\0") ++ else: ++ eventname,filename,stage,result=message[1:5] ++ ++ if not stage in self.events: ++ self.events[stage]=[{},{}] ++ ++ hashofsucesses=self.events[stage][0] ++ hashoffailures=self.events[stage][1] ++ ++ if result=="DENIED": ++ print("Blocking an access to %s" % filename) ++ ++ if result=="OK": ++ if not filename in hashofsucesses: ++ hashofsucesses[filename]=[False,False] ++ ++ readed_or_writed=hashofsucesses[filename] ++ ++ if eventname=="read": ++ readed_or_writed[0]=True ++ elif eventname=="write": ++ readed_or_writed[1]=True ++ ++ elif result[0:3]=="ERR" or result=="DENIED": ++ if not filename in hashoffailures: ++ hashoffailures[filename]=[False,False] ++ notfound_or_blocked=hashoffailures[filename] ++ ++ if result=="ERR/2": ++ notfound_or_blocked[0]=True ++ elif result=="DENIED": ++ notfound_or_blocked[1]=True ++ ++ else: ++ print("Error in logger module<->analyser protocol") ++ ++ except IndexError: ++ print("IndexError while parsing %s" % record) ++ except IOError as e: ++ if e.errno!=4: # handling "Interrupted system call" errors ++ raise ++ ++ # if main thread doesnt exists then exit ++ if not self.main_thread.is_alive(): ++ break ++ epoll.unregister(self.socket_logger.fileno()) ++ epoll.close() ++ self.socket_logger.close() ++ ++ def stop(self): ++ """ Stops the log server. Returns all events """ ++ ++ self.alive=False ++ ++ # Block the main thread until listener exists ++ self.listen_thread.join() ++ ++ # We assume portage clears tmp folder, so no deleting a socket file ++ # We assume that no new socket data will arrive after this moment ++ return self.events +diff -urN /usr/lib/portage/pym/portage/const.py ./pym/portage/const.py +--- /usr/lib/portage/pym/portage/const.py 2012-05-28 16:20:40.766712558 +0600 ++++ ./pym/portage/const.py 2012-06-01 21:39:51.363837853 +0600 +@@ -67,6 +67,8 @@ + BASH_BINARY = "/bin/bash" + MOVE_BINARY = "/bin/mv" + PRELINK_BINARY = "/usr/sbin/prelink" ++AUTODEP_LIBRARY = "/usr/lib/file_hook.so" ++ + + INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" + REPO_NAME_FILE = "repo_name" +@@ -89,7 +91,7 @@ + "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", + "ccache", "chflags", "clean-logs", + "collision-protect", "compress-build-logs", "compressdebug", +- "config-protect-if-modified", ++ "config-protect-if-modified", "depcheck", "depcheckstrict", + "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", + "fail-clean", "force-mirror", "force-prefix", "getbinpkg", + "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", +diff -urN /usr/lib/portage/pym/portage/const.py.rej ./pym/portage/const.py.rej +--- /usr/lib/portage/pym/portage/const.py.rej 1970-01-01 05:00:00.000000000 +0500 ++++ ./pym/portage/const.py.rej 2012-06-01 21:37:22.803844102 +0600 +@@ -0,0 +1,12 @@ ++--- pym/portage/const.py +++++ pym/portage/const.py ++@@ -90,7 +92,8 @@ ++ SUPPORTED_FEATURES = frozenset([ ++ "allow-missing-manifests", ++ "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", ++- "ccache", "chflags", "collision-protect", "compress-build-logs", +++ "ccache", "chflags", "collision-protect", "compress-build-logs", +++ "depcheck", "depcheckstrict", ++ "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", ++ "fail-clean", "fixpackages", "force-mirror", "getbinpkg", ++ "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", +diff -urN /usr/lib/portage/pym/portage/package/ebuild/_config/special_env_vars.py ./pym/portage/package/ebuild/_config/special_env_vars.py +--- /usr/lib/portage/pym/portage/package/ebuild/_config/special_env_vars.py 2012-05-28 16:20:40.870712551 +0600 ++++ ./pym/portage/package/ebuild/_config/special_env_vars.py 2012-06-01 21:37:22.804844102 +0600 +@@ -101,8 +101,8 @@ + # other variables inherited from the calling environment + environ_whitelist += [ + "CVS_RSH", "ECHANGELOG_USER", +- "GPG_AGENT_INFO", +- "SSH_AGENT_PID", "SSH_AUTH_SOCK", ++ "GPG_AGENT_INFO", "LOG_SOCKET", ++ "SSH_AGENT_PID", "SSH_AUTH_SOCK" + "STY", "WINDOW", "XAUTHORITY", + ] + +diff -urN /usr/lib/portage/pym/portage/package/ebuild/doebuild.py ./pym/portage/package/ebuild/doebuild.py +--- /usr/lib/portage/pym/portage/package/ebuild/doebuild.py 2012-05-28 16:20:40.860712554 +0600 ++++ ./pym/portage/package/ebuild/doebuild.py 2012-06-01 21:37:22.805844102 +0600 +@@ -1222,6 +1222,9 @@ + nosandbox = ("sandbox" not in features and \ + "usersandbox" not in features) + ++ if "depcheck" in features or "depcheckstrict" in features: ++ nosandbox = True ++ + if not portage.process.sandbox_capable: + nosandbox = True + +@@ -1401,7 +1404,10 @@ + keywords["opt_name"] = "[%s/%s]" % \ + (mysettings.get("CATEGORY",""), mysettings.get("PF","")) + +- if free or "SANDBOX_ACTIVE" in os.environ: ++ if "depcheck" in features or "depcheckstrict" in features: ++ keywords["opt_name"] += " bash" ++ spawn_func = portage.process.spawn_autodep ++ elif free or "SANDBOX_ACTIVE" in os.environ: + keywords["opt_name"] += " bash" + spawn_func = portage.process.spawn_bash + elif fakeroot: +diff -urN /usr/lib/portage/pym/portage/process.py ./pym/portage/process.py +--- /usr/lib/portage/pym/portage/process.py 2012-05-28 16:20:40.768712558 +0600 ++++ ./pym/portage/process.py 2012-06-01 21:37:22.806844102 +0600 +@@ -18,7 +18,7 @@ + 'portage.util:dump_traceback', + ) + +-from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY ++from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY, AUTODEP_LIBRARY + from portage.exception import CommandNotFound + + try: +@@ -53,6 +53,9 @@ + sandbox_capable = (os.path.isfile(SANDBOX_BINARY) and + os.access(SANDBOX_BINARY, os.X_OK)) + ++autodep_capable = (os.path.isfile(AUTODEP_LIBRARY) and ++ os.access(AUTODEP_LIBRARY, os.X_OK)) ++ + fakeroot_capable = (os.path.isfile(FAKEROOT_BINARY) and + os.access(FAKEROOT_BINARY, os.X_OK)) + +@@ -80,6 +83,16 @@ + args.append(mycommand) + return spawn(args, opt_name=opt_name, **keywords) + ++def spawn_autodep(mycommand, opt_name=None, **keywords): ++ if not autodep_capable: ++ return spawn_bash(mycommand, opt_name=opt_name, **keywords) ++ if "env" not in keywords or "LOG_SOCKET" not in keywords["env"]: ++ return spawn_bash(mycommand, opt_name=opt_name, **keywords) ++ ++ # Core part: tell the loader to preload logging library ++ keywords["env"]["LD_PRELOAD"]=AUTODEP_LIBRARY ++ return spawn_bash(mycommand, opt_name=opt_name, **keywords) ++ + def spawn_sandbox(mycommand, opt_name=None, **keywords): + if not sandbox_capable: + return spawn_bash(mycommand, opt_name=opt_name, **keywords) diff --git a/portage_with_autodep/pym/_emerge/AbstractDepPriority.py b/portage_with_autodep/pym/_emerge/AbstractDepPriority.py index 94a9379..94f26ef 100644 --- a/portage_with_autodep/pym/_emerge/AbstractDepPriority.py +++ b/portage_with_autodep/pym/_emerge/AbstractDepPriority.py @@ -1,8 +1,8 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import copy -from _emerge.SlotObject import SlotObject +from portage.util.SlotObject import SlotObject class AbstractDepPriority(SlotObject): __slots__ = ("buildtime", "runtime", "runtime_post") diff --git a/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo b/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo Binary files differnew file mode 100644 index 0000000..b6a9871 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/AbstractDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py index 4147ecb..c7b8f83 100644 --- a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py +++ b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.py @@ -19,8 +19,9 @@ from portage.util import apply_secpass_permissions class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ - ('_build_dir', '_ipc_daemon', '_exit_command',) + ('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id') _phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',) + _phases_interactive_whitelist = ('config',) # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It @@ -92,7 +93,20 @@ class AbstractEbuildProcess(SpawnProcess): else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) - SpawnProcess._start(self) + if self.fd_pipes is None: + self.fd_pipes = {} + null_fd = None + if 0 not in self.fd_pipes and \ + self.phase not in self._phases_interactive_whitelist and \ + "interactive" not in self.settings.get("PROPERTIES", "").split(): + null_fd = os.open('/dev/null', os.O_RDONLY) + self.fd_pipes[0] = null_fd + + try: + SpawnProcess._start(self) + finally: + if null_fd is not None: + os.close(null_fd) def _init_ipc_fifos(self): @@ -143,13 +157,29 @@ class AbstractEbuildProcess(SpawnProcess): def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. - self.scheduler.schedule(self._reg_id, timeout=self._exit_timeout) - if self._registered: - # If it doesn't exit naturally in a reasonable amount - # of time, kill it (solves bug #278895). We try to avoid - # this when possible since it makes sandbox complain about - # being killed by a signal. - self.cancel() + self._exit_timeout_id = \ + self.scheduler.timeout_add(self._exit_timeout, + self._exit_command_timeout_cb) + + def _exit_command_timeout_cb(self): + if self._registered: + # If it doesn't exit naturally in a reasonable amount + # of time, kill it (solves bug #278895). We try to avoid + # this when possible since it makes sandbox complain about + # being killed by a signal. + self.cancel() + self._exit_timeout_id = \ + self.scheduler.timeout_add(self._cancel_timeout, + self._cancel_timeout_cb) + else: + self._exit_timeout_id = None + + return False # only run once + + def _cancel_timeout_cb(self): + self._exit_timeout_id = None + self.wait() + return False # only run once def _orphan_process_warn(self): phase = self.phase @@ -239,6 +269,10 @@ class AbstractEbuildProcess(SpawnProcess): def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) + if self._exit_timeout_id is not None: + self.scheduler.source_remove(self._exit_timeout_id) + self._exit_timeout_id = None + if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: diff --git a/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo Binary files differnew file mode 100644 index 0000000..b55f9c2 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/AbstractEbuildProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/AbstractPollTask.py b/portage_with_autodep/pym/_emerge/AbstractPollTask.py index f7f3a95..2c84709 100644 --- a/portage_with_autodep/pym/_emerge/AbstractPollTask.py +++ b/portage_with_autodep/pym/_emerge/AbstractPollTask.py @@ -1,44 +1,111 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import array +import errno import logging +import os from portage.util import writemsg_level from _emerge.AsynchronousTask import AsynchronousTask -from _emerge.PollConstants import PollConstants + class AbstractPollTask(AsynchronousTask): __slots__ = ("scheduler",) + \ ("_registered",) _bufsize = 4096 - _exceptional_events = PollConstants.POLLERR | PollConstants.POLLNVAL - _registered_events = PollConstants.POLLIN | PollConstants.POLLHUP | \ - _exceptional_events + + @property + def _exceptional_events(self): + return self.scheduler.IO_ERR | self.scheduler.IO_NVAL + + @property + def _registered_events(self): + return self.scheduler.IO_IN | self.scheduler.IO_HUP | \ + self._exceptional_events def isAlive(self): return bool(self._registered) - def _read_buf(self, f, event): + def _read_array(self, f, event): """ + NOTE: array.fromfile() is used here only for testing purposes, + because it has bugs in all known versions of Python (including + Python 2.7 and Python 3.2). See PipeReaderArrayTestCase. + | POLLIN | RETURN | BIT | VALUE | --------------------------------------------------- | 1 | Read self._bufsize into an instance of - | | array.array('B') and return it, ignoring + | | array.array('B') and return it, handling | | EOFError and IOError. An empty array | | indicates EOF. | --------------------------------------------------- | 0 | None """ buf = None - if event & PollConstants.POLLIN: + if event & self.scheduler.IO_IN: buf = array.array('B') try: buf.fromfile(f, self._bufsize) - except (EOFError, IOError): + except EOFError: pass + except TypeError: + # Python 3.2: + # TypeError: read() didn't return bytes + pass + except IOError as e: + # EIO happens with pty on Linux after the + # slave end of the pty has been closed. + if e.errno == errno.EIO: + # EOF: return empty string of bytes + pass + elif e.errno == errno.EAGAIN: + # EAGAIN: return None + buf = None + else: + raise + + if buf is not None: + try: + # Python >=3.2 + buf = buf.tobytes() + except AttributeError: + buf = buf.tostring() + + return buf + + def _read_buf(self, fd, event): + """ + | POLLIN | RETURN + | BIT | VALUE + | --------------------------------------------------- + | 1 | Read self._bufsize into a string of bytes, + | | handling EAGAIN and EIO. An empty string + | | of bytes indicates EOF. + | --------------------------------------------------- + | 0 | None + """ + # NOTE: array.fromfile() is no longer used here because it has + # bugs in all known versions of Python (including Python 2.7 + # and Python 3.2). + buf = None + if event & self.scheduler.IO_IN: + try: + buf = os.read(fd, self._bufsize) + except OSError as e: + # EIO happens with pty on Linux after the + # slave end of the pty has been closed. + if e.errno == errno.EIO: + # EOF: return empty string of bytes + buf = b'' + elif e.errno == errno.EAGAIN: + # EAGAIN: return None + buf = None + else: + raise + return buf def _unregister(self): @@ -56,7 +123,32 @@ class AbstractPollTask(AsynchronousTask): self._log_poll_exception(event) self._unregister() self.cancel() - elif event & PollConstants.POLLHUP: + self.wait() + elif event & self.scheduler.IO_HUP: self._unregister() self.wait() + def _wait(self): + if self.returncode is not None: + return self.returncode + self._wait_loop() + return self.returncode + + def _wait_loop(self, timeout=None): + + if timeout is None: + while self._registered: + self.scheduler.iteration() + return + + def timeout_cb(): + timeout_cb.timed_out = True + return False + timeout_cb.timed_out = False + timeout_cb.timeout_id = self.scheduler.timeout_add(timeout, timeout_cb) + + try: + while self._registered and not timeout_cb.timed_out: + self.scheduler.iteration() + finally: + self.scheduler.unregister(timeout_cb.timeout_id) diff --git a/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo b/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo Binary files differnew file mode 100644 index 0000000..06ef6b9 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/AbstractPollTask.pyo diff --git a/portage_with_autodep/pym/_emerge/AsynchronousLock.py b/portage_with_autodep/pym/_emerge/AsynchronousLock.py index 637ba73..587aa46 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousLock.py +++ b/portage_with_autodep/pym/_emerge/AsynchronousLock.py @@ -1,15 +1,16 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import dummy_threading import fcntl +import errno import logging import sys try: import threading except ImportError: - import dummy_threading as threading + threading = dummy_threading import portage from portage import os @@ -19,7 +20,6 @@ from portage.locks import lockfile, unlockfile from portage.util import writemsg_level from _emerge.AbstractPollTask import AbstractPollTask from _emerge.AsynchronousTask import AsynchronousTask -from _emerge.PollConstants import PollConstants from _emerge.SpawnProcess import SpawnProcess class AsynchronousLock(AsynchronousTask): @@ -35,7 +35,7 @@ class AsynchronousLock(AsynchronousTask): __slots__ = ('path', 'scheduler',) + \ ('_imp', '_force_async', '_force_dummy', '_force_process', \ - '_force_thread', '_waiting') + '_force_thread') _use_process_by_default = True @@ -66,8 +66,7 @@ class AsynchronousLock(AsynchronousTask): def _imp_exit(self, imp): # call exit listeners - if not self._waiting: - self.wait() + self.wait() def _cancel(self): if isinstance(self._imp, AsynchronousTask): @@ -81,9 +80,7 @@ class AsynchronousLock(AsynchronousTask): def _wait(self): if self.returncode is not None: return self.returncode - self._waiting = True self.returncode = self._imp.wait() - self._waiting = False return self.returncode def unlock(self): @@ -114,13 +111,13 @@ class _LockThread(AbstractPollTask): def _start(self): pr, pw = os.pipe() self._files = {} - self._files['pipe_read'] = os.fdopen(pr, 'rb', 0) - self._files['pipe_write'] = os.fdopen(pw, 'wb', 0) - for k, f in self._files.items(): - fcntl.fcntl(f.fileno(), fcntl.F_SETFL, - fcntl.fcntl(f.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) - self._reg_id = self.scheduler.register(self._files['pipe_read'].fileno(), - PollConstants.POLLIN, self._output_handler) + self._files['pipe_read'] = pr + self._files['pipe_write'] = pw + for f in self._files.values(): + fcntl.fcntl(f, fcntl.F_SETFL, + fcntl.fcntl(f, fcntl.F_GETFL) | os.O_NONBLOCK) + self._reg_id = self.scheduler.register(self._files['pipe_read'], + self.scheduler.IO_IN, self._output_handler) self._registered = True threading_mod = threading if self._force_dummy: @@ -130,26 +127,27 @@ class _LockThread(AbstractPollTask): def _run_lock(self): self._lock_obj = lockfile(self.path, wantnewlockfile=True) - self._files['pipe_write'].write(b'\0') + os.write(self._files['pipe_write'], b'\0') def _output_handler(self, f, event): - buf = self._read_buf(self._files['pipe_read'], event) + buf = None + if event & self.scheduler.IO_IN: + try: + buf = os.read(self._files['pipe_read'], self._bufsize) + except OSError as e: + if e.errno not in (errno.EAGAIN,): + raise if buf: self._unregister() self.returncode = os.EX_OK self.wait() + return True + def _cancel(self): # There's currently no way to force thread termination. pass - def _wait(self): - if self.returncode is not None: - return self.returncode - if self._registered: - self.scheduler.schedule(self._reg_id) - return self.returncode - def unlock(self): if self._lock_obj is None: raise AssertionError('not locked') @@ -171,7 +169,7 @@ class _LockThread(AbstractPollTask): if self._files is not None: for f in self._files.values(): - f.close() + os.close(f) self._files = None class _LockProcess(AbstractPollTask): @@ -190,12 +188,12 @@ class _LockProcess(AbstractPollTask): in_pr, in_pw = os.pipe() out_pr, out_pw = os.pipe() self._files = {} - self._files['pipe_in'] = os.fdopen(in_pr, 'rb', 0) - self._files['pipe_out'] = os.fdopen(out_pw, 'wb', 0) + self._files['pipe_in'] = in_pr + self._files['pipe_out'] = out_pw fcntl.fcntl(in_pr, fcntl.F_SETFL, fcntl.fcntl(in_pr, fcntl.F_GETFL) | os.O_NONBLOCK) self._reg_id = self.scheduler.register(in_pr, - PollConstants.POLLIN, self._output_handler) + self.scheduler.IO_IN, self._output_handler) self._registered = True self._proc = SpawnProcess( args=[portage._python_interpreter, @@ -209,9 +207,22 @@ class _LockProcess(AbstractPollTask): os.close(in_pw) def _proc_exit(self, proc): + + if self._files is not None: + # Close pipe_out if it's still open, since it's useless + # after the process has exited. This helps to avoid + # "ResourceWarning: unclosed file" since Python 3.2. + try: + pipe_out = self._files.pop('pipe_out') + except KeyError: + pass + else: + os.close(pipe_out) + if proc.returncode != os.EX_OK: # Typically, this will happen due to the # process being killed by a signal. + if not self._acquired: # If the lock hasn't been aquired yet, the # caller can check the returncode and handle @@ -242,21 +253,22 @@ class _LockProcess(AbstractPollTask): self._proc.poll() return self.returncode - def _wait(self): - if self.returncode is not None: - return self.returncode - if self._registered: - self.scheduler.schedule(self._reg_id) - return self.returncode - def _output_handler(self, f, event): - buf = self._read_buf(self._files['pipe_in'], event) + buf = None + if event & self.scheduler.IO_IN: + try: + buf = os.read(self._files['pipe_in'], self._bufsize) + except OSError as e: + if e.errno not in (errno.EAGAIN,): + raise if buf: self._acquired = True self._unregister() self.returncode = os.EX_OK self.wait() + return True + def _unregister(self): self._registered = False @@ -270,7 +282,7 @@ class _LockProcess(AbstractPollTask): except KeyError: pass else: - pipe_in.close() + os.close(pipe_in) def unlock(self): if self._proc is None: @@ -281,8 +293,8 @@ class _LockProcess(AbstractPollTask): raise AssertionError("lock process failed with returncode %s" \ % (self.returncode,)) self._unlocked = True - self._files['pipe_out'].write(b'\0') - self._files['pipe_out'].close() + os.write(self._files['pipe_out'], b'\0') + os.close(self._files['pipe_out']) self._files = None self._proc.wait() self._proc = None diff --git a/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo b/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo Binary files differnew file mode 100644 index 0000000..5f3cfbb --- /dev/null +++ b/portage_with_autodep/pym/_emerge/AsynchronousLock.pyo diff --git a/portage_with_autodep/pym/_emerge/AsynchronousTask.py b/portage_with_autodep/pym/_emerge/AsynchronousTask.py index 36522ca..7a193ce 100644 --- a/portage_with_autodep/pym/_emerge/AsynchronousTask.py +++ b/portage_with_autodep/pym/_emerge/AsynchronousTask.py @@ -1,8 +1,11 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import signal + from portage import os -from _emerge.SlotObject import SlotObject +from portage.util.SlotObject import SlotObject + class AsynchronousTask(SlotObject): """ Subclasses override _wait() and _poll() so that calls @@ -14,7 +17,10 @@ class AsynchronousTask(SlotObject): """ __slots__ = ("background", "cancelled", "returncode") + \ - ("_exit_listeners", "_exit_listener_stack", "_start_listeners") + ("_exit_listeners", "_exit_listener_stack", "_start_listeners", + "_waiting") + + _cancelled_returncode = - signal.SIGINT def start(self): """ @@ -42,7 +48,12 @@ class AsynchronousTask(SlotObject): def wait(self): if self.returncode is None: - self._wait() + if not self._waiting: + self._waiting = True + try: + self._wait() + finally: + self._waiting = False self._wait_hook() return self.returncode @@ -50,10 +61,17 @@ class AsynchronousTask(SlotObject): return self.returncode def cancel(self): + """ + Cancel the task, but do not wait for exit status. If asynchronous exit + notification is desired, then use addExitListener to add a listener + before calling this method. + NOTE: Synchronous waiting for status is not supported, since it would + be vulnerable to hitting the recursion limit when a large number of + tasks need to be terminated simultaneously, like in bug #402335. + """ if not self.cancelled: self.cancelled = True self._cancel() - self.wait() def _cancel(self): """ @@ -62,6 +80,17 @@ class AsynchronousTask(SlotObject): """ pass + def _was_cancelled(self): + """ + If cancelled, set returncode if necessary and return True. + Otherwise, return False. + """ + if self.cancelled: + if self.returncode is None: + self.returncode = self._cancelled_returncode + return True + return False + def addStartListener(self, f): """ The function will be called with one argument, a reference to self. @@ -123,7 +152,11 @@ class AsynchronousTask(SlotObject): self._exit_listener_stack = self._exit_listeners self._exit_listeners = None - self._exit_listener_stack.reverse() + # Execute exit listeners in reverse order, so that + # the last added listener is executed first. This + # allows SequentialTaskQueue to decrement its running + # task count as soon as one of its tasks exits, so that + # the value is accurate when other listeners execute. while self._exit_listener_stack: self._exit_listener_stack.pop()(self) diff --git a/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo b/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo Binary files differnew file mode 100644 index 0000000..b8d67ea --- /dev/null +++ b/portage_with_autodep/pym/_emerge/AsynchronousTask.pyo diff --git a/portage_with_autodep/pym/_emerge/AtomArg.pyo b/portage_with_autodep/pym/_emerge/AtomArg.pyo Binary files differnew file mode 100644 index 0000000..b8f59cf --- /dev/null +++ b/portage_with_autodep/pym/_emerge/AtomArg.pyo diff --git a/portage_with_autodep/pym/_emerge/Binpkg.py b/portage_with_autodep/pym/_emerge/Binpkg.py index bc6511e..ea8a1ad 100644 --- a/portage_with_autodep/pym/_emerge/Binpkg.py +++ b/portage_with_autodep/pym/_emerge/Binpkg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.EbuildPhase import EbuildPhase @@ -9,15 +9,18 @@ from _emerge.CompositeTask import CompositeTask from _emerge.BinpkgVerifier import BinpkgVerifier from _emerge.EbuildMerge import EbuildMerge from _emerge.EbuildBuildDir import EbuildBuildDir +from _emerge.SpawnProcess import SpawnProcess from portage.eapi import eapi_exports_replace_vars -from portage.util import writemsg +from portage.util import ensure_dirs, writemsg import portage from portage import os +from portage import shutil from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode import io import logging +import textwrap from portage.output import colorize class Binpkg(CompositeTask): @@ -25,7 +28,8 @@ class Binpkg(CompositeTask): __slots__ = ("find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ - ("_bintree", "_build_dir", "_ebuild_path", "_fetched_pkg", + ("_bintree", "_build_dir", "_build_prefix", + "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") def _writemsg_level(self, msg, level=0, noiselevel=0): @@ -83,13 +87,12 @@ class Binpkg(CompositeTask): waiting_msg = ("Fetching '%s' " + \ "in the background. " + \ - "To view fetch progress, run `tail -f " + \ + "To view fetch progress, run `tail -f %s" + \ "/var/log/emerge-fetch.log` in another " + \ - "terminal.") % prefetcher.pkg_path + "terminal.") % (prefetcher.pkg_path, settings["EPREFIX"]) msg_prefix = colorize("GOOD", " * ") - from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ - for line in wrap(waiting_msg, 65)) + for line in textwrap.wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) @@ -101,6 +104,10 @@ class Binpkg(CompositeTask): def _prefetch_exit(self, prefetcher): + if self._was_cancelled(): + self.wait() + return + pkg = self.pkg pkg_count = self.pkg_count if not (self.opts.pretend or self.opts.fetchonly): @@ -299,10 +306,68 @@ class Binpkg(CompositeTask): self._start_task(extractor, self._extractor_exit) def _extractor_exit(self, extractor): - if self._final_exit(extractor) != os.EX_OK: + if self._default_exit(extractor) != os.EX_OK: self._unlock_builddir() self._writemsg_level("!!! Error Extracting '%s'\n" % \ self._pkg_path, noiselevel=-1, level=logging.ERROR) + self.wait() + return + + try: + with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), + encoding=_encodings['fs'], errors='strict'), mode='r', + encoding=_encodings['repo.content'], errors='replace') as f: + self._build_prefix = f.read().rstrip('\n') + except IOError: + self._build_prefix = "" + + if self._build_prefix == self.settings["EPREFIX"]: + ensure_dirs(self.settings["ED"]) + self._current_task = None + self.returncode = os.EX_OK + self.wait() + return + + chpathtool = SpawnProcess( + args=[portage._python_interpreter, + os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), + self.settings["D"], self._build_prefix, self.settings["EPREFIX"]], + background=self.background, env=self.settings.environ(), + scheduler=self.scheduler, + logfile=self.settings.get('PORTAGE_LOG_FILE')) + self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) + self._start_task(chpathtool, self._chpathtool_exit) + + def _chpathtool_exit(self, chpathtool): + if self._final_exit(chpathtool) != os.EX_OK: + self._unlock_builddir() + self._writemsg_level("!!! Error Adjusting Prefix to %s" % + (self.settings["EPREFIX"],), + noiselevel=-1, level=logging.ERROR) + self.wait() + return + + # We want to install in "our" prefix, not the binary one + with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), + encoding=_encodings['fs'], errors='strict'), mode='w', + encoding=_encodings['repo.content'], errors='strict') as f: + f.write(self.settings["EPREFIX"] + "\n") + + # Move the files to the correct location for merge. + image_tmp_dir = os.path.join( + self.settings["PORTAGE_BUILDDIR"], "image_tmp") + build_d = os.path.join(self.settings["D"], + self._build_prefix.lstrip(os.sep)) + if not os.path.isdir(build_d): + # Assume this is a virtual package or something. + shutil.rmtree(self._image_dir) + ensure_dirs(self.settings["ED"]) + else: + os.rename(build_d, image_tmp_dir) + shutil.rmtree(self._image_dir) + ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep))) + os.rename(image_tmp_dir, self.settings["ED"]) + self.wait() def _unlock_builddir(self): @@ -312,13 +377,13 @@ class Binpkg(CompositeTask): self._build_dir.unlock() def create_install_task(self): - task = EbuildMerge(find_blockers=self.find_blockers, + task = EbuildMerge(exit_hook=self._install_exit, + find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, pkg_path=self._pkg_path, scheduler=self.scheduler, settings=self.settings, tree=self._tree, world_atom=self.world_atom) - task.addExitListener(self._install_exit) return task def _install_exit(self, task): diff --git a/portage_with_autodep/pym/_emerge/Binpkg.pyo b/portage_with_autodep/pym/_emerge/Binpkg.pyo Binary files differnew file mode 100644 index 0000000..4499b9d --- /dev/null +++ b/portage_with_autodep/pym/_emerge/Binpkg.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.py b/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.py index f68971b..5ba1495 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.py +++ b/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.py @@ -1,4 +1,4 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno @@ -38,7 +38,7 @@ class BinpkgEnvExtractor(CompositeTask): background=self.background, env=self.settings.environ(), scheduler=self.scheduler, - logfile=self.settings.get('PORTAGE_LOGFILE')) + logfile=self.settings.get('PORTAGE_LOG_FILE')) self._start_task(extractor_proc, self._extractor_exit) @@ -59,7 +59,7 @@ class BinpkgEnvExtractor(CompositeTask): # This is a signal to ebuild.sh, so that it knows to filter # out things like SANDBOX_{DENY,PREDICT,READ,WRITE} that # would be preserved between normal phases. - open(_unicode_encode(self._get_dest_env_path() + '.raw'), 'w') + open(_unicode_encode(self._get_dest_env_path() + '.raw'), 'wb').close() self._current_task = None self.returncode = os.EX_OK diff --git a/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo b/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo Binary files differnew file mode 100644 index 0000000..21c2e13 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/BinpkgEnvExtractor.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py index d1630f2..f25cbf9 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py +++ b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.py @@ -1,9 +1,8 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SpawnProcess import SpawnProcess import portage -import os import signal class BinpkgExtractorAsync(SpawnProcess): diff --git a/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo Binary files differnew file mode 100644 index 0000000..f8498f7 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/BinpkgExtractorAsync.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgFetcher.py b/portage_with_autodep/pym/_emerge/BinpkgFetcher.py index baea4d6..f415e2e 100644 --- a/portage_with_autodep/pym/_emerge/BinpkgFetcher.py +++ b/portage_with_autodep/pym/_emerge/BinpkgFetcher.py @@ -28,9 +28,6 @@ class BinpkgFetcher(SpawnProcess): def _start(self): - if self.cancelled: - return - pkg = self.pkg pretend = self.pretend bintree = pkg.root_config.trees["bintree"] diff --git a/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo b/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo Binary files differnew file mode 100644 index 0000000..482e55e --- /dev/null +++ b/portage_with_autodep/pym/_emerge/BinpkgFetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo b/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo Binary files differnew file mode 100644 index 0000000..c890cac --- /dev/null +++ b/portage_with_autodep/pym/_emerge/BinpkgPrefetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo b/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo Binary files differnew file mode 100644 index 0000000..21f770e --- /dev/null +++ b/portage_with_autodep/pym/_emerge/BinpkgVerifier.pyo diff --git a/portage_with_autodep/pym/_emerge/Blocker.pyo b/portage_with_autodep/pym/_emerge/Blocker.pyo Binary files differnew file mode 100644 index 0000000..b9e56bc --- /dev/null +++ b/portage_with_autodep/pym/_emerge/Blocker.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerCache.py b/portage_with_autodep/pym/_emerge/BlockerCache.py index 5c4f43e..fce81f8 100644 --- a/portage_with_autodep/pym/_emerge/BlockerCache.py +++ b/portage_with_autodep/pym/_emerge/BlockerCache.py @@ -1,6 +1,7 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import errno import sys from portage.util import writemsg from portage.data import secpass @@ -15,6 +16,9 @@ except ImportError: if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class BlockerCache(portage.cache.mappings.MutableMapping): """This caches blockers of installed packages so that dep_check does not @@ -58,8 +62,11 @@ class BlockerCache(portage.cache.mappings.MutableMapping): self._cache_data = mypickle.load() f.close() del f - except (IOError, OSError, EOFError, ValueError, pickle.UnpicklingError) as e: - if isinstance(e, pickle.UnpicklingError): + except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e: + if isinstance(e, EnvironmentError) and \ + getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): + pass + else: writemsg("!!! Error loading '%s': %s\n" % \ (self._cache_filename, str(e)), noiselevel=-1) del e @@ -141,7 +148,7 @@ class BlockerCache(portage.cache.mappings.MutableMapping): f.close() portage.util.apply_secpass_permissions( self._cache_filename, gid=portage.portage_gid, mode=0o644) - except (IOError, OSError) as e: + except (IOError, OSError): pass self._modified.clear() @@ -155,8 +162,8 @@ class BlockerCache(portage.cache.mappings.MutableMapping): @param blocker_data: An object with counter and atoms attributes. @type blocker_data: BlockerData """ - self._cache_data["blockers"][cpv] = \ - (blocker_data.counter, tuple(str(x) for x in blocker_data.atoms)) + self._cache_data["blockers"][_unicode(cpv)] = (blocker_data.counter, + tuple(_unicode(x) for x in blocker_data.atoms)) self._modified.add(cpv) def __iter__(self): @@ -176,7 +183,7 @@ class BlockerCache(portage.cache.mappings.MutableMapping): def __getitem__(self, cpv): """ @rtype: BlockerData - @returns: An object with counter and atoms attributes. + @return: An object with counter and atoms attributes. """ return self.BlockerData(*self._cache_data["blockers"][cpv]) diff --git a/portage_with_autodep/pym/_emerge/BlockerCache.pyo b/portage_with_autodep/pym/_emerge/BlockerCache.pyo Binary files differnew file mode 100644 index 0000000..41554e1 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/BlockerCache.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerDB.py b/portage_with_autodep/pym/_emerge/BlockerDB.py index 4819749..459affd 100644 --- a/portage_with_autodep/pym/_emerge/BlockerDB.py +++ b/portage_with_autodep/pym/_emerge/BlockerDB.py @@ -25,7 +25,7 @@ class BlockerDB(object): self._dep_check_trees = None self._fake_vartree = fake_vartree self._dep_check_trees = { - self._vartree.root : { + self._vartree.settings["EROOT"] : { "porttree" : fake_vartree, "vartree" : fake_vartree, }} @@ -36,7 +36,8 @@ class BlockerDB(object): new_pkg is planned to be installed. This ignores build-time blockers, since new_pkg is assumed to be built already. """ - blocker_cache = BlockerCache(self._vartree.root, self._vartree.dbapi) + blocker_cache = BlockerCache(None, + self._vartree.dbapi) dep_keys = ["RDEPEND", "PDEPEND"] settings = self._vartree.settings stale_cache = set(blocker_cache) diff --git a/portage_with_autodep/pym/_emerge/BlockerDB.pyo b/portage_with_autodep/pym/_emerge/BlockerDB.pyo Binary files differnew file mode 100644 index 0000000..dfab0aa --- /dev/null +++ b/portage_with_autodep/pym/_emerge/BlockerDB.pyo diff --git a/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo b/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo Binary files differnew file mode 100644 index 0000000..c3b554c --- /dev/null +++ b/portage_with_autodep/pym/_emerge/BlockerDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/CompositeTask.py b/portage_with_autodep/pym/_emerge/CompositeTask.py index 644a69b..3e43478 100644 --- a/portage_with_autodep/pym/_emerge/CompositeTask.py +++ b/portage_with_autodep/pym/_emerge/CompositeTask.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AsynchronousTask import AsynchronousTask @@ -60,7 +60,8 @@ class CompositeTask(AsynchronousTask): self._current_task = None break else: - self.scheduler.schedule(condition=self._task_queued_wait) + while not self._task_queued_wait(): + self.scheduler.iteration() if self.returncode is not None: break elif self.cancelled: @@ -103,7 +104,7 @@ class CompositeTask(AsynchronousTask): Subclasses can use this as a generic task exit callback. @rtype: int - @returns: The task.returncode attribute. + @return: The task.returncode attribute. """ self._assert_current(task) if task.returncode != os.EX_OK: diff --git a/portage_with_autodep/pym/_emerge/CompositeTask.pyo b/portage_with_autodep/pym/_emerge/CompositeTask.pyo Binary files differnew file mode 100644 index 0000000..adc8cae --- /dev/null +++ b/portage_with_autodep/pym/_emerge/CompositeTask.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPriority.pyo b/portage_with_autodep/pym/_emerge/DepPriority.pyo Binary files differnew file mode 100644 index 0000000..4028a36 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/DepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo b/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo Binary files differnew file mode 100644 index 0000000..5e0f710 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/DepPriorityNormalRange.pyo diff --git a/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo Binary files differnew file mode 100644 index 0000000..5309bcd --- /dev/null +++ b/portage_with_autodep/pym/_emerge/DepPrioritySatisfiedRange.pyo diff --git a/portage_with_autodep/pym/_emerge/Dependency.py b/portage_with_autodep/pym/_emerge/Dependency.py index 0f746b6..c2d36b2 100644 --- a/portage_with_autodep/pym/_emerge/Dependency.py +++ b/portage_with_autodep/pym/_emerge/Dependency.py @@ -1,8 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from portage.util.SlotObject import SlotObject from _emerge.DepPriority import DepPriority -from _emerge.SlotObject import SlotObject + class Dependency(SlotObject): __slots__ = ("atom", "blocker", "child", "depth", "parent", "onlydeps", "priority", "root", diff --git a/portage_with_autodep/pym/_emerge/Dependency.pyo b/portage_with_autodep/pym/_emerge/Dependency.pyo Binary files differnew file mode 100644 index 0000000..f53e0ed --- /dev/null +++ b/portage_with_autodep/pym/_emerge/Dependency.pyo diff --git a/portage_with_autodep/pym/_emerge/DependencyArg.pyo b/portage_with_autodep/pym/_emerge/DependencyArg.pyo Binary files differnew file mode 100644 index 0000000..916a762 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/DependencyArg.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBinpkg.py b/portage_with_autodep/pym/_emerge/EbuildBinpkg.py index b7d43ba..34a6aef 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBinpkg.py +++ b/portage_with_autodep/pym/_emerge/EbuildBinpkg.py @@ -1,4 +1,4 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.CompositeTask import CompositeTask @@ -34,6 +34,10 @@ class EbuildBinpkg(CompositeTask): self.settings.pop("PORTAGE_BINPKG_TMPFILE", None) if self._default_exit(package_phase) != os.EX_OK: + try: + os.unlink(self._binpkg_tmpfile) + except OSError: + pass self.wait() return diff --git a/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo b/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo Binary files differnew file mode 100644 index 0000000..2acfc87 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildBinpkg.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBuild.py b/portage_with_autodep/pym/_emerge/EbuildBuild.py index 1c423a3..5a48f8e 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuild.py +++ b/portage_with_autodep/pym/_emerge/EbuildBuild.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.EbuildExecuter import EbuildExecuter @@ -32,12 +32,13 @@ class EbuildBuild(CompositeTask): pkg = self.pkg settings = self.settings - rval = _check_temp_dir(settings) - if rval != os.EX_OK: - self.returncode = rval - self._current_task = None - self.wait() - return + if not self.opts.fetchonly: + rval = _check_temp_dir(settings) + if rval != os.EX_OK: + self.returncode = rval + self._current_task = None + self.wait() + return root_config = pkg.root_config tree = "porttree" @@ -108,6 +109,10 @@ class EbuildBuild(CompositeTask): def _prefetch_exit(self, prefetcher): + if self._was_cancelled(): + self.wait() + return + opts = self.opts pkg = self.pkg settings = self.settings @@ -225,9 +230,11 @@ class EbuildBuild(CompositeTask): #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ - not opts.buildpkg + "buildpkg" not in features and \ + opts.buildpkg != 'n' - if opts.buildpkg or self._issyspkg: + if ("buildpkg" in features or self._issyspkg) \ + and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): self._buildpkg = True @@ -406,7 +413,8 @@ class EbuildBuild(CompositeTask): ebuild_path = self._ebuild_path tree = self._tree - task = EbuildMerge(find_blockers=self.find_blockers, + task = EbuildMerge(exit_hook=self._install_exit, + find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, @@ -419,7 +427,6 @@ class EbuildBuild(CompositeTask): (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) - task.addExitListener(self._install_exit) return task def _install_exit(self, task): diff --git a/portage_with_autodep/pym/_emerge/EbuildBuild.pyo b/portage_with_autodep/pym/_emerge/EbuildBuild.pyo Binary files differnew file mode 100644 index 0000000..19d913c --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildBuild.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildBuildDir.py b/portage_with_autodep/pym/_emerge/EbuildBuildDir.py index ddc5fe0..9773bd7 100644 --- a/portage_with_autodep/pym/_emerge/EbuildBuildDir.py +++ b/portage_with_autodep/pym/_emerge/EbuildBuildDir.py @@ -1,11 +1,12 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.AsynchronousLock import AsynchronousLock -from _emerge.SlotObject import SlotObject + import portage from portage import os from portage.exception import PortageException +from portage.util.SlotObject import SlotObject import errno class EbuildBuildDir(SlotObject): diff --git a/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo b/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo Binary files differnew file mode 100644 index 0000000..2846579 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildBuildDir.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildExecuter.py b/portage_with_autodep/pym/_emerge/EbuildExecuter.py index f8febd4..fd663a4 100644 --- a/portage_with_autodep/pym/_emerge/EbuildExecuter.py +++ b/portage_with_autodep/pym/_emerge/EbuildExecuter.py @@ -12,7 +12,7 @@ from portage.package.ebuild.doebuild import _prepare_fake_distdir class EbuildExecuter(CompositeTask): - __slots__ = ("pkg", "scheduler", "settings") + __slots__ = ("pkg", "settings") _phases = ("prepare", "configure", "compile", "test", "install") @@ -34,8 +34,6 @@ class EbuildExecuter(CompositeTask): cleanup = 0 portage.prepare_build_dirs(pkg.root, settings, cleanup) - portdb = pkg.root_config.trees['porttree'].dbapi - ebuild_path = settings['EBUILD'] alist = settings.configdict["pkg"].get("A", "").split() _prepare_fake_distdir(settings, alist) diff --git a/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo b/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo Binary files differnew file mode 100644 index 0000000..592a0c9 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildExecuter.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildFetcher.py b/portage_with_autodep/pym/_emerge/EbuildFetcher.py index feb68d0..c0a7fdd 100644 --- a/portage_with_autodep/pym/_emerge/EbuildFetcher.py +++ b/portage_with_autodep/pym/_emerge/EbuildFetcher.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import traceback @@ -21,7 +21,7 @@ class EbuildFetcher(SpawnProcess): __slots__ = ("config_pool", "ebuild_path", "fetchonly", "fetchall", "pkg", "prefetch") + \ - ("_digests", "_settings", "_uri_map") + ("_digests", "_manifest", "_settings", "_uri_map") def already_fetched(self, settings): """ @@ -40,7 +40,7 @@ class EbuildFetcher(SpawnProcess): digests = self._get_digests() distdir = settings["DISTDIR"] - allow_missing = "allow-missing-manifests" in settings.features + allow_missing = self._get_manifest().allow_missing for filename in uri_map: # Use stat rather than lstat since fetch() creates @@ -163,10 +163,15 @@ class EbuildFetcher(SpawnProcess): pid = os.fork() if pid != 0: + if not isinstance(pid, int): + raise AssertionError( + "fork returned non-integer: %s" % (repr(pid),)) portage.process.spawned_pids.append(pid) return [pid] - portage.process._setup_pipes(fd_pipes) + portage.locks._close_fds() + # Disable close_fds since we don't exec (see _setup_pipes docstring). + portage.process._setup_pipes(fd_pipes, close_fds=False) # Use default signal handlers in order to avoid problems # killing subprocesses as reported in bug #353239. @@ -179,7 +184,7 @@ class EbuildFetcher(SpawnProcess): not in ('yes', 'true') rval = 1 - allow_missing = 'allow-missing-manifests' in self._settings.features + allow_missing = self._get_manifest().allow_missing try: if fetch(self._uri_map, self._settings, fetchonly=self.fetchonly, digests=copy.deepcopy(self._get_digests()), @@ -203,11 +208,16 @@ class EbuildFetcher(SpawnProcess): raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) return self.ebuild_path + def _get_manifest(self): + if self._manifest is None: + pkgdir = os.path.dirname(self._get_ebuild_path()) + self._manifest = self.pkg.root_config.settings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))).load_manifest(pkgdir, None) + return self._manifest + def _get_digests(self): - if self._digests is not None: - return self._digests - self._digests = portage.Manifest(os.path.dirname( - self._get_ebuild_path()), None).getTypeDigests("DIST") + if self._digests is None: + self._digests = self._get_manifest().getTypeDigests("DIST") return self._digests def _get_uri_map(self): diff --git a/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo b/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo Binary files differnew file mode 100644 index 0000000..ddc92d1 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildFetcher.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildFetchonly.py b/portage_with_autodep/pym/_emerge/EbuildFetchonly.py index b898971..f88ea96 100644 --- a/portage_with_autodep/pym/_emerge/EbuildFetchonly.py +++ b/portage_with_autodep/pym/_emerge/EbuildFetchonly.py @@ -1,10 +1,10 @@ -# Copyright 1999-2010 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from _emerge.SlotObject import SlotObject import portage from portage import os from portage.elog.messages import eerror +from portage.util.SlotObject import SlotObject class EbuildFetchonly(SlotObject): @@ -21,7 +21,7 @@ class EbuildFetchonly(SlotObject): debug = settings.get("PORTAGE_DEBUG") == "1" rval = portage.doebuild(ebuild_path, "fetch", - settings["ROOT"], settings, debug=debug, + settings=settings, debug=debug, listonly=self.pretend, fetchonly=1, fetchall=self.fetch_all, mydbapi=portdb, tree="porttree") diff --git a/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo b/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo Binary files differnew file mode 100644 index 0000000..c54a1db --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildFetchonly.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.py b/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.py index 5dabe34..8414d20 100644 --- a/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.py +++ b/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.py @@ -1,14 +1,15 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno import logging import pickle from portage import os +from portage.exception import TryAgain from portage.localization import _ +from portage.locks import lockfile, unlockfile from portage.util import writemsg_level from _emerge.FifoIpcDaemon import FifoIpcDaemon -from _emerge.PollConstants import PollConstants class EbuildIpcDaemon(FifoIpcDaemon): """ @@ -34,7 +35,7 @@ class EbuildIpcDaemon(FifoIpcDaemon): def _input_handler(self, fd, event): # Read the whole pickle in a single atomic read() call. data = None - if event & PollConstants.POLLIN: + if event & self.scheduler.IO_IN: # For maximum portability, use os.read() here since # array.fromfile() and file.read() are both known to # erroneously return an empty string from this @@ -84,6 +85,30 @@ class EbuildIpcDaemon(FifoIpcDaemon): if reply_hook is not None: reply_hook() + elif event & self.scheduler.IO_HUP: + # This can be triggered due to a race condition which happens when + # the previous _reopen_input() call occurs before the writer has + # closed the pipe (see bug #401919). It's not safe to re-open + # without a lock here, since it's possible that another writer will + # write something to the pipe just before we close it, and in that + # case the write will be lost. Therefore, try for a non-blocking + # lock, and only re-open the pipe if the lock is acquired. + lock_filename = os.path.join( + os.path.dirname(self.input_fifo), '.ipc_lock') + try: + lock_obj = lockfile(lock_filename, unlinkfile=True, + flags=os.O_NONBLOCK) + except TryAgain: + # We'll try again when another IO_HUP event arrives. + pass + else: + try: + self._reopen_input() + finally: + unlockfile(lock_obj) + + return True + def _send_reply(self, reply): # File streams are in unbuffered mode since we do atomic # read and write of whole pickles. Use non-blocking mode so diff --git a/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo b/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo Binary files differnew file mode 100644 index 0000000..7a9588f --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildIpcDaemon.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildMerge.py b/portage_with_autodep/pym/_emerge/EbuildMerge.py index 9c35988..df0778c 100644 --- a/portage_with_autodep/pym/_emerge/EbuildMerge.py +++ b/portage_with_autodep/pym/_emerge/EbuildMerge.py @@ -7,7 +7,7 @@ from portage.dbapi._MergeProcess import MergeProcess class EbuildMerge(CompositeTask): - __slots__ = ("find_blockers", "logger", "ldpath_mtimes", + __slots__ = ("exit_hook", "find_blockers", "logger", "ldpath_mtimes", "pkg", "pkg_count", "pkg_path", "pretend", "settings", "tree", "world_atom") @@ -35,6 +35,7 @@ class EbuildMerge(CompositeTask): def _merge_exit(self, merge_task): if self._final_exit(merge_task) != os.EX_OK: + self.exit_hook(self) self.wait() return @@ -53,4 +54,5 @@ class EbuildMerge(CompositeTask): logger.log(" ::: completed emerge (%s of %s) %s to %s" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg.root)) + self.exit_hook(self) self.wait() diff --git a/portage_with_autodep/pym/_emerge/EbuildMerge.pyo b/portage_with_autodep/pym/_emerge/EbuildMerge.pyo Binary files differnew file mode 100644 index 0000000..662c681 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildMerge.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py index e53298b..c2d3747 100644 --- a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py +++ b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.py @@ -1,15 +1,19 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.SubProcess import SubProcess -from _emerge.PollConstants import PollConstants import sys from portage.cache.mappings import slot_dict_class import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.package.ebuild._eapi_invalid:eapi_invalid', +) from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode + +import errno import fcntl import io @@ -20,37 +24,44 @@ class EbuildMetadataPhase(SubProcess): used to extract metadata from the ebuild. """ - __slots__ = ("cpv", "ebuild_path", "fd_pipes", "metadata_callback", - "ebuild_mtime", "metadata", "portdb", "repo_path", "settings") + \ - ("_raw_metadata",) + __slots__ = ("cpv", "eapi_supported", "ebuild_hash", "fd_pipes", + "metadata", "portdb", "repo_path", "settings") + \ + ("_eapi", "_eapi_lineno", "_raw_metadata",) _file_names = ("ebuild",) _files_dict = slot_dict_class(_file_names, prefix="") _metadata_fd = 9 def _start(self): + ebuild_path = self.ebuild_hash.location + + with io.open(_unicode_encode(ebuild_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + self._eapi, self._eapi_lineno = portage._parse_eapi_ebuild_head(f) + + parsed_eapi = self._eapi + if parsed_eapi is None: + parsed_eapi = "0" + + if not parsed_eapi: + # An empty EAPI setting is invalid. + self._eapi_invalid(None) + self._set_returncode((self.pid, 1 << 8)) + self.wait() + return + + self.eapi_supported = portage.eapi_is_supported(parsed_eapi) + if not self.eapi_supported: + self.metadata = {"EAPI": parsed_eapi} + self._set_returncode((self.pid, os.EX_OK << 8)) + self.wait() + return + settings = self.settings settings.setcpv(self.cpv) - ebuild_path = self.ebuild_path - - eapi = None - if eapi is None and \ - 'parse-eapi-ebuild-head' in settings.features: - eapi = portage._parse_eapi_ebuild_head( - io.open(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace')) - - if eapi is not None: - if not portage.eapi_is_supported(eapi): - self.metadata_callback(self.cpv, self.ebuild_path, - self.repo_path, {'EAPI' : eapi}, self.ebuild_mtime) - self._set_returncode((self.pid, os.EX_OK << 8)) - self.wait() - return - - settings.configdict['pkg']['EAPI'] = eapi + settings.configdict['pkg']['EAPI'] = parsed_eapi debug = settings.get("PORTAGE_DEBUG") == "1" master_fd = None @@ -61,7 +72,8 @@ class EbuildMetadataPhase(SubProcess): else: fd_pipes = {} - fd_pipes.setdefault(0, sys.stdin.fileno()) + null_input = open('/dev/null', 'rb') + fd_pipes.setdefault(0, null_input.fileno()) fd_pipes.setdefault(1, sys.stdout.fileno()) fd_pipes.setdefault(2, sys.stderr.fileno()) @@ -72,7 +84,6 @@ class EbuildMetadataPhase(SubProcess): if fd == sys.stderr.fileno(): sys.stderr.flush() - fd_pipes_orig = fd_pipes.copy() self._files = self._files_dict() files = self._files @@ -83,17 +94,18 @@ class EbuildMetadataPhase(SubProcess): fd_pipes[self._metadata_fd] = slave_fd self._raw_metadata = [] - files.ebuild = os.fdopen(master_fd, 'rb', 0) - self._reg_id = self.scheduler.register(files.ebuild.fileno(), + files.ebuild = master_fd + self._reg_id = self.scheduler.register(files.ebuild, self._registered_events, self._output_handler) self._registered = True retval = portage.doebuild(ebuild_path, "depend", - settings["ROOT"], settings, debug, + settings=settings, debug=debug, mydbapi=self.portdb, tree="porttree", fd_pipes=fd_pipes, returnpid=True) os.close(slave_fd) + null_input.close() if isinstance(retval, int): # doebuild failed before spawning @@ -107,27 +119,81 @@ class EbuildMetadataPhase(SubProcess): def _output_handler(self, fd, event): - if event & PollConstants.POLLIN: - self._raw_metadata.append(self._files.ebuild.read()) - if not self._raw_metadata[-1]: - self._unregister() - self.wait() + if event & self.scheduler.IO_IN: + while True: + try: + self._raw_metadata.append( + os.read(self._files.ebuild, self._bufsize)) + except OSError as e: + if e.errno not in (errno.EAGAIN,): + raise + break + else: + if not self._raw_metadata[-1]: + self._unregister() + self.wait() + break self._unregister_if_appropriate(event) + return True + def _set_returncode(self, wait_retval): SubProcess._set_returncode(self, wait_retval) - if self.returncode == os.EX_OK: - metadata_lines = ''.join(_unicode_decode(chunk, - encoding=_encodings['repo.content'], errors='replace') - for chunk in self._raw_metadata).splitlines() + # self._raw_metadata is None when _start returns + # early due to an unsupported EAPI detected with + # FEATURES=parse-eapi-ebuild-head + if self.returncode == os.EX_OK and \ + self._raw_metadata is not None: + metadata_lines = _unicode_decode(b''.join(self._raw_metadata), + encoding=_encodings['repo.content'], + errors='replace').splitlines() + metadata_valid = True if len(portage.auxdbkeys) != len(metadata_lines): # Don't trust bash's returncode if the # number of lines is incorrect. - self.returncode = 1 + metadata_valid = False else: - metadata = zip(portage.auxdbkeys, metadata_lines) - self.metadata = self.metadata_callback(self.cpv, - self.ebuild_path, self.repo_path, metadata, - self.ebuild_mtime) + metadata = dict(zip(portage.auxdbkeys, metadata_lines)) + parsed_eapi = self._eapi + if parsed_eapi is None: + parsed_eapi = "0" + self.eapi_supported = \ + portage.eapi_is_supported(metadata["EAPI"]) + if (not metadata["EAPI"] or self.eapi_supported) and \ + metadata["EAPI"] != parsed_eapi: + self._eapi_invalid(metadata) + if 'parse-eapi-ebuild-head' in self.settings.features: + metadata_valid = False + + if metadata_valid: + # Since we're supposed to be able to efficiently obtain the + # EAPI from _parse_eapi_ebuild_head, we don't write cache + # entries for unsupported EAPIs. + if self.eapi_supported: + + if metadata.get("INHERITED", False): + metadata["_eclasses_"] = \ + self.portdb.repositories.get_repo_for_location( + self.repo_path).eclass_db.get_eclass_data( + metadata["INHERITED"].split()) + else: + metadata["_eclasses_"] = {} + metadata.pop("INHERITED", None) + + self.portdb._write_cache(self.cpv, + self.repo_path, metadata, self.ebuild_hash) + else: + metadata = {"EAPI": metadata["EAPI"]} + self.metadata = metadata + else: + self.returncode = 1 + def _eapi_invalid(self, metadata): + repo_name = self.portdb.getRepositoryName(self.repo_path) + if metadata is not None: + eapi_var = metadata["EAPI"] + else: + eapi_var = None + eapi_invalid(self, self.cpv, repo_name, self.settings, + eapi_var, self._eapi, self._eapi_lineno) diff --git a/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo Binary files differnew file mode 100644 index 0000000..fcc0874 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildMetadataPhase.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.py b/portage_with_autodep/pym/_emerge/EbuildPhase.py index 82c165d..36ca8b0 100644 --- a/portage_with_autodep/pym/_emerge/EbuildPhase.py +++ b/portage_with_autodep/pym/_emerge/EbuildPhase.py @@ -33,12 +33,14 @@ class EbuildPhase(CompositeTask): ("_ebuild_lock",) # FEATURES displayed prior to setup phase - _features_display = ("ccache", "depcheck", "depcheckstrict" "distcc", - "distcc-pump", "fakeroot", + _features_display = ( + "ccache", "compressdebug", "depcheck", "depcheckstrict", + "distcc", "distcc-pump", "fakeroot", "installsources", "keeptemp", "keepwork", "nostrip", "preserve-libs", "sandbox", "selinux", "sesandbox", "splitdebug", "suidctl", "test", "userpriv", - "usersandbox") + "usersandbox" + ) # Locked phases _locked_phases = ("setup", "preinst", "postinst", "prerm", "postrm") @@ -274,13 +276,15 @@ class EbuildPhase(CompositeTask): temp_file = open(_unicode_encode(temp_log, encoding=_encodings['fs'], errors='strict'), 'rb') - log_file = self._open_log(log_path) + log_file, log_file_real = self._open_log(log_path) for line in temp_file: log_file.write(line) temp_file.close() log_file.close() + if log_file_real is not log_file: + log_file_real.close() os.unlink(temp_log) def _open_log(self, log_path): @@ -288,11 +292,12 @@ class EbuildPhase(CompositeTask): f = open(_unicode_encode(log_path, encoding=_encodings['fs'], errors='strict'), mode='ab') + f_real = f if log_path.endswith('.gz'): f = gzip.GzipFile(filename='', mode='ab', fileobj=f) - return f + return (f, f_real) def _die_hooks(self): self.returncode = None diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej b/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej new file mode 100644 index 0000000..0f061da --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildPhase.py.rej @@ -0,0 +1,12 @@ +--- pym/_emerge/EbuildPhase.py ++++ pym/_emerge/EbuildPhase.py +@@ -33,7 +33,8 @@ + ("_ebuild_lock",) + + # FEATURES displayed prior to setup phase +- _features_display = ("ccache", "distcc", "distcc-pump", "fakeroot", ++ _features_display = ("ccache", "depcheck", "depcheckstrict" "distcc", ++ "distcc-pump", "fakeroot", + "installsources", "keeptemp", "keepwork", "nostrip", + "preserve-libs", "sandbox", "selinux", "sesandbox", + "splitdebug", "suidctl", "test", "userpriv", diff --git a/portage_with_autodep/pym/_emerge/EbuildPhase.pyo b/portage_with_autodep/pym/_emerge/EbuildPhase.pyo Binary files differnew file mode 100644 index 0000000..4c73313 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildPhase.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildProcess.pyo b/portage_with_autodep/pym/_emerge/EbuildProcess.pyo Binary files differnew file mode 100644 index 0000000..52f6cdf --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo Binary files differnew file mode 100644 index 0000000..1f3e925 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/EbuildSpawnProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/EventsAnalyser.py b/portage_with_autodep/pym/_emerge/EventsAnalyser.py index 7e416e7..65ece7b 100644 --- a/portage_with_autodep/pym/_emerge/EventsAnalyser.py +++ b/portage_with_autodep/pym/_emerge/EventsAnalyser.py @@ -18,51 +18,39 @@ class PortageUtils: self.metadata_keys = [k for k in portage.auxdbkeys if not k.startswith("UNUSED_")] self.use=self.settings["USE"] - def get_best_visible_pkg(self,pkg,db="portdb"): + def get_best_visible_pkg(self,pkg): """ Gets best candidate on installing. Returns empty string if no found :param pkg: package name - :param db: name of db to look. Can be "vardb" or "portdb" """ try: - if db=="portdb": - return self.portdbapi.xmatch("bestmatch-visible", pkg) - elif db=="vardb": - return self.vardbapi.match(pkg)[0] - else: - return '' + return self.portdbapi.xmatch("bestmatch-visible", pkg) except: return '' # non-recursive dependency getter - def get_dep(self,pkg,dep_type=["RDEPEND","DEPEND"],db="portdb"): + def get_dep(self,pkg,dep_type=["RDEPEND","DEPEND"]): """ Gets current dependencies of a package. Looks in portage db :param pkg: name of package :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ["RDEPEND", "DEPEND"] - :param db: name of db to look. Can be "vardb" or "portdb" :returns: **set** of packages names """ ret=set() - pkg = self.get_best_visible_pkg(pkg,db) + pkg = self.get_best_visible_pkg(pkg) if not pkg: return ret # we found the best visible match in common tree - if db=="portdb": - aux_get=self.portdbapi.aux_get - elif db=="vardb": - aux_get=self.vardbapi.aux_get - else: - return ret - metadata = dict(zip(self.metadata_keys, aux_get(pkg, self.metadata_keys))) + metadata = dict(zip(self.metadata_keys, + self.portdbapi.aux_get(pkg, self.metadata_keys))) dep_str = " ".join(metadata[k] for k in dep_type) # the IUSE default are very important for us @@ -94,7 +82,7 @@ class PortageUtils: return ret # recursive dependency getter - def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"],db="portdb"): + def get_deps(self,pkg,dep_type=["RDEPEND","DEPEND"]): """ Gets current dependencies of a package on any depth All dependencies **must** be installed @@ -102,20 +90,19 @@ class PortageUtils: :param pkg: name of package :param dep_type: type of dependencies to recurse. Can be ["DEPEND"] or ["RDEPEND", "DEPEND"] - :param db: name of db to look. Can be "vardb" or "portdb" :returns: **set** of packages names """ ret=set() - #import pdb; pdb.set_trace() + # get porttree dependencies on the first package - pkg = self.get_best_visible_pkg(pkg,db) + pkg = self.portdbapi.xmatch("bestmatch-visible", pkg) if not pkg: return ret known_packages=set() - unknown_packages=self.get_dep(pkg,dep_type,db) + unknown_packages=self.get_dep(pkg,dep_type) ret=ret.union(unknown_packages) while unknown_packages: @@ -124,40 +111,36 @@ class PortageUtils: continue known_packages.add(p) - current_deps=self.get_dep(p,dep_type,'vardb') - unknown_packages=unknown_packages.union(current_deps) - ret=ret.union(current_deps) - - #metadata = dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, self.metadata_keys))) + metadata = dict(zip(self.metadata_keys, self.vardbapi.aux_get(p, self.metadata_keys))) - #dep_str = " ".join(metadata[k] for k in dep_type) + dep_str = " ".join(metadata[k] for k in dep_type) # the IUSE default are very important for us - #iuse_defaults=[ - # u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] + iuse_defaults=[ + u[1:] for u in metadata.get("IUSE",'').split() if u.startswith("+")] - #use=self.use.split() + use=self.use.split() - #for u in iuse_defaults: - # if u not in use: - # use.append(u) + for u in iuse_defaults: + if u not in use: + use.append(u) - #success, atoms = portage.dep_check(dep_str, None, self.settings, - # myuse=use, myroot=self.settings["ROOT"], - # trees={self.settings["ROOT"]:{"vartree":self.vartree,"porttree": self.vartree}}) - - #if not success: - # continue - - #for atom in atoms: - # atomname = self.vartree.dep_bestmatch(atom) - # if not atomname: - # continue - # - # for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): - # for pkg in self.vartree.dep_match(unvirt_pkg): - # ret.add(pkg) - # unknown_packages.add(pkg) + success, atoms = portage.dep_check(dep_str, None, self.settings, + myuse=use, myroot=self.settings["ROOT"], + trees={self.settings["ROOT"]:{"vartree":self.vartree,"porttree": self.vartree}}) + + if not success: + continue + + for atom in atoms: + atomname = self.vartree.dep_bestmatch(atom) + if not atomname: + continue + + for unvirt_pkg in expand_new_virt(self.vardbapi,'='+atomname): + for pkg in self.vartree.dep_match(unvirt_pkg): + ret.add(pkg) + unknown_packages.add(pkg) return ret def get_deps_for_package_building(self, pkg): @@ -165,13 +148,12 @@ class PortageUtils: returns buildtime dependencies of current package and all runtime dependencies of that buildtime dependencies """ - buildtime_deps=self.get_dep(pkg, ["DEPEND"],"portdb") + buildtime_deps=self.get_dep(pkg, ["DEPEND"]) runtime_deps=set() for dep in buildtime_deps: - runtime_deps|=self.get_deps(dep,["RDEPEND","PDEPEND"],"vardb") + runtime_deps=runtime_deps.union(self.get_deps(dep,["RDEPEND"])) - ret = buildtime_deps | runtime_deps - + ret=buildtime_deps.union(runtime_deps) return ret def get_system_packages_list(self): @@ -187,19 +169,7 @@ class PortageUtils: for pkg in self.vartree.dep_match(unvirt_pkg): ret.append(pkg) return ret - - def get_system_packages_rdeps(self): - """ - returns runtime dependencies of packages from system set - - :returns: **list** of package names - """ - ret=set() - - for pkg in self.get_system_packages_list(): - ret=ret.union(self.get_deps(pkg,["RDEPEND"])) - return list(ret) - + class GentoolkitUtils: """ @@ -207,7 +177,7 @@ class GentoolkitUtils: internals. """ - def getpackagesbyfiles(self,files): + def getpackagesbyfiles(files): """ :param files: list of filenames :returns: **dictionary** file->package, if file doesn't belong to any @@ -226,30 +196,17 @@ class GentoolkitUtils: stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE, bufsize=4096) - out,err=proc.communicate(b"\n".join(listtocheck)) + out,err=proc.communicate("\n".join(listtocheck).encode("utf8")) - lines=out.split(b"\n") + lines=out.decode("utf8").split("\n") #print lines line_re=re.compile(r"^([^ ]+)\s+\(([^)]+)\)$") for line in lines: - try: - line=line.decode("utf-8") - except UnicodeDecodeError: - portage.util.writemsg("Util qfile returned non-utf8 string: %s\n" % line) - - #import pdb; pdb.set_trace() - if len(line)==0: continue match=line_re.match(line) if match: - try: - ret[match.group(2).encode("utf-8")]=match.group(1) - except UnicodeEncodeError: - portage.util.writemsg( - "Util qfile failed to encode string %s to unicode\n" % - match.group(2)) - + ret[match.group(2)]=match.group(1) else: portage.util.writemsg("Util qfile returned unparsable string: %s\n" % line) @@ -259,7 +216,7 @@ class GentoolkitUtils: return ret - def getfilesbypackages(self,packagenames): + def getfilesbypackages(packagenames): """ :param packagename: name of package @@ -273,7 +230,7 @@ class GentoolkitUtils: out,err=proc.communicate() - ret=out.split(b"\n") + ret=out.decode("utf8").split("\n") if ret==['']: ret=[] except OSError as e: @@ -281,7 +238,7 @@ class GentoolkitUtils: return ret - def get_all_packages_files(self): + def get_all_packages_files(): """ Memory-hungry operation @@ -295,7 +252,7 @@ class GentoolkitUtils: out,err=proc.communicate() - ret=out.split(b"\n") + ret=out.decode("utf8").split("\n") except OSError as e: portage.util.writemsg("Error while launching qfile: %s\n" % e) @@ -306,28 +263,25 @@ class FilterProcGenerator: portageutils=PortageUtils(settings=settings) deps_all=portageutils.get_deps_for_package_building(pkgname) - deps_portage=portageutils.get_dep('sys-apps/portage',["RDEPEND"]) + deps_portage=portageutils.get_dep('portage',["RDEPEND"]) system_packages=portageutils.get_system_packages_list() - system_deps=portageutils.get_system_packages_rdeps() - allfiles=GentoolkitUtils().get_all_packages_files() + allfiles=GentoolkitUtils.get_all_packages_files() portage.util.writemsg("All files list recieved, waiting for " \ "a list of allowed files\n") - allowedpkgs=system_packages+system_deps - allowedpkgs+=list(deps_portage)+list(deps_all) - allowedpkgs+=["app-portage/autodep"] + allowedpkgs=system_packages+list(deps_portage)+list(deps_all) - allowedfiles=GentoolkitUtils().getfilesbypackages(allowedpkgs) + allowedfiles=GentoolkitUtils.getfilesbypackages(allowedpkgs) #for pkg in allowedpkgs: # allowedfiles+=GentoolkitUtils.getfilesbypackage(pkg) #import pdb; pdb.set_trace() # manually add all python interpreters to this list - allowedfiles+=GentoolkitUtils().getfilesbypackages(['python']) + allowedfiles+=GentoolkitUtils.getfilesbypackages(['python']) allowedfiles=set(allowedfiles) deniedfiles=allfiles-allowedfiles @@ -350,10 +304,9 @@ class EventsAnalyser: self.deps_all=self.portageutils.get_deps_for_package_building(pkgname) self.deps_direct=self.portageutils.get_dep(pkgname,["DEPEND"]) - self.deps_portage=self.portageutils.get_dep('sys-apps/portage',["RDEPEND"]) - + self.deps_portage=self.portageutils.get_dep('portage',["RDEPEND"]) + self.system_packages=self.portageutils.get_system_packages_list() - self.system_deps=self.portageutils.get_system_packages_rdeps() # All analyse work is here # get unique filenames @@ -365,7 +318,7 @@ class EventsAnalyser: filenames=filenames.union(fail_events) filenames=list(filenames) - file_to_package=GentoolkitUtils().getpackagesbyfiles(filenames) + file_to_package=GentoolkitUtils.getpackagesbyfiles(filenames) # This part is completly unreadable. # It converting one complex struct(returned by getfsevents) to another complex # struct which good for generating output. @@ -420,8 +373,7 @@ class EventsAnalyser: stagesorder={"clean":1,"setup":2,"unpack":3,"prepare":4,"configure":5,"compile":6,"test":7, "install":8,"preinst":9,"postinst":10,"prerm":11,"postrm":12,"unknown":13} packagesinfo=self.packagesinfo - # print information grouped by package - #print(packagesinfo.keys()) + # print information grouped by package for package in sorted(packagesinfo): # not showing special directory package if package=="directory": @@ -429,14 +381,12 @@ class EventsAnalyser: if package=="unknown": continue - + is_pkg_in_dep=package in self.deps_all is_pkg_in_portage_dep=package in self.deps_portage is_pkg_in_system=package in self.system_packages - is_pkg_in_system_dep=package in self.system_deps is_pkg_python="dev-lang/python" in package - is_pkg_self="app-portage/autodep" in package stages=[] for stage in sorted(packagesinfo[package].keys(), key=stagesorder.get): @@ -475,10 +425,6 @@ class EventsAnalyser: portage.util.writemsg("[SYSTEM]") elif is_pkg_in_portage_dep: portage.util.writemsg("[PORTAGE DEP]") - elif is_pkg_in_system_dep: - portage.util.writemsg("[SYSTEM DEP]") - elif is_pkg_self: - portage.util.writemsg("[AUTODEP]") elif is_pkg_python: portage.util.writemsg("[INTERPRETER]") elif not self.is_package_useful(package,stages,filenames.keys()): @@ -505,12 +451,7 @@ class EventsAnalyser: for filename in filenames: event_info=tuple(filenames[filename]) - try: - portage.util.writemsg( - " %-56s %-21s\n" % (filename.decode('utf-8'),action[event_info])) - except UnicodeDecodeError: - portage.util.writemsg( - " %-56s %-21s\n" % ('NON-UTF8-FILENAME',action[event_info])) + portage.util.writemsg(" %-56s %-21s\n" % (filename,action[event_info])) filescounter+=1 if filescounter>10: portage.util.writemsg(" ... and %d more ...\n" % (len(filenames)-10)) @@ -529,7 +470,7 @@ class EventsAnalyser: """ some basic heuristics here to cut part of packages """ excluded_paths=set( - [b'/etc/sandbox.d/'] + ['/etc/sandbox.d/'] ) excluded_packages=set( @@ -560,9 +501,8 @@ class EventsAnalyser: continue # test 1: package is not useful if all files are *.desktop or *.xml or *.m4 - if not (f.endswith(b".desktop") or f.endswith(b".xml") or - f.endswith(b".m4") or f.endswith(b".pc")): - break + if not (f.endswith(".desktop") or f.endswith(".xml") or f.endswith(".m4") or f.endswith(".pc")): + break else: return False # we get here if cycle ends not with break diff --git a/portage_with_autodep/pym/_emerge/EventsLogger.py b/portage_with_autodep/pym/_emerge/EventsLogger.py index 1ade9fd..68b3c67 100644 --- a/portage_with_autodep/pym/_emerge/EventsLogger.py +++ b/portage_with_autodep/pym/_emerge/EventsLogger.py @@ -100,69 +100,62 @@ class EventsLogger(threading.Thread): continue #import pdb; pdb.set_trace() - #try: - message=record.split(b"\0") - #except UnicodeDecodeError: - # print("Bad message %s" % record) - # continue + try: + message=record.decode("utf8").split("\0") + except UnicodeDecodeError: + print("Bad message %s" % record) + continue # continue #print(message) try: - eventname,filename,stage,result=message[1:5] - eventname=eventname.decode("utf-8") - stage=stage.decode("utf-8") - result=result.decode("utf-8") - except IndexError: - print("IndexError while parsing %s" % record) - except ValueError: - print("ValueError while parsing %s" % record) - except UnicodeDecodeError: - print("UnicodeDecodeError while parsing %s" % record) - - if result=="ASKING": - if self.filter_proc(eventname,filename,stage): - s.sendall(b"ALLOW\0") + if message[4]=="ASKING": + if self.filter_proc(message[1],message[2],message[3]): + s.sendall(b"ALLOW\0") + else: + # TODO: log through portage infrastructure + #print("Blocking an access to %s" % message[2]) + s.sendall(b"DENY\0") else: - # TODO: log through portage infrastructure - #print("Blocking an access to %s" % message[2]) - s.sendall(b"DENY\0") - else: - if not stage in self.events: - self.events[stage]=[{},{}] - - hashofsucesses=self.events[stage][0] - hashoffailures=self.events[stage][1] - - if result=="DENIED": - print("Blocking an access to %s" % filename) - - if result=="OK": - if not filename in hashofsucesses: - hashofsucesses[filename]=[False,False] - - readed_or_writed=hashofsucesses[filename] - - if eventname=="read": - readed_or_writed[0]=True - elif eventname=="write": - readed_or_writed[1]=True - - elif result[0:3]=="ERR" or result=="DENIED": - if not filename in hashoffailures: - hashoffailures[filename]=[False,False] - notfound_or_blocked=hashoffailures[filename] - - if result=="ERR/2": - notfound_or_blocked[0]=True - elif result=="DENIED": - notfound_or_blocked[1]=True + eventname,filename,stage,result=message[1:5] - else: - print("Error in logger module<->analyser protocol") + if not stage in self.events: + self.events[stage]=[{},{}] + hashofsucesses=self.events[stage][0] + hashoffailures=self.events[stage][1] + + if result=="DENIED": + print("Blocking an access to %s" % filename) + + if result=="OK": + if not filename in hashofsucesses: + hashofsucesses[filename]=[False,False] + + readed_or_writed=hashofsucesses[filename] + + if eventname=="read": + readed_or_writed[0]=True + elif eventname=="write": + readed_or_writed[1]=True + + elif result[0:3]=="ERR" or result=="DENIED": + if not filename in hashoffailures: + hashoffailures[filename]=[False,False] + notfound_or_blocked=hashoffailures[filename] + + if result=="ERR/2": + notfound_or_blocked[0]=True + elif result=="DENIED": + notfound_or_blocked[1]=True + + else: + print("Error in logger module<->analyser protocol") + + except IndexError: + print("IndexError while parsing %s" % record) except IOError as e: if e.errno!=4: # handling "Interrupted system call" errors raise @@ -184,5 +177,4 @@ class EventsLogger(threading.Thread): # We assume portage clears tmp folder, so no deleting a socket file # We assume that no new socket data will arrive after this moment - #print(self.events) return self.events diff --git a/portage_with_autodep/pym/_emerge/FakeVartree.py b/portage_with_autodep/pym/_emerge/FakeVartree.py index a11966f..d4dbe97 100644 --- a/portage_with_autodep/pym/_emerge/FakeVartree.py +++ b/portage_with_autodep/pym/_emerge/FakeVartree.py @@ -2,6 +2,7 @@ # Distributed under the terms of the GNU General Public License v2 import sys +import warnings import portage from portage import os @@ -37,8 +38,10 @@ class FakeVartree(vartree): global updates are necessary (updates are performed when necessary if there is not a matching ebuild in the tree). Instances of this class are not populated until the sync() method is called.""" - def __init__(self, root_config, pkg_cache=None, pkg_root_config=None): + def __init__(self, root_config, pkg_cache=None, pkg_root_config=None, + dynamic_deps=True): self._root_config = root_config + self._dynamic_deps = dynamic_deps if pkg_root_config is None: pkg_root_config = self._root_config self._pkg_root_config = pkg_root_config @@ -47,7 +50,6 @@ class FakeVartree(vartree): real_vartree = root_config.trees["vartree"] self._real_vardb = real_vartree.dbapi portdb = root_config.trees["porttree"].dbapi - self.root = real_vartree.root self.settings = real_vartree.settings mykeys = list(real_vartree.dbapi._aux_cache_keys) if "_mtime_" not in mykeys: @@ -55,19 +57,30 @@ class FakeVartree(vartree): self._db_keys = mykeys self._pkg_cache = pkg_cache self.dbapi = FakeVardbapi(real_vartree.settings) + self.dbapi._aux_cache_keys = set(self._db_keys) # Initialize variables needed for lazy cache pulls of the live ebuild # metadata. This ensures that the vardb lock is released ASAP, without # being delayed in case cache generation is triggered. self._aux_get = self.dbapi.aux_get - self.dbapi.aux_get = self._aux_get_wrapper self._match = self.dbapi.match - self.dbapi.match = self._match_wrapper + if dynamic_deps: + self.dbapi.aux_get = self._aux_get_wrapper + self.dbapi.match = self._match_wrapper self._aux_get_history = set() self._portdb_keys = ["EAPI", "DEPEND", "RDEPEND", "PDEPEND"] self._portdb = portdb self._global_updates = None + @property + def root(self): + warnings.warn("The root attribute of " + "_emerge.FakeVartree.FakeVartree" + " is deprecated. Use " + "settings['ROOT'] instead.", + DeprecationWarning, stacklevel=3) + return self.settings['ROOT'] + def _match_wrapper(self, cpv, use_cache=1): """ Make sure the metadata in Package instances gets updated for any @@ -147,15 +160,14 @@ class FakeVartree(vartree): self.dbapi.aux_get = self._aux_get self.settings._populate_treeVirtuals_if_needed(self) finally: - self.dbapi.aux_get = self._aux_get_wrapper + if self._dynamic_deps: + self.dbapi.aux_get = self._aux_get_wrapper def _sync(self): real_vardb = self._root_config.trees["vartree"].dbapi current_cpv_set = frozenset(real_vardb.cpv_all()) pkg_vardb = self.dbapi - pkg_cache = self._pkg_cache - aux_get_history = self._aux_get_history # Remove any packages that have been uninstalled. for pkg in list(pkg_vardb): diff --git a/portage_with_autodep/pym/_emerge/FakeVartree.pyo b/portage_with_autodep/pym/_emerge/FakeVartree.pyo Binary files differnew file mode 100644 index 0000000..8707391 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/FakeVartree.pyo diff --git a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py index a716dac..fcc4ab4 100644 --- a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py +++ b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage import os @@ -15,14 +15,14 @@ class FifoIpcDaemon(AbstractPollTask): def _start(self): self._files = self._files_dict() - input_fd = os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) # File streams are in unbuffered mode since we do atomic # read and write of whole pickles. - self._files.pipe_in = os.fdopen(input_fd, 'rb', 0) + self._files.pipe_in = \ + os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) self._reg_id = self.scheduler.register( - self._files.pipe_in.fileno(), + self._files.pipe_in, self._registered_events, self._input_handler) self._registered = True @@ -32,12 +32,12 @@ class FifoIpcDaemon(AbstractPollTask): Re-open the input stream, in order to suppress POLLHUP events (bug #339976). """ - self._files.pipe_in.close() - input_fd = os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) - self._files.pipe_in = os.fdopen(input_fd, 'rb', 0) self.scheduler.unregister(self._reg_id) + os.close(self._files.pipe_in) + self._files.pipe_in = \ + os.open(self.input_fifo, os.O_RDONLY|os.O_NONBLOCK) self._reg_id = self.scheduler.register( - self._files.pipe_in.fileno(), + self._files.pipe_in, self._registered_events, self._input_handler) def isAlive(self): @@ -51,14 +51,9 @@ class FifoIpcDaemon(AbstractPollTask): def _wait(self): if self.returncode is not None: return self.returncode - - if self._registered: - self.scheduler.schedule(self._reg_id) - self._unregister() - + self._wait_loop() if self.returncode is None: self.returncode = os.EX_OK - return self.returncode def _input_handler(self, fd, event): @@ -77,5 +72,5 @@ class FifoIpcDaemon(AbstractPollTask): if self._files is not None: for f in self._files.values(): - f.close() + os.close(f) self._files = None diff --git a/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo Binary files differnew file mode 100644 index 0000000..6d7c4f9 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/FifoIpcDaemon.pyo diff --git a/portage_with_autodep/pym/_emerge/JobStatusDisplay.py b/portage_with_autodep/pym/_emerge/JobStatusDisplay.py index 1949232..5b9b221 100644 --- a/portage_with_autodep/pym/_emerge/JobStatusDisplay.py +++ b/portage_with_autodep/pym/_emerge/JobStatusDisplay.py @@ -97,7 +97,7 @@ class JobStatusDisplay(object): """ Initialize term control codes. @rtype: bool - @returns: True if term codes were successfully initialized, + @return: True if term codes were successfully initialized, False otherwise. """ @@ -209,24 +209,26 @@ class JobStatusDisplay(object): def display(self): """ Display status on stdout, but only if something has - changed since the last call. + changed since the last call. This always returns True, + for continuous scheduling via timeout_add. """ if self.quiet: - return + return True current_time = time.time() time_delta = current_time - self._last_display_time if self._displayed and \ not self._changed: if not self._isatty: - return + return True if time_delta < self._min_display_latency: - return + return True self._last_display_time = current_time self._changed = False self._display_status() + return True def _display_status(self): # Don't use len(self._completed_tasks) here since that also @@ -289,4 +291,11 @@ class JobStatusDisplay(object): self._update(color_output.getvalue()) if self.xterm_titles: - xtermTitle(" ".join(plain_output.split())) + # If the HOSTNAME variable is exported, include it + # in the xterm title, just like emergelog() does. + # See bug #390699. + title_str = " ".join(plain_output.split()) + hostname = os.environ.get("HOSTNAME") + if hostname is not None: + title_str = "%s: %s" % (hostname, title_str) + xtermTitle(title_str) diff --git a/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo b/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo Binary files differnew file mode 100644 index 0000000..f79b2c2 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/JobStatusDisplay.pyo diff --git a/portage_with_autodep/pym/_emerge/MergeListItem.py b/portage_with_autodep/pym/_emerge/MergeListItem.py index 2176bf6..8086c68 100644 --- a/portage_with_autodep/pym/_emerge/MergeListItem.py +++ b/portage_with_autodep/pym/_emerge/MergeListItem.py @@ -68,7 +68,7 @@ class MergeListItem(CompositeTask): pkg_repo_name = "unknown repo" msg += " from %s" % pkg_repo_name - if pkg.root != "/": + if pkg.root_config.settings["ROOT"] != "/": msg += " %s %s" % (preposition, pkg.root) if not build_opts.pretend: diff --git a/portage_with_autodep/pym/_emerge/MergeListItem.pyo b/portage_with_autodep/pym/_emerge/MergeListItem.pyo Binary files differnew file mode 100644 index 0000000..168a227 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/MergeListItem.pyo diff --git a/portage_with_autodep/pym/_emerge/MetadataRegen.py b/portage_with_autodep/pym/_emerge/MetadataRegen.py index 8103175..e82015f 100644 --- a/portage_with_autodep/pym/_emerge/MetadataRegen.py +++ b/portage_with_autodep/pym/_emerge/MetadataRegen.py @@ -1,8 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage from portage import os +from portage.dep import _repo_separator from _emerge.EbuildMetadataPhase import EbuildMetadataPhase from _emerge.PollScheduler import PollScheduler @@ -10,7 +11,7 @@ class MetadataRegen(PollScheduler): def __init__(self, portdb, cp_iter=None, consumer=None, max_jobs=None, max_load=None): - PollScheduler.__init__(self) + PollScheduler.__init__(self, main=True) self._portdb = portdb self._global_cleanse = False if cp_iter is None: @@ -33,10 +34,11 @@ class MetadataRegen(PollScheduler): self.returncode = os.EX_OK self._error_count = 0 self._running_tasks = set() + self._remaining_tasks = True def _terminate_tasks(self): - while self._running_tasks: - self._running_tasks.pop().cancel() + for task in list(self._running_tasks): + task.cancel() def _iter_every_cp(self): portage.writemsg_stdout("Listing available packages...\n") @@ -60,27 +62,32 @@ class MetadataRegen(PollScheduler): break cp_set.add(cp) portage.writemsg_stdout("Processing %s\n" % cp) - cpv_list = portdb.cp_list(cp) - for cpv in cpv_list: - if self._terminated_tasks: - break - valid_pkgs.add(cpv) - ebuild_path, repo_path = portdb.findname2(cpv) - if ebuild_path is None: - raise AssertionError("ebuild not found for '%s'" % cpv) - metadata, st, emtime = portdb._pull_valid_cache( - cpv, ebuild_path, repo_path) - if metadata is not None: - if consumer is not None: - consumer(cpv, ebuild_path, - repo_path, metadata) - continue - - yield EbuildMetadataPhase(cpv=cpv, ebuild_path=ebuild_path, - ebuild_mtime=emtime, - metadata_callback=portdb._metadata_callback, - portdb=portdb, repo_path=repo_path, - settings=portdb.doebuild_settings) + # We iterate over portdb.porttrees, since it's common to + # tweak this attribute in order to adjust repo selection. + for mytree in portdb.porttrees: + repo = portdb.repositories.get_repo_for_location(mytree) + cpv_list = portdb.cp_list(cp, mytree=[repo.location]) + for cpv in cpv_list: + if self._terminated_tasks: + break + valid_pkgs.add(cpv) + ebuild_path, repo_path = portdb.findname2(cpv, myrepo=repo.name) + if ebuild_path is None: + raise AssertionError("ebuild not found for '%s%s%s'" % (cpv, _repo_separator, repo.name)) + metadata, ebuild_hash = portdb._pull_valid_cache( + cpv, ebuild_path, repo_path) + if metadata is not None: + if consumer is not None: + consumer(cpv, repo_path, metadata, ebuild_hash, True) + continue + + yield EbuildMetadataPhase(cpv=cpv, + ebuild_hash=ebuild_hash, + portdb=portdb, repo_path=repo_path, + settings=portdb.doebuild_settings) + + def _keep_scheduling(self): + return self._remaining_tasks and not self._terminated_tasks def run(self): @@ -88,11 +95,7 @@ class MetadataRegen(PollScheduler): from portage.cache.cache_errors import CacheError dead_nodes = {} - while self._schedule(): - self._poll_loop() - - while self._jobs: - self._poll_loop() + self._main_loop() if self._terminated_tasks: self.returncode = 1 @@ -140,26 +143,21 @@ class MetadataRegen(PollScheduler): pass def _schedule_tasks(self): - """ - @rtype: bool - @returns: True if there may be remaining tasks to schedule, - False otherwise. - """ if self._terminated_tasks: - return False + return while self._can_add_job(): try: metadata_process = next(self._process_iter) except StopIteration: - return False + self._remaining_tasks = False + return self._jobs += 1 self._running_tasks.add(metadata_process) metadata_process.scheduler = self.sched_iface metadata_process.addExitListener(self._metadata_exit) metadata_process.start() - return True def _metadata_exit(self, metadata_process): self._jobs -= 1 @@ -176,9 +174,10 @@ class MetadataRegen(PollScheduler): # On failure, still notify the consumer (in this case the metadata # argument is None). self._consumer(metadata_process.cpv, - metadata_process.ebuild_path, metadata_process.repo_path, - metadata_process.metadata) + metadata_process.metadata, + metadata_process.ebuild_hash, + metadata_process.eapi_supported) self._schedule() diff --git a/portage_with_autodep/pym/_emerge/MetadataRegen.pyo b/portage_with_autodep/pym/_emerge/MetadataRegen.pyo Binary files differnew file mode 100644 index 0000000..6c8788f --- /dev/null +++ b/portage_with_autodep/pym/_emerge/MetadataRegen.pyo diff --git a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py index ce0ab14..afa44fb 100644 --- a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py +++ b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.py @@ -29,5 +29,11 @@ class MiscFunctionsProcess(AbstractEbuildProcess): AbstractEbuildProcess._start(self) def _spawn(self, args, **kwargs): - self.settings.pop("EBUILD_PHASE", None) - return spawn(" ".join(args), self.settings, **kwargs) + # Temporarily unset EBUILD_PHASE so that bashrc code doesn't + # think this is a real phase. + phase_backup = self.settings.pop("EBUILD_PHASE", None) + try: + return spawn(" ".join(args), self.settings, **kwargs) + finally: + if phase_backup is not None: + self.settings["EBUILD_PHASE"] = phase_backup diff --git a/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo Binary files differnew file mode 100644 index 0000000..e3f5344 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/MiscFunctionsProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/Package.py b/portage_with_autodep/pym/_emerge/Package.py index 20c72b4..c04fa1f 100644 --- a/portage_with_autodep/pym/_emerge/Package.py +++ b/portage_with_autodep/pym/_emerge/Package.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -9,9 +9,9 @@ from portage.cache.mappings import slot_dict_class from portage.const import EBUILD_PHASES from portage.dep import Atom, check_required_use, use_reduce, \ paren_enclose, _slot_re, _slot_separator, _repo_separator +from portage.versions import _pkg_str, _unknown_repo from portage.eapi import eapi_has_iuse_defaults, eapi_has_required_use from portage.exception import InvalidDependString -from portage.repository.config import _gen_valid_repo from _emerge.Task import Task if sys.hexversion >= 0x3000000: @@ -26,7 +26,7 @@ class Package(Task): "root_config", "type_name", "category", "counter", "cp", "cpv_split", "inherited", "invalid", "iuse", "masks", "mtime", - "pf", "pv_split", "root", "slot", "slot_atom", "visible",) + \ + "pf", "root", "slot", "slot_atom", "version", "visible",) + \ ("_raw_metadata", "_use",) metadata_keys = [ @@ -38,7 +38,7 @@ class Package(Task): _dep_keys = ('DEPEND', 'PDEPEND', 'RDEPEND',) _use_conditional_misc_keys = ('LICENSE', 'PROPERTIES', 'RESTRICT') - UNKNOWN_REPO = "__unknown__" + UNKNOWN_REPO = _unknown_repo def __init__(self, **kwargs): Task.__init__(self, **kwargs) @@ -49,7 +49,6 @@ class Package(Task): self.metadata = _PackageMetadataWrapper(self, self._raw_metadata) if not self.built: self.metadata['CHOST'] = self.root_config.settings.get('CHOST', '') - self.cp = portage.cpv_getkey(self.cpv) slot = self.slot if _slot_re.match(slot) is None: self._invalid_metadata('SLOT.invalid', @@ -57,6 +56,11 @@ class Package(Task): # Avoid an InvalidAtom exception when creating slot_atom. # This package instance will be masked due to empty SLOT. slot = '0' + self.cpv = _pkg_str(self.cpv, slot=slot, + repo=self.metadata.get('repository', '')) + self.cp = self.cpv.cp + # sync metadata with validated repo (may be UNKNOWN_REPO) + self.metadata['repository'] = self.cpv.repo if (self.iuse.enabled or self.iuse.disabled) and \ not eapi_has_iuse_defaults(self.metadata["EAPI"]): if not self.installed: @@ -64,14 +68,10 @@ class Package(Task): "IUSE contains defaults, but EAPI doesn't allow them") self.slot_atom = portage.dep.Atom("%s%s%s" % (self.cp, _slot_separator, slot)) self.category, self.pf = portage.catsplit(self.cpv) - self.cpv_split = portage.catpkgsplit(self.cpv) - self.pv_split = self.cpv_split[1:] + self.cpv_split = self.cpv.cpv_split + self.version = self.cpv.version if self.inherited is None: self.inherited = frozenset() - repo = _gen_valid_repo(self.metadata.get('repository', '')) - if not repo: - repo = self.UNKNOWN_REPO - self.metadata['repository'] = repo self._validate_deps() self.masks = self._masks() @@ -84,7 +84,7 @@ class Package(Task): self._hash_key = Package._gen_hash_key(cpv=self.cpv, installed=self.installed, onlydeps=self.onlydeps, - operation=self.operation, repo_name=repo, + operation=self.operation, repo_name=self.cpv.repo, root_config=self.root_config, type_name=self.type_name) self._hash_value = hash(self._hash_key) @@ -239,11 +239,6 @@ class Package(Task): if mask_atom is not None: masks['package.mask'] = mask_atom - system_mask = settings._getProfileMaskAtom( - self.cpv, self.metadata) - if system_mask is not None: - masks['profile.system'] = system_mask - try: missing_licenses = settings._getMissingLicenses( self.cpv, self.metadata) @@ -276,7 +271,6 @@ class Package(Task): return False if 'package.mask' in masks or \ - 'profile.system' in masks or \ 'LICENSE' in masks: return False @@ -367,15 +361,15 @@ class Package(Task): % (portage.output.colorize(cpv_color, self.cpv + _repo_separator + self.repo) , self.type_name) if self.type_name == "installed": - if self.root != "/": - s += " in '%s'" % self.root + if self.root_config.settings['ROOT'] != "/": + s += " in '%s'" % self.root_config.settings['ROOT'] if self.operation == "uninstall": s += " scheduled for uninstall" else: if self.operation == "merge": s += " scheduled for merge" - if self.root != "/": - s += " to '%s'" % self.root + if self.root_config.settings['ROOT'] != "/": + s += " to '%s'" % self.root_config.settings['ROOT'] s += ")" return s @@ -497,7 +491,7 @@ class Package(Task): def is_valid_flag(self, flags): """ - @returns: True if all flags are valid USE values which may + @return: True if all flags are valid USE values which may be specified in USE dependencies, False otherwise. """ if isinstance(flags, basestring): @@ -511,7 +505,7 @@ class Package(Task): def get_missing_iuse(self, flags): """ - @returns: A list of flags missing from IUSE. + @return: A list of flags missing from IUSE. """ if isinstance(flags, basestring): flags = [flags] @@ -535,28 +529,28 @@ class Package(Task): def __lt__(self, other): if other.cp != self.cp: return False - if portage.pkgcmp(self.pv_split, other.pv_split) < 0: + if portage.vercmp(self.version, other.version) < 0: return True return False def __le__(self, other): if other.cp != self.cp: return False - if portage.pkgcmp(self.pv_split, other.pv_split) <= 0: + if portage.vercmp(self.version, other.version) <= 0: return True return False def __gt__(self, other): if other.cp != self.cp: return False - if portage.pkgcmp(self.pv_split, other.pv_split) > 0: + if portage.vercmp(self.version, other.version) > 0: return True return False def __ge__(self, other): if other.cp != self.cp: return False - if portage.pkgcmp(self.pv_split, other.pv_split) >= 0: + if portage.vercmp(self.version, other.version) >= 0: return True return False diff --git a/portage_with_autodep/pym/_emerge/Package.pyo b/portage_with_autodep/pym/_emerge/Package.pyo Binary files differnew file mode 100644 index 0000000..3d37317 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/Package.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageArg.pyo b/portage_with_autodep/pym/_emerge/PackageArg.pyo Binary files differnew file mode 100644 index 0000000..c50e145 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/PackageArg.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageMerge.py b/portage_with_autodep/pym/_emerge/PackageMerge.py index f8fa04a..eed34e9 100644 --- a/portage_with_autodep/pym/_emerge/PackageMerge.py +++ b/portage_with_autodep/pym/_emerge/PackageMerge.py @@ -28,7 +28,7 @@ class PackageMerge(CompositeTask): counter_str, colorize("GOOD", pkg.cpv)) - if pkg.root != "/": + if pkg.root_config.settings["ROOT"] != "/": msg += " %s %s" % (preposition, pkg.root) if not self.merge.build_opts.fetchonly and \ diff --git a/portage_with_autodep/pym/_emerge/PackageMerge.pyo b/portage_with_autodep/pym/_emerge/PackageMerge.pyo Binary files differnew file mode 100644 index 0000000..6403d1b --- /dev/null +++ b/portage_with_autodep/pym/_emerge/PackageMerge.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageUninstall.pyo b/portage_with_autodep/pym/_emerge/PackageUninstall.pyo Binary files differnew file mode 100644 index 0000000..847c749 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/PackageUninstall.pyo diff --git a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py index a692bb6..0f7be44 100644 --- a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py +++ b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.py @@ -1,8 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys from portage.dbapi import dbapi +from portage.dbapi.dep_expand import dep_expand class PackageVirtualDbapi(dbapi): """ @@ -76,20 +77,24 @@ class PackageVirtualDbapi(dbapi): self._match_cache = {} def match(self, origdep, use_cache=1): - result = self._match_cache.get(origdep) + atom = dep_expand(origdep, mydb=self, settings=self.settings) + cache_key = (atom, atom.unevaluated_atom) + result = self._match_cache.get(cache_key) if result is not None: return result[:] - result = dbapi.match(self, origdep, use_cache=use_cache) - self._match_cache[origdep] = result + result = list(self._iter_match(atom, self.cp_list(atom.cp))) + self._match_cache[cache_key] = result return result[:] def cpv_exists(self, cpv, myrepo=None): return cpv in self._cpv_map def cp_list(self, mycp, use_cache=1): - cachelist = self._match_cache.get(mycp) - # cp_list() doesn't expand old-style virtuals - if cachelist and cachelist[0].startswith(mycp): + # NOTE: Cache can be safely shared with the match cache, since the + # match cache uses the result from dep_expand for the cache_key. + cache_key = (mycp, mycp) + cachelist = self._match_cache.get(cache_key) + if cachelist is not None: return cachelist[:] cpv_list = self._cp_map.get(mycp) if cpv_list is None: @@ -97,8 +102,7 @@ class PackageVirtualDbapi(dbapi): else: cpv_list = [pkg.cpv for pkg in cpv_list] self._cpv_sort_ascending(cpv_list) - if not (not cpv_list and mycp.startswith("virtual/")): - self._match_cache[mycp] = cpv_list + self._match_cache[cache_key] = cpv_list return cpv_list[:] def cp_all(self): diff --git a/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo Binary files differnew file mode 100644 index 0000000..a1a850f --- /dev/null +++ b/portage_with_autodep/pym/_emerge/PackageVirtualDbapi.pyo diff --git a/portage_with_autodep/pym/_emerge/PipeReader.py b/portage_with_autodep/pym/_emerge/PipeReader.py index 375c98f..90febdf 100644 --- a/portage_with_autodep/pym/_emerge/PipeReader.py +++ b/portage_with_autodep/pym/_emerge/PipeReader.py @@ -1,11 +1,9 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage import os from _emerge.AbstractPollTask import AbstractPollTask -from _emerge.PollConstants import PollConstants import fcntl -import array class PipeReader(AbstractPollTask): @@ -17,16 +15,22 @@ class PipeReader(AbstractPollTask): """ __slots__ = ("input_files",) + \ - ("_read_data", "_reg_ids") + ("_read_data", "_reg_ids", "_use_array") def _start(self): self._reg_ids = set() self._read_data = [] - for k, f in self.input_files.items(): + + if self._use_array: + output_handler = self._array_output_handler + else: + output_handler = self._output_handler + + for f in self.input_files.values(): fcntl.fcntl(f.fileno(), fcntl.F_SETFL, fcntl.fcntl(f.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) self._reg_ids.add(self.scheduler.register(f.fileno(), - self._registered_events, self._output_handler)) + self._registered_events, output_handler)) self._registered = True def isAlive(self): @@ -39,11 +43,7 @@ class PipeReader(AbstractPollTask): def _wait(self): if self.returncode is not None: return self.returncode - - if self._registered: - self.scheduler.schedule(self._reg_ids) - self._unregister() - + self._wait_loop() self.returncode = os.EX_OK return self.returncode @@ -57,26 +57,42 @@ class PipeReader(AbstractPollTask): def _output_handler(self, fd, event): - if event & PollConstants.POLLIN: + while True: + data = self._read_buf(fd, event) + if data is None: + break + if data: + self._read_data.append(data) + else: + self._unregister() + self.wait() + break + + self._unregister_if_appropriate(event) + + return True - for f in self.input_files.values(): - if fd == f.fileno(): - break + def _array_output_handler(self, fd, event): - buf = array.array('B') - try: - buf.fromfile(f, self._bufsize) - except (EOFError, IOError): - pass + for f in self.input_files.values(): + if f.fileno() == fd: + break - if buf: - self._read_data.append(buf.tostring()) + while True: + data = self._read_array(f, event) + if data is None: + break + if data: + self._read_data.append(data) else: self._unregister() self.wait() + break self._unregister_if_appropriate(event) + return True + def _unregister(self): """ Unregister from the scheduler and close open files. diff --git a/portage_with_autodep/pym/_emerge/PipeReader.pyo b/portage_with_autodep/pym/_emerge/PipeReader.pyo Binary files differnew file mode 100644 index 0000000..2f53e7d --- /dev/null +++ b/portage_with_autodep/pym/_emerge/PipeReader.pyo diff --git a/portage_with_autodep/pym/_emerge/PollScheduler.py b/portage_with_autodep/pym/_emerge/PollScheduler.py index a2b5c24..965dc20 100644 --- a/portage_with_autodep/pym/_emerge/PollScheduler.py +++ b/portage_with_autodep/pym/_emerge/PollScheduler.py @@ -1,11 +1,8 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import gzip import errno -import logging -import select -import time try: import threading @@ -15,36 +12,55 @@ except ImportError: from portage import _encodings from portage import _unicode_encode from portage.util import writemsg_level +from portage.util.SlotObject import SlotObject +from portage.util._eventloop.EventLoop import EventLoop +from portage.util._eventloop.global_event_loop import global_event_loop -from _emerge.SlotObject import SlotObject from _emerge.getloadavg import getloadavg -from _emerge.PollConstants import PollConstants -from _emerge.PollSelectAdapter import PollSelectAdapter class PollScheduler(object): class _sched_iface_class(SlotObject): - __slots__ = ("output", "register", "schedule", "unregister") + __slots__ = ("IO_ERR", "IO_HUP", "IO_IN", "IO_NVAL", "IO_OUT", + "IO_PRI", "child_watch_add", + "idle_add", "io_add_watch", "iteration", + "output", "register", "run", + "source_remove", "timeout_add", "unregister") - def __init__(self): + def __init__(self, main=False): + """ + @param main: If True then use global_event_loop(), otherwise use + a local EventLoop instance (default is False, for safe use in + a non-main thread) + @type main: bool + """ self._terminated = threading.Event() self._terminated_tasks = False self._max_jobs = 1 self._max_load = None self._jobs = 0 - self._poll_event_queue = [] - self._poll_event_handlers = {} - self._poll_event_handler_ids = {} - # Increment id for each new handler. - self._event_handler_id = 0 - self._poll_obj = create_poll_instance() self._scheduling = False self._background = False + if main: + self._event_loop = global_event_loop() + else: + self._event_loop = EventLoop(main=False) self.sched_iface = self._sched_iface_class( + IO_ERR=self._event_loop.IO_ERR, + IO_HUP=self._event_loop.IO_HUP, + IO_IN=self._event_loop.IO_IN, + IO_NVAL=self._event_loop.IO_NVAL, + IO_OUT=self._event_loop.IO_OUT, + IO_PRI=self._event_loop.IO_PRI, + child_watch_add=self._event_loop.child_watch_add, + idle_add=self._event_loop.idle_add, + io_add_watch=self._event_loop.io_add_watch, + iteration=self._event_loop.iteration, output=self._task_output, - register=self._register, - schedule=self._schedule_wait, - unregister=self._unregister) + register=self._event_loop.io_add_watch, + source_remove=self._event_loop.source_remove, + timeout_add=self._event_loop.timeout_add, + unregister=self._event_loop.source_remove) def terminate(self): """ @@ -55,17 +71,47 @@ class PollScheduler(object): """ self._terminated.set() + def _termination_check(self): + """ + Calls _terminate_tasks() if appropriate. It's guaranteed not to + call it while _schedule_tasks() is being called. The check should + be executed for each iteration of the event loop, for response to + termination signals at the earliest opportunity. It always returns + True, for continuous scheduling via idle_add. + """ + if not self._scheduling and \ + self._terminated.is_set() and \ + not self._terminated_tasks: + self._scheduling = True + try: + self._terminated_tasks = True + self._terminate_tasks() + finally: + self._scheduling = False + return True + def _terminate_tasks(self): """ Send signals to terminate all tasks. This is called once - from self._schedule() in the event dispatching thread. This - prevents it from being called while the _schedule_tasks() + from _keep_scheduling() or _is_work_scheduled() in the event + dispatching thread. It will not be called while the _schedule_tasks() implementation is running, in order to avoid potential interference. All tasks should be cleaned up at the earliest opportunity, but not necessarily before this method returns. + Typically, this method will send kill signals and return without + waiting for exit status. This allows basic cleanup to occur, such as + flushing of buffered output to logs. """ raise NotImplementedError() + def _keep_scheduling(self): + """ + @rtype: bool + @return: True if there may be remaining tasks to schedule, + False otherwise. + """ + return False + def _schedule_tasks(self): """ This is called from inside the _schedule() method, which @@ -79,10 +125,10 @@ class PollScheduler(object): Unless this method is used to perform user interface updates, or something like that, the first thing it should do is check the state of _terminated_tasks and if that is True then it - should return False immediately (since there's no need to + should return immediately (since there's no need to schedule anything after _terminate_tasks() has been called). """ - raise NotImplementedError() + pass def _schedule(self): """ @@ -95,15 +141,32 @@ class PollScheduler(object): return False self._scheduling = True try: + self._schedule_tasks() + finally: + self._scheduling = False - if self._terminated.is_set() and \ - not self._terminated_tasks: - self._terminated_tasks = True - self._terminate_tasks() + def _main_loop(self): + term_check_id = self.sched_iface.idle_add(self._termination_check) + try: + # Populate initial event sources. We only need to do + # this once here, since it can be called during the + # loop from within event handlers. + self._schedule() + + # Loop while there are jobs to be scheduled. + while self._keep_scheduling(): + self.sched_iface.iteration() - return self._schedule_tasks() + # Clean shutdown of previously scheduled jobs. In the + # case of termination, this allows for basic cleanup + # such as flushing of buffered output to logs. + while self._is_work_scheduled(): + self.sched_iface.iteration() finally: - self._scheduling = False + self.sched_iface.source_remove(term_check_id) + + def _is_work_scheduled(self): + return bool(self._running_job_count()) def _running_job_count(self): return self._jobs @@ -132,183 +195,6 @@ class PollScheduler(object): return True - def _poll(self, timeout=None): - """ - All poll() calls pass through here. The poll events - are added directly to self._poll_event_queue. - In order to avoid endless blocking, this raises - StopIteration if timeout is None and there are - no file descriptors to poll. - """ - if not self._poll_event_handlers: - self._schedule() - if timeout is None and \ - not self._poll_event_handlers: - raise StopIteration( - "timeout is None and there are no poll() event handlers") - - # The following error is known to occur with Linux kernel versions - # less than 2.6.24: - # - # select.error: (4, 'Interrupted system call') - # - # This error has been observed after a SIGSTOP, followed by SIGCONT. - # Treat it similar to EAGAIN if timeout is None, otherwise just return - # without any events. - while True: - try: - self._poll_event_queue.extend(self._poll_obj.poll(timeout)) - break - except select.error as e: - writemsg_level("\n!!! select error: %s\n" % (e,), - level=logging.ERROR, noiselevel=-1) - del e - if timeout is not None: - break - - def _next_poll_event(self, timeout=None): - """ - Since the _schedule_wait() loop is called by event - handlers from _poll_loop(), maintain a central event - queue for both of them to share events from a single - poll() call. In order to avoid endless blocking, this - raises StopIteration if timeout is None and there are - no file descriptors to poll. - """ - if not self._poll_event_queue: - self._poll(timeout) - if not self._poll_event_queue: - raise StopIteration() - return self._poll_event_queue.pop() - - def _poll_loop(self): - - event_handlers = self._poll_event_handlers - event_handled = False - - try: - while event_handlers: - f, event = self._next_poll_event() - handler, reg_id = event_handlers[f] - handler(f, event) - event_handled = True - except StopIteration: - event_handled = True - - if not event_handled: - raise AssertionError("tight loop") - - def _schedule_yield(self): - """ - Schedule for a short period of time chosen by the scheduler based - on internal state. Synchronous tasks should call this periodically - in order to allow the scheduler to service pending poll events. The - scheduler will call poll() exactly once, without blocking, and any - resulting poll events will be serviced. - """ - event_handlers = self._poll_event_handlers - events_handled = 0 - - if not event_handlers: - return bool(events_handled) - - if not self._poll_event_queue: - self._poll(0) - - try: - while event_handlers and self._poll_event_queue: - f, event = self._next_poll_event() - handler, reg_id = event_handlers[f] - handler(f, event) - events_handled += 1 - except StopIteration: - events_handled += 1 - - return bool(events_handled) - - def _register(self, f, eventmask, handler): - """ - @rtype: Integer - @return: A unique registration id, for use in schedule() or - unregister() calls. - """ - if f in self._poll_event_handlers: - raise AssertionError("fd %d is already registered" % f) - self._event_handler_id += 1 - reg_id = self._event_handler_id - self._poll_event_handler_ids[reg_id] = f - self._poll_event_handlers[f] = (handler, reg_id) - self._poll_obj.register(f, eventmask) - return reg_id - - def _unregister(self, reg_id): - f = self._poll_event_handler_ids[reg_id] - self._poll_obj.unregister(f) - if self._poll_event_queue: - # Discard any unhandled events that belong to this file, - # in order to prevent these events from being erroneously - # delivered to a future handler that is using a reallocated - # file descriptor of the same numeric value (causing - # extremely confusing bugs). - remaining_events = [] - discarded_events = False - for event in self._poll_event_queue: - if event[0] == f: - discarded_events = True - else: - remaining_events.append(event) - - if discarded_events: - self._poll_event_queue[:] = remaining_events - - del self._poll_event_handlers[f] - del self._poll_event_handler_ids[reg_id] - - def _schedule_wait(self, wait_ids=None, timeout=None, condition=None): - """ - Schedule until wait_id is not longer registered - for poll() events. - @type wait_id: int - @param wait_id: a task id to wait for - """ - event_handlers = self._poll_event_handlers - handler_ids = self._poll_event_handler_ids - event_handled = False - - if isinstance(wait_ids, int): - wait_ids = frozenset([wait_ids]) - - start_time = None - remaining_timeout = timeout - timed_out = False - if timeout is not None: - start_time = time.time() - try: - while (wait_ids is None and event_handlers) or \ - (wait_ids is not None and wait_ids.intersection(handler_ids)): - f, event = self._next_poll_event(timeout=remaining_timeout) - handler, reg_id = event_handlers[f] - handler(f, event) - event_handled = True - if condition is not None and condition(): - break - if timeout is not None: - elapsed_time = time.time() - start_time - if elapsed_time < 0: - # The system clock has changed such that start_time - # is now in the future, so just assume that the - # timeout has already elapsed. - timed_out = True - break - remaining_timeout = timeout - 1000 * elapsed_time - if remaining_timeout <= 0: - timed_out = True - break - except StopIteration: - event_handled = True - - return event_handled - def _task_output(self, msg, log_path=None, background=None, level=0, noiselevel=-1): """ @@ -333,6 +219,7 @@ class PollScheduler(object): f = open(_unicode_encode(log_path, encoding=_encodings['fs'], errors='strict'), mode='ab') + f_real = f except IOError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise @@ -349,50 +236,5 @@ class PollScheduler(object): f.write(_unicode_encode(msg)) f.close() - -_can_poll_device = None - -def can_poll_device(): - """ - Test if it's possible to use poll() on a device such as a pty. This - is known to fail on Darwin. - @rtype: bool - @returns: True if poll() on a device succeeds, False otherwise. - """ - - global _can_poll_device - if _can_poll_device is not None: - return _can_poll_device - - if not hasattr(select, "poll"): - _can_poll_device = False - return _can_poll_device - - try: - dev_null = open('/dev/null', 'rb') - except IOError: - _can_poll_device = False - return _can_poll_device - - p = select.poll() - p.register(dev_null.fileno(), PollConstants.POLLIN) - - invalid_request = False - for f, event in p.poll(): - if event & PollConstants.POLLNVAL: - invalid_request = True - break - dev_null.close() - - _can_poll_device = not invalid_request - return _can_poll_device - -def create_poll_instance(): - """ - Create an instance of select.poll, or an instance of - PollSelectAdapter there is no poll() implementation or - it is broken somehow. - """ - if can_poll_device(): - return select.poll() - return PollSelectAdapter() + if f_real is not f: + f_real.close() diff --git a/portage_with_autodep/pym/_emerge/PollScheduler.pyo b/portage_with_autodep/pym/_emerge/PollScheduler.pyo Binary files differnew file mode 100644 index 0000000..b7e52be --- /dev/null +++ b/portage_with_autodep/pym/_emerge/PollScheduler.pyo diff --git a/portage_with_autodep/pym/_emerge/ProgressHandler.pyo b/portage_with_autodep/pym/_emerge/ProgressHandler.pyo Binary files differnew file mode 100644 index 0000000..83e2f7f --- /dev/null +++ b/portage_with_autodep/pym/_emerge/ProgressHandler.pyo diff --git a/portage_with_autodep/pym/_emerge/QueueScheduler.py b/portage_with_autodep/pym/_emerge/QueueScheduler.py index a4ab328..206087c 100644 --- a/portage_with_autodep/pym/_emerge/QueueScheduler.py +++ b/portage_with_autodep/pym/_emerge/QueueScheduler.py @@ -1,8 +1,6 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import time - from _emerge.PollScheduler import PollScheduler class QueueScheduler(PollScheduler): @@ -12,8 +10,8 @@ class QueueScheduler(PollScheduler): run() method returns when no tasks remain. """ - def __init__(self, max_jobs=None, max_load=None): - PollScheduler.__init__(self) + def __init__(self, main=True, max_jobs=None, max_load=None): + PollScheduler.__init__(self, main=main) if max_jobs is None: max_jobs = 1 @@ -36,51 +34,44 @@ class QueueScheduler(PollScheduler): def run(self, timeout=None): - start_time = None - timed_out = False - remaining_timeout = timeout + timeout_callback = None if timeout is not None: - start_time = time.time() - - while self._schedule(): - self._schedule_wait(timeout=remaining_timeout) - if timeout is not None: - elapsed_time = time.time() - start_time - if elapsed_time < 0: - # The system clock has changed such that start_time - # is now in the future, so just assume that the - # timeout has already elapsed. - timed_out = True - break - remaining_timeout = timeout - 1000 * elapsed_time - if remaining_timeout <= 0: - timed_out = True + def timeout_callback(): + timeout_callback.timed_out = True + return False + timeout_callback.timed_out = False + timeout_callback.timeout_id = self.sched_iface.timeout_add( + timeout, timeout_callback) + + term_check_id = self.sched_iface.idle_add(self._termination_check) + try: + while not (timeout_callback is not None and + timeout_callback.timed_out): + # We don't have any callbacks to trigger _schedule(), + # so we have to call it explicitly here. + self._schedule() + if self._keep_scheduling(): + self.sched_iface.iteration() + else: break - if timeout is None or not timed_out: - while self._running_job_count(): - self._schedule_wait(timeout=remaining_timeout) - if timeout is not None: - elapsed_time = time.time() - start_time - if elapsed_time < 0: - # The system clock has changed such that start_time - # is now in the future, so just assume that the - # timeout has already elapsed. - timed_out = True - break - remaining_timeout = timeout - 1000 * elapsed_time - if remaining_timeout <= 0: - timed_out = True - break + while self._is_work_scheduled() and \ + not (timeout_callback is not None and + timeout_callback.timed_out): + self.sched_iface.iteration() + finally: + self.sched_iface.source_remove(term_check_id) + if timeout_callback is not None: + self.sched_iface.unregister(timeout_callback.timeout_id) def _schedule_tasks(self): """ @rtype: bool - @returns: True if there may be remaining tasks to schedule, + @return: True if there may be remaining tasks to schedule, False otherwise. """ if self._terminated_tasks: - return False + return while self._can_add_job(): n = self._max_jobs - self._running_job_count() @@ -88,12 +79,10 @@ class QueueScheduler(PollScheduler): break if not self._start_next_job(n): - return False + return - for q in self._queues: - if q: - return True - return False + def _keep_scheduling(self): + return not self._terminated_tasks and any(self._queues) def _running_job_count(self): job_count = 0 diff --git a/portage_with_autodep/pym/_emerge/QueueScheduler.pyo b/portage_with_autodep/pym/_emerge/QueueScheduler.pyo Binary files differnew file mode 100644 index 0000000..88de3ea --- /dev/null +++ b/portage_with_autodep/pym/_emerge/QueueScheduler.pyo diff --git a/portage_with_autodep/pym/_emerge/RootConfig.py b/portage_with_autodep/pym/_emerge/RootConfig.py index d84f108..bb0d768 100644 --- a/portage_with_autodep/pym/_emerge/RootConfig.py +++ b/portage_with_autodep/pym/_emerge/RootConfig.py @@ -19,7 +19,7 @@ class RootConfig(object): def __init__(self, settings, trees, setconfig): self.trees = trees self.settings = settings - self.root = self.settings["ROOT"] + self.root = self.settings['EROOT'] self.setconfig = setconfig if setconfig is None: self.sets = {} diff --git a/portage_with_autodep/pym/_emerge/RootConfig.pyo b/portage_with_autodep/pym/_emerge/RootConfig.pyo Binary files differnew file mode 100644 index 0000000..fad3022 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/RootConfig.pyo diff --git a/portage_with_autodep/pym/_emerge/Scheduler.py b/portage_with_autodep/pym/_emerge/Scheduler.py index 6412d82..30a7e10 100644 --- a/portage_with_autodep/pym/_emerge/Scheduler.py +++ b/portage_with_autodep/pym/_emerge/Scheduler.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -7,10 +7,8 @@ from collections import deque import gc import gzip import logging -import shutil import signal import sys -import tempfile import textwrap import time import warnings @@ -28,9 +26,12 @@ from portage.output import colorize, create_color_func, red bad = create_color_func("BAD") from portage._sets import SETPREFIX from portage._sets.base import InternalPackageSet -from portage.util import writemsg, writemsg_level +from portage.util import ensure_dirs, writemsg, writemsg_level +from portage.util.SlotObject import SlotObject from portage.package.ebuild.digestcheck import digestcheck from portage.package.ebuild.digestgen import digestgen +from portage.package.ebuild.doebuild import (_check_temp_dir, + _prepare_self_update) from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs import _emerge @@ -44,6 +45,7 @@ from _emerge.create_depgraph_params import create_depgraph_params from _emerge.create_world_atom import create_world_atom from _emerge.DepPriority import DepPriority from _emerge.depgraph import depgraph, resume_depgraph +from _emerge.EbuildBuildDir import EbuildBuildDir from _emerge.EbuildFetcher import EbuildFetcher from _emerge.EbuildPhase import EbuildPhase from _emerge.emergelog import emergelog @@ -52,12 +54,9 @@ from _emerge._find_deep_system_runtime_deps import _find_deep_system_runtime_dep from _emerge._flush_elog_mod_echo import _flush_elog_mod_echo from _emerge.JobStatusDisplay import JobStatusDisplay from _emerge.MergeListItem import MergeListItem -from _emerge.MiscFunctionsProcess import MiscFunctionsProcess from _emerge.Package import Package from _emerge.PackageMerge import PackageMerge from _emerge.PollScheduler import PollScheduler -from _emerge.RootConfig import RootConfig -from _emerge.SlotObject import SlotObject from _emerge.SequentialTaskQueue import SequentialTaskQueue if sys.hexversion >= 0x3000000: @@ -77,17 +76,12 @@ class Scheduler(PollScheduler): frozenset(["--pretend", "--fetchonly", "--fetch-all-uri"]) - _opts_no_restart = frozenset(["--buildpkgonly", + _opts_no_self_update = frozenset(["--buildpkgonly", "--fetchonly", "--fetch-all-uri", "--pretend"]) - _bad_resume_opts = set(["--ask", "--changelog", - "--resume", "--skipfirst"]) - - class _iface_class(SlotObject): + class _iface_class(PollScheduler._sched_iface_class): __slots__ = ("fetch", - "output", "register", "schedule", - "scheduleSetup", "scheduleUnpack", "scheduleYield", - "unregister") + "scheduleSetup", "scheduleUnpack") class _fetch_iface_class(SlotObject): __slots__ = ("log_file", "schedule") @@ -96,7 +90,7 @@ class Scheduler(PollScheduler): ("merge", "jobs", "ebuild_locks", "fetch", "unpack"), prefix="") class _build_opts_class(SlotObject): - __slots__ = ("buildpkg", "buildpkgonly", + __slots__ = ("buildpkg", "buildpkg_exclude", "buildpkgonly", "fetch_all_uri", "fetchonly", "pretend") class _binpkg_opts_class(SlotObject): @@ -141,8 +135,9 @@ class Scheduler(PollScheduler): portage.exception.PortageException.__init__(self, value) def __init__(self, settings, trees, mtimedb, myopts, - spinner, mergelist=None, favorites=None, graph_config=None): - PollScheduler.__init__(self) + spinner, mergelist=None, favorites=None, graph_config=None, + uninstall_only=False): + PollScheduler.__init__(self, main=True) if mergelist is not None: warnings.warn("The mergelist parameter of the " + \ @@ -151,16 +146,22 @@ class Scheduler(PollScheduler): DeprecationWarning, stacklevel=2) self.settings = settings - self.target_root = settings["ROOT"] + self.target_root = settings["EROOT"] self.trees = trees self.myopts = myopts self._spinner = spinner self._mtimedb = mtimedb self._favorites = favorites + self._uninstall_only = uninstall_only self._args_set = InternalPackageSet(favorites, allow_repo=True) self._build_opts = self._build_opts_class() + for k in self._build_opts.__slots__: - setattr(self._build_opts, k, "--" + k.replace("_", "-") in myopts) + setattr(self._build_opts, k, myopts.get("--" + k.replace("_", "-"))) + self._build_opts.buildpkg_exclude = InternalPackageSet( \ + initial_atoms=" ".join(myopts.get("--buildpkg-exclude", [])).split(), \ + allow_wildcard=True, allow_repo=True) + self._binpkg_opts = self._binpkg_opts_class() for k in self._binpkg_opts.__slots__: setattr(self._binpkg_opts, k, "--" + k.replace("_", "-") in myopts) @@ -202,10 +203,7 @@ class Scheduler(PollScheduler): if max_jobs is None: max_jobs = 1 self._set_max_jobs(max_jobs) - - # The root where the currently running - # portage instance is installed. - self._running_root = trees["/"]["root_config"] + self._running_root = trees[trees._running_eroot]["root_config"] self.edebug = 0 if settings.get("PORTAGE_DEBUG", "") == "1": self.edebug = 1 @@ -219,13 +217,11 @@ class Scheduler(PollScheduler): fetch_iface = self._fetch_iface_class(log_file=self._fetch_log, schedule=self._schedule_fetch) self._sched_iface = self._iface_class( - fetch=fetch_iface, output=self._task_output, - register=self._register, - schedule=self._schedule_wait, + fetch=fetch_iface, scheduleSetup=self._schedule_setup, scheduleUnpack=self._schedule_unpack, - scheduleYield=self._schedule_yield, - unregister=self._unregister) + **dict((k, getattr(self.sched_iface, k)) + for k in self.sched_iface.__slots__)) self._prefetchers = weakref.WeakValueDictionary() self._pkg_queue = [] @@ -277,7 +273,7 @@ class Scheduler(PollScheduler): if self._parallel_fetch: # clear out existing fetch log if it exists try: - open(self._fetch_log, 'w') + open(self._fetch_log, 'w').close() except EnvironmentError: pass @@ -289,10 +285,37 @@ class Scheduler(PollScheduler): self._running_portage = self._pkg(cpv, "installed", self._running_root, installed=True) + def _handle_self_update(self): + + if self._opts_no_self_update.intersection(self.myopts): + return os.EX_OK + + for x in self._mergelist: + if not isinstance(x, Package): + continue + if x.operation != "merge": + continue + if x.root != self._running_root.root: + continue + if not portage.dep.match_from_list( + portage.const.PORTAGE_PACKAGE_ATOM, [x]): + continue + if self._running_portage is None or \ + self._running_portage.cpv != x.cpv or \ + '9999' in x.cpv or \ + 'git' in x.inherited or \ + 'git-2' in x.inherited: + rval = _check_temp_dir(self.settings) + if rval != os.EX_OK: + return rval + _prepare_self_update(self.settings) + break + + return os.EX_OK + def _terminate_tasks(self): self._status_display.quiet = True - while self._running_tasks: - task_id, task = self._running_tasks.popitem() + for task in list(self._running_tasks.values()): task.cancel() for q in self._task_queues.values(): q.clear() @@ -304,10 +327,13 @@ class Scheduler(PollScheduler): """ self._set_graph_config(graph_config) self._blocker_db = {} + dynamic_deps = self.myopts.get("--dynamic-deps", "y") != "n" for root in self.trees: + if self._uninstall_only: + continue if graph_config is None: fake_vartree = FakeVartree(self.trees[root]["root_config"], - pkg_cache=self._pkg_cache) + pkg_cache=self._pkg_cache, dynamic_deps=dynamic_deps) fake_vartree.sync() else: fake_vartree = graph_config.trees[root]['vartree'] @@ -324,52 +350,6 @@ class Scheduler(PollScheduler): self._set_graph_config(None) gc.collect() - def _poll(self, timeout=None): - - self._schedule() - - if timeout is None: - while True: - if not self._poll_event_handlers: - self._schedule() - if not self._poll_event_handlers: - raise StopIteration( - "timeout is None and there are no poll() event handlers") - previous_count = len(self._poll_event_queue) - PollScheduler._poll(self, timeout=self._max_display_latency) - self._status_display.display() - if previous_count != len(self._poll_event_queue): - break - - elif timeout <= self._max_display_latency: - PollScheduler._poll(self, timeout=timeout) - if timeout == 0: - # The display is updated by _schedule() above, so it would be - # redundant to update it here when timeout is 0. - pass - else: - self._status_display.display() - - else: - remaining_timeout = timeout - start_time = time.time() - while True: - previous_count = len(self._poll_event_queue) - PollScheduler._poll(self, - timeout=min(self._max_display_latency, remaining_timeout)) - self._status_display.display() - if previous_count != len(self._poll_event_queue): - break - elapsed_time = time.time() - start_time - if elapsed_time < 0: - # The system clock has changed such that start_time - # is now in the future, so just assume that the - # timeout has already elapsed. - break - remaining_timeout = timeout - 1000 * elapsed_time - if remaining_timeout <= 0: - break - def _set_max_jobs(self, max_jobs): self._max_jobs = max_jobs self._task_queues.jobs.max_jobs = max_jobs @@ -381,11 +361,11 @@ class Scheduler(PollScheduler): Check if background mode is enabled and adjust states as necessary. @rtype: bool - @returns: True if background mode is enabled, False otherwise. + @return: True if background mode is enabled, False otherwise. """ background = (self._max_jobs is True or \ self._max_jobs > 1 or "--quiet" in self.myopts \ - or "--quiet-build" in self.myopts) and \ + or self.myopts.get("--quiet-build") == "y") and \ not bool(self._opts_no_background.intersection(self.myopts)) if background: @@ -398,7 +378,7 @@ class Scheduler(PollScheduler): msg = [""] for pkg in interactive_tasks: pkg_str = " " + colorize("INFORM", str(pkg.cpv)) - if pkg.root != "/": + if pkg.root_config.settings["ROOT"] != "/": pkg_str += " for " + pkg.root msg.append(pkg_str) msg.append("") @@ -741,7 +721,6 @@ class Scheduler(PollScheduler): self._status_msg("Starting parallel fetch") prefetchers = self._prefetchers - getbinpkg = "--getbinpkg" in self.myopts for pkg in self._mergelist: # mergelist can contain solved Blocker instances @@ -749,15 +728,13 @@ class Scheduler(PollScheduler): continue prefetcher = self._create_prefetcher(pkg) if prefetcher is not None: - self._task_queues.fetch.add(prefetcher) + # This will start the first prefetcher immediately, so that + # self._task() won't discard it. This avoids a case where + # the first prefetcher is discarded, causing the second + # prefetcher to occupy the fetch queue before the first + # fetcher has an opportunity to execute. prefetchers[pkg] = prefetcher - - # Start the first prefetcher immediately so that self._task() - # won't discard it. This avoids a case where the first - # prefetcher is discarded, causing the second prefetcher to - # occupy the fetch queue before the first fetcher has an - # opportunity to execute. - self._task_queues.fetch.schedule() + self._task_queues.fetch.add(prefetcher) def _create_prefetcher(self, pkg): """ @@ -785,100 +762,6 @@ class Scheduler(PollScheduler): return prefetcher - def _is_restart_scheduled(self): - """ - Check if the merge list contains a replacement - for the current running instance, that will result - in restart after merge. - @rtype: bool - @returns: True if a restart is scheduled, False otherwise. - """ - if self._opts_no_restart.intersection(self.myopts): - return False - - mergelist = self._mergelist - - for i, pkg in enumerate(mergelist): - if self._is_restart_necessary(pkg) and \ - i != len(mergelist) - 1: - return True - - return False - - def _is_restart_necessary(self, pkg): - """ - @return: True if merging the given package - requires restart, False otherwise. - """ - - # Figure out if we need a restart. - if pkg.root == self._running_root.root and \ - portage.match_from_list( - portage.const.PORTAGE_PACKAGE_ATOM, [pkg]): - if self._running_portage is None: - return True - elif pkg.cpv != self._running_portage.cpv or \ - '9999' in pkg.cpv or \ - 'git' in pkg.inherited or \ - 'git-2' in pkg.inherited: - return True - return False - - def _restart_if_necessary(self, pkg): - """ - Use execv() to restart emerge. This happens - if portage upgrades itself and there are - remaining packages in the list. - """ - - if self._opts_no_restart.intersection(self.myopts): - return - - if not self._is_restart_necessary(pkg): - return - - if pkg == self._mergelist[-1]: - return - - self._main_loop_cleanup() - - logger = self._logger - pkg_count = self._pkg_count - mtimedb = self._mtimedb - bad_resume_opts = self._bad_resume_opts - - logger.log(" ::: completed emerge (%s of %s) %s to %s" % \ - (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg.root)) - - logger.log(" *** RESTARTING " + \ - "emerge via exec() after change of " + \ - "portage version.") - - mtimedb["resume"]["mergelist"].remove(list(pkg)) - mtimedb.commit() - portage.run_exitfuncs() - # Don't trust sys.argv[0] here because eselect-python may modify it. - emerge_binary = os.path.join(portage.const.PORTAGE_BIN_PATH, 'emerge') - mynewargv = [emerge_binary, "--resume"] - resume_opts = self.myopts.copy() - # For automatic resume, we need to prevent - # any of bad_resume_opts from leaking in - # via EMERGE_DEFAULT_OPTS. - resume_opts["--ignore-default-opts"] = True - for myopt, myarg in resume_opts.items(): - if myopt not in bad_resume_opts: - if myarg is True: - mynewargv.append(myopt) - elif isinstance(myarg, list): - # arguments like --exclude that use 'append' action - for x in myarg: - mynewargv.append("%s=%s" % (myopt, x)) - else: - mynewargv.append("%s=%s" % (myopt, myarg)) - # priority only needs to be adjusted on the first run - os.environ["PORTAGE_NICENESS"] = "0" - os.execv(mynewargv[0], mynewargv) - def _run_pkg_pretend(self): """ Since pkg_pretend output may be important, this method sends all @@ -912,11 +795,48 @@ class Scheduler(PollScheduler): root_config = x.root_config settings = self.pkgsettings[root_config.root] settings.setcpv(x) - tmpdir = tempfile.mkdtemp() - tmpdir_orig = settings["PORTAGE_TMPDIR"] - settings["PORTAGE_TMPDIR"] = tmpdir + + # setcpv/package.env allows for per-package PORTAGE_TMPDIR so we + # have to validate it for each package + rval = _check_temp_dir(settings) + if rval != os.EX_OK: + return rval + + build_dir_path = os.path.join( + os.path.realpath(settings["PORTAGE_TMPDIR"]), + "portage", x.category, x.pf) + existing_buildir = os.path.isdir(build_dir_path) + settings["PORTAGE_BUILDDIR"] = build_dir_path + build_dir = EbuildBuildDir(scheduler=sched_iface, + settings=settings) + build_dir.lock() + current_task = None try: + + # Clean up the existing build dir, in case pkg_pretend + # checks for available space (bug #390711). + if existing_buildir: + if x.built: + tree = "bintree" + infloc = os.path.join(build_dir_path, "build-info") + ebuild_path = os.path.join(infloc, x.pf + ".ebuild") + else: + tree = "porttree" + portdb = root_config.trees["porttree"].dbapi + ebuild_path = portdb.findname(x.cpv, myrepo=x.repo) + if ebuild_path is None: + raise AssertionError( + "ebuild not found for '%s'" % x.cpv) + portage.package.ebuild.doebuild.doebuild_environment( + ebuild_path, "clean", settings=settings, + db=self.trees[settings['EROOT']][tree].dbapi) + clean_phase = EbuildPhase(background=False, + phase='clean', scheduler=sched_iface, settings=settings) + current_task = clean_phase + clean_phase.start() + clean_phase.wait() + if x.built: tree = "bintree" bintree = root_config.trees["bintree"].dbapi.bintree @@ -935,6 +855,7 @@ class Scheduler(PollScheduler): verifier = BinpkgVerifier(pkg=x, scheduler=sched_iface) + current_task = verifier verifier.start() if verifier.wait() != os.EX_OK: failures += 1 @@ -943,8 +864,8 @@ class Scheduler(PollScheduler): if fetched: bintree.inject(x.cpv, filename=fetched) tbz2_file = bintree.getname(x.cpv) - infloc = os.path.join(tmpdir, x.category, x.pf, "build-info") - os.makedirs(infloc) + infloc = os.path.join(build_dir_path, "build-info") + ensure_dirs(infloc) portage.xpak.tbz2(tbz2_file).unpackinfo(infloc) ebuild_path = os.path.join(infloc, x.pf + ".ebuild") settings.configdict["pkg"]["EMERGE_FROM"] = "binary" @@ -964,7 +885,8 @@ class Scheduler(PollScheduler): portage.package.ebuild.doebuild.doebuild_environment(ebuild_path, "pretend", settings=settings, - db=self.trees[settings["ROOT"]][tree].dbapi) + db=self.trees[settings['EROOT']][tree].dbapi) + prepare_build_dirs(root_config.root, settings, cleanup=0) vardb = root_config.trees['vartree'].dbapi @@ -976,14 +898,21 @@ class Scheduler(PollScheduler): phase="pretend", scheduler=sched_iface, settings=settings) + current_task = pretend_phase pretend_phase.start() ret = pretend_phase.wait() if ret != os.EX_OK: failures += 1 portage.elog.elog_process(x.cpv, settings) finally: - shutil.rmtree(tmpdir) - settings["PORTAGE_TMPDIR"] = tmpdir_orig + if current_task is not None and current_task.isAlive(): + current_task.cancel() + current_task.wait() + clean_phase = EbuildPhase(background=False, + phase='clean', scheduler=sched_iface, settings=settings) + clean_phase.start() + clean_phase.wait() + build_dir.unlock() if failures: return 1 @@ -1003,6 +932,10 @@ class Scheduler(PollScheduler): except self._unknown_internal_error: return 1 + rval = self._handle_self_update() + if rval != os.EX_OK: + return rval + for root in self.trees: root_config = self.trees[root]["root_config"] @@ -1131,10 +1064,8 @@ class Scheduler(PollScheduler): # If only one package failed then just show it's # whole log for easy viewing. failed_pkg = self._failed_pkgs_all[-1] - build_dir = failed_pkg.build_dir log_file = None - - log_paths = [failed_pkg.build_log] + log_file_real = None log_path = self._locate_failure_log(failed_pkg) if log_path is not None: @@ -1145,6 +1076,7 @@ class Scheduler(PollScheduler): pass else: if log_path.endswith('.gz'): + log_file_real = log_file log_file = gzip.GzipFile(filename='', mode='rb', fileobj=log_file) @@ -1157,6 +1089,8 @@ class Scheduler(PollScheduler): noiselevel=-1) finally: log_file.close() + if log_file_real is not None: + log_file_real.close() failure_log_shown = True # Dump mod_echo output now since it tends to flood the terminal. @@ -1228,9 +1162,6 @@ class Scheduler(PollScheduler): def _locate_failure_log(self, failed_pkg): - build_dir = failed_pkg.build_dir - log_file = None - log_paths = [failed_pkg.build_log] for log_path in log_paths: @@ -1272,7 +1203,7 @@ class Scheduler(PollScheduler): # Skip this if $ROOT != / since it shouldn't matter if there # are unsatisfied system runtime deps in this case. - if pkg.root != '/': + if pkg.root_config.settings["ROOT"] != "/": return completed_tasks = self._completed_tasks @@ -1350,8 +1281,6 @@ class Scheduler(PollScheduler): if pkg.installed: return - self._restart_if_necessary(pkg) - # Call mtimedb.commit() after each merge so that # --resume still works after being interrupted # by reboot, sigkill or similar. @@ -1411,12 +1340,16 @@ class Scheduler(PollScheduler): def _merge(self): + if self._opts_no_background.intersection(self.myopts): + self._set_max_jobs(1) + self._add_prefetchers() self._add_packages() - pkg_queue = self._pkg_queue failed_pkgs = self._failed_pkgs portage.locks._quiet = self._background portage.elog.add_listener(self._elog_listener) + display_timeout_id = self.sched_iface.timeout_add( + self._max_display_latency, self._status_display.display) rval = os.EX_OK try: @@ -1425,6 +1358,7 @@ class Scheduler(PollScheduler): self._main_loop_cleanup() portage.locks._quiet = False portage.elog.remove_listener(self._elog_listener) + self.sched_iface.source_remove(display_timeout_id) if failed_pkgs: rval = failed_pkgs[-1].returncode @@ -1505,7 +1439,7 @@ class Scheduler(PollScheduler): merge order @type later: set @rtype: bool - @returns: True if the package is dependent, False otherwise. + @return: True if the package is dependent, False otherwise. """ graph = self._digraph @@ -1553,24 +1487,7 @@ class Scheduler(PollScheduler): return temp_settings def _deallocate_config(self, settings): - self._config_pool[settings["ROOT"]].append(settings) - - def _main_loop(self): - - # Only allow 1 job max if a restart is scheduled - # due to portage update. - if self._is_restart_scheduled() or \ - self._opts_no_background.intersection(self.myopts): - self._set_max_jobs(1) - - while self._schedule(): - self._poll_loop() - - while True: - self._schedule() - if not self._is_work_scheduled(): - break - self._poll_loop() + self._config_pool[settings['EROOT']].append(settings) def _keep_scheduling(self): return bool(not self._terminated_tasks and self._pkg_queue and \ @@ -1583,6 +1500,8 @@ class Scheduler(PollScheduler): while True: + state_change = 0 + # When the number of jobs and merges drops to zero, # process a single merge from _merge_wait_queue if # it's not empty. We only process one since these are @@ -1593,37 +1512,34 @@ class Scheduler(PollScheduler): not self._task_queues.merge): task = self._merge_wait_queue.popleft() task.addExitListener(self._merge_wait_exit_handler) + self._merge_wait_scheduled.append(task) self._task_queues.merge.add(task) self._status_display.merges = len(self._task_queues.merge) - self._merge_wait_scheduled.append(task) + state_change += 1 - self._schedule_tasks_imp() - self._status_display.display() + if self._schedule_tasks_imp(): + state_change += 1 - state_change = 0 - for q in self._task_queues.values(): - if q.schedule(): - state_change += 1 + self._status_display.display() # Cancel prefetchers if they're the only reason # the main poll loop is still running. if self._failed_pkgs and not self._build_opts.fetchonly and \ not self._is_work_scheduled() and \ self._task_queues.fetch: + # Since this happens asynchronously, it doesn't count in + # state_change (counting it triggers an infinite loop). self._task_queues.fetch.clear() - state_change += 1 if not (state_change or \ (self._merge_wait_queue and not self._jobs and not self._task_queues.merge)): break - return self._keep_scheduling() - def _job_delay(self): """ @rtype: bool - @returns: True if job scheduling should be delayed, False otherwise. + @return: True if job scheduling should be delayed, False otherwise. """ if self._jobs and self._max_load is not None: @@ -1641,7 +1557,7 @@ class Scheduler(PollScheduler): def _schedule_tasks_imp(self): """ @rtype: bool - @returns: True if state changed, False otherwise. + @return: True if state changed, False otherwise. """ state_change = 0 @@ -1709,7 +1625,14 @@ class Scheduler(PollScheduler): "installed", pkg.root_config, installed=True, operation="uninstall") - prefetcher = self._prefetchers.pop(pkg, None) + try: + prefetcher = self._prefetchers.pop(pkg, None) + except KeyError: + # KeyError observed with PyPy 1.8, despite None given as default. + # Note that PyPy 1.8 has the same WeakValueDictionary code as + # CPython 2.7, so it may be possible for CPython to raise KeyError + # here as well. + prefetcher = None if prefetcher is not None and not prefetcher.isAlive(): try: self._task_queues.fetch._task_queue.remove(prefetcher) @@ -1738,7 +1661,7 @@ class Scheduler(PollScheduler): pkg = failed_pkg.pkg msg = "%s to %s %s" % \ (bad("Failed"), action, colorize("INFORM", pkg.cpv)) - if pkg.root != "/": + if pkg.root_config.settings["ROOT"] != "/": msg += " %s %s" % (preposition, pkg.root) log_path = self._locate_failure_log(failed_pkg) @@ -1791,7 +1714,7 @@ class Scheduler(PollScheduler): Use the current resume list to calculate a new one, dropping any packages with unsatisfied deps. @rtype: bool - @returns: True if successful, False otherwise. + @return: True if successful, False otherwise. """ print(colorize("GOOD", "*** Resuming merge...")) @@ -1868,7 +1791,7 @@ class Scheduler(PollScheduler): pkg = task msg = "emerge --keep-going:" + \ " %s" % (pkg.cpv,) - if pkg.root != "/": + if pkg.root_config.settings["ROOT"] != "/": msg += " for %s" % (pkg.root,) msg += " dropped due to unsatisfied dependency." for line in textwrap.wrap(msg, msg_width): diff --git a/portage_with_autodep/pym/_emerge/Scheduler.pyo b/portage_with_autodep/pym/_emerge/Scheduler.pyo Binary files differnew file mode 100644 index 0000000..5555703 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/Scheduler.pyo diff --git a/portage_with_autodep/pym/_emerge/SequentialTaskQueue.py b/portage_with_autodep/pym/_emerge/SequentialTaskQueue.py index c1c98c4..8090893 100644 --- a/portage_with_autodep/pym/_emerge/SequentialTaskQueue.py +++ b/portage_with_autodep/pym/_emerge/SequentialTaskQueue.py @@ -1,13 +1,15 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import sys -from _emerge.SlotObject import SlotObject from collections import deque +import sys + +from portage.util.SlotObject import SlotObject + class SequentialTaskQueue(SlotObject): __slots__ = ("max_jobs", "running_tasks") + \ - ("_dirty", "_scheduling", "_task_queue") + ("_scheduling", "_task_queue") def __init__(self, **kwargs): SlotObject.__init__(self, **kwargs) @@ -15,50 +17,34 @@ class SequentialTaskQueue(SlotObject): self.running_tasks = set() if self.max_jobs is None: self.max_jobs = 1 - self._dirty = True def add(self, task): self._task_queue.append(task) - self._dirty = True + self.schedule() def addFront(self, task): self._task_queue.appendleft(task) - self._dirty = True + self.schedule() def schedule(self): - if not self._dirty: - return False - - if not self: - return False - if self._scheduling: # Ignore any recursive schedule() calls triggered via # self._task_exit(). - return False + return self._scheduling = True - - task_queue = self._task_queue - running_tasks = self.running_tasks - max_jobs = self.max_jobs - state_changed = False - - while task_queue and \ - (max_jobs is True or len(running_tasks) < max_jobs): - task = task_queue.popleft() - cancelled = getattr(task, "cancelled", None) - if not cancelled: - running_tasks.add(task) - task.addExitListener(self._task_exit) - task.start() - state_changed = True - - self._dirty = False - self._scheduling = False - - return state_changed + try: + while self._task_queue and (self.max_jobs is True or + len(self.running_tasks) < self.max_jobs): + task = self._task_queue.popleft() + cancelled = getattr(task, "cancelled", None) + if not cancelled: + self.running_tasks.add(task) + task.addExitListener(self._task_exit) + task.start() + finally: + self._scheduling = False def _task_exit(self, task): """ @@ -68,16 +54,22 @@ class SequentialTaskQueue(SlotObject): """ self.running_tasks.remove(task) if self._task_queue: - self._dirty = True + self.schedule() def clear(self): + """ + Clear the task queue and asynchronously terminate any running tasks. + """ self._task_queue.clear() - running_tasks = self.running_tasks - while running_tasks: - task = running_tasks.pop() - task.removeExitListener(self._task_exit) + for task in list(self.running_tasks): task.cancel() - self._dirty = False + + def wait(self): + """ + Synchronously wait for all running tasks to exit. + """ + while self.running_tasks: + next(iter(self.running_tasks)).wait() def __bool__(self): return bool(self._task_queue or self.running_tasks) diff --git a/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo b/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo Binary files differnew file mode 100644 index 0000000..3ab65c9 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/SequentialTaskQueue.pyo diff --git a/portage_with_autodep/pym/_emerge/SetArg.pyo b/portage_with_autodep/pym/_emerge/SetArg.pyo Binary files differnew file mode 100644 index 0000000..5a3d9d9 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/SetArg.pyo diff --git a/portage_with_autodep/pym/_emerge/SpawnProcess.py b/portage_with_autodep/pym/_emerge/SpawnProcess.py index b72971c..9fbc964 100644 --- a/portage_with_autodep/pym/_emerge/SpawnProcess.py +++ b/portage_with_autodep/pym/_emerge/SpawnProcess.py @@ -26,29 +26,16 @@ class SpawnProcess(SubProcess): "path_lookup", "pre_exec") __slots__ = ("args",) + \ - _spawn_kwarg_names + ("_selinux_type",) + _spawn_kwarg_names + ("_log_file_real", "_selinux_type",) _file_names = ("log", "process", "stdout") _files_dict = slot_dict_class(_file_names, prefix="") def _start(self): - if self.cancelled: - return - if self.fd_pipes is None: self.fd_pipes = {} fd_pipes = self.fd_pipes - fd_pipes.setdefault(0, sys.stdin.fileno()) - fd_pipes.setdefault(1, sys.stdout.fileno()) - fd_pipes.setdefault(2, sys.stderr.fileno()) - - # flush any pending output - for fd in fd_pipes.values(): - if fd == sys.stdout.fileno(): - sys.stdout.flush() - if fd == sys.stderr.fileno(): - sys.stderr.flush() self._files = self._files_dict() files = self._files @@ -56,34 +43,46 @@ class SpawnProcess(SubProcess): master_fd, slave_fd = self._pipe(fd_pipes) fcntl.fcntl(master_fd, fcntl.F_SETFL, fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + files.process = master_fd logfile = None if self._can_log(slave_fd): logfile = self.logfile null_input = None - fd_pipes_orig = fd_pipes.copy() - if self.background: + if not self.background or 0 in fd_pipes: + # Subclasses such as AbstractEbuildProcess may have already passed + # in a null file descriptor in fd_pipes, so use that when given. + pass + else: # TODO: Use job control functions like tcsetpgrp() to control # access to stdin. Until then, use /dev/null so that any # attempts to read from stdin will immediately return EOF # instead of blocking indefinitely. - null_input = open('/dev/null', 'rb') - fd_pipes[0] = null_input.fileno() - else: - fd_pipes[0] = fd_pipes_orig[0] + null_input = os.open('/dev/null', os.O_RDWR) + fd_pipes[0] = null_input + + fd_pipes.setdefault(0, sys.stdin.fileno()) + fd_pipes.setdefault(1, sys.stdout.fileno()) + fd_pipes.setdefault(2, sys.stderr.fileno()) + + # flush any pending output + for fd in fd_pipes.values(): + if fd == sys.stdout.fileno(): + sys.stdout.flush() + if fd == sys.stderr.fileno(): + sys.stderr.flush() - # WARNING: It is very important to use unbuffered mode here, - # in order to avoid issue 5380 with python3. - files.process = os.fdopen(master_fd, 'rb', 0) if logfile is not None: + fd_pipes_orig = fd_pipes.copy() fd_pipes[1] = slave_fd fd_pipes[2] = slave_fd files.log = open(_unicode_encode(logfile, encoding=_encodings['fs'], errors='strict'), mode='ab') if logfile.endswith('.gz'): + self._log_file_real = files.log files.log = gzip.GzipFile(filename='', mode='ab', fileobj=files.log) @@ -92,7 +91,7 @@ class SpawnProcess(SubProcess): mode=0o660) if not self.background: - files.stdout = os.fdopen(os.dup(fd_pipes_orig[1]), 'wb') + files.stdout = os.dup(fd_pipes_orig[1]) output_handler = self._output_handler @@ -116,7 +115,7 @@ class SpawnProcess(SubProcess): kwargs["returnpid"] = True kwargs.pop("logfile", None) - self._reg_id = self.scheduler.register(files.process.fileno(), + self._reg_id = self.scheduler.register(files.process, self._registered_events, output_handler) self._registered = True @@ -124,7 +123,7 @@ class SpawnProcess(SubProcess): os.close(slave_fd) if null_input is not None: - null_input.close() + os.close(null_input) if isinstance(retval, int): # spawn failed @@ -161,22 +160,30 @@ class SpawnProcess(SubProcess): def _output_handler(self, fd, event): files = self._files - buf = self._read_buf(files.process, event) + while True: + buf = self._read_buf(fd, event) + + if buf is None: + # not a POLLIN event, EAGAIN, etc... + break - if buf is not None: + if not buf: + # EOF + self._unregister() + self.wait() + break - if buf: + else: if not self.background: write_successful = False failures = 0 while True: try: if not write_successful: - buf.tofile(files.stdout) + os.write(files.stdout, buf) write_successful = True - files.stdout.flush() break - except IOError as e: + except OSError as e: if e.errno != errno.EAGAIN: raise del e @@ -198,22 +205,17 @@ class SpawnProcess(SubProcess): # inherit stdio file descriptors from portage # (maybe it can't be avoided with # PROPERTIES=interactive). - fcntl.fcntl(files.stdout.fileno(), fcntl.F_SETFL, - fcntl.fcntl(files.stdout.fileno(), + fcntl.fcntl(files.stdout, fcntl.F_SETFL, + fcntl.fcntl(files.stdout, fcntl.F_GETFL) ^ os.O_NONBLOCK) - try: - buf.tofile(files.log) - except TypeError: - # array.tofile() doesn't work with GzipFile - files.log.write(buf.tostring()) + files.log.write(buf) files.log.flush() - else: - self._unregister() - self.wait() self._unregister_if_appropriate(event) + return True + def _dummy_handler(self, fd, event): """ This method is mainly interested in detecting EOF, since @@ -221,15 +223,26 @@ class SpawnProcess(SubProcess): monitor the process from inside a poll() loop. """ - buf = self._read_buf(self._files.process, event) + while True: + buf = self._read_buf(fd, event) - if buf is not None: + if buf is None: + # not a POLLIN event, EAGAIN, etc... + break - if buf: - pass - else: + if not buf: + # EOF self._unregister() self.wait() + break self._unregister_if_appropriate(event) + return True + + def _unregister(self): + super(SpawnProcess, self)._unregister() + if self._log_file_real is not None: + # Avoid "ResourceWarning: unclosed file" since python 3.2. + self._log_file_real.close() + self._log_file_real = None diff --git a/portage_with_autodep/pym/_emerge/SpawnProcess.pyo b/portage_with_autodep/pym/_emerge/SpawnProcess.pyo Binary files differnew file mode 100644 index 0000000..7a6142e --- /dev/null +++ b/portage_with_autodep/pym/_emerge/SpawnProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/SubProcess.py b/portage_with_autodep/pym/_emerge/SubProcess.py index b99cf0b..76b313f 100644 --- a/portage_with_autodep/pym/_emerge/SubProcess.py +++ b/portage_with_autodep/pym/_emerge/SubProcess.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage import os @@ -16,6 +16,10 @@ class SubProcess(AbstractPollTask): # serve this purpose alone. _dummy_pipe_fd = 9 + # This is how much time we allow for waitpid to succeed after + # we've sent a kill signal to our subprocess. + _cancel_timeout = 1000 # 1 second + def _poll(self): if self.returncode is not None: return self.returncode @@ -60,8 +64,7 @@ class SubProcess(AbstractPollTask): if self._registered: if self.cancelled: - timeout = 1000 - self.scheduler.schedule(self._reg_id, timeout=timeout) + self._wait_loop(timeout=self._cancel_timeout) if self._registered: try: os.kill(self.pid, signal.SIGKILL) @@ -69,41 +72,39 @@ class SubProcess(AbstractPollTask): if e.errno != errno.ESRCH: raise del e - self.scheduler.schedule(self._reg_id, timeout=timeout) + self._wait_loop(timeout=self._cancel_timeout) if self._registered: self._orphan_process_warn() else: - self.scheduler.schedule(self._reg_id) - self._unregister() + self._wait_loop() + if self.returncode is not None: return self.returncode - try: - # With waitpid and WNOHANG, only check the - # first element of the tuple since the second - # element may vary (bug #337465). - wait_retval = os.waitpid(self.pid, os.WNOHANG) - except OSError as e: - if e.errno != errno.ECHILD: - raise - del e - self._set_returncode((self.pid, 1 << 8)) - else: - if wait_retval[0] != 0: - self._set_returncode(wait_retval) - else: - try: - wait_retval = os.waitpid(self.pid, 0) - except OSError as e: - if e.errno != errno.ECHILD: - raise - del e - self._set_returncode((self.pid, 1 << 8)) - else: - self._set_returncode(wait_retval) + if not isinstance(self.pid, int): + # Get debug info for bug #403697. + raise AssertionError( + "%s: pid is non-integer: %s" % + (self.__class__.__name__, repr(self.pid))) + + self._waitpid_loop() return self.returncode + def _waitpid_loop(self): + source_id = self.scheduler.child_watch_add( + self.pid, self._waitpid_cb) + try: + while self.returncode is None: + self.scheduler.iteration() + finally: + self.scheduler.source_remove(source_id) + + def _waitpid_cb(self, pid, condition, user_data=None): + if pid != self.pid: + raise AssertionError("expected pid %s, got %s" % (self.pid, pid)) + self._set_returncode((pid, condition)) + def _orphan_process_warn(self): pass @@ -120,7 +121,10 @@ class SubProcess(AbstractPollTask): if self._files is not None: for f in self._files.values(): - f.close() + if isinstance(f, int): + os.close(f) + else: + f.close() self._files = None def _set_returncode(self, wait_retval): @@ -129,6 +133,7 @@ class SubProcess(AbstractPollTask): subprocess.Popen.returncode: A negative value -N indicates that the child was terminated by signal N (Unix only). """ + self._unregister() pid, status = wait_retval diff --git a/portage_with_autodep/pym/_emerge/SubProcess.pyo b/portage_with_autodep/pym/_emerge/SubProcess.pyo Binary files differnew file mode 100644 index 0000000..26e13e1 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/SubProcess.pyo diff --git a/portage_with_autodep/pym/_emerge/Task.py b/portage_with_autodep/pym/_emerge/Task.py index efbe3a9..40f5066 100644 --- a/portage_with_autodep/pym/_emerge/Task.py +++ b/portage_with_autodep/pym/_emerge/Task.py @@ -1,7 +1,8 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from _emerge.SlotObject import SlotObject +from portage.util.SlotObject import SlotObject + class Task(SlotObject): __slots__ = ("_hash_key", "_hash_value") diff --git a/portage_with_autodep/pym/_emerge/Task.pyo b/portage_with_autodep/pym/_emerge/Task.pyo Binary files differnew file mode 100644 index 0000000..2958cb1 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/Task.pyo diff --git a/portage_with_autodep/pym/_emerge/TaskScheduler.py b/portage_with_autodep/pym/_emerge/TaskScheduler.py index 83c0cbe..583bfe3 100644 --- a/portage_with_autodep/pym/_emerge/TaskScheduler.py +++ b/portage_with_autodep/pym/_emerge/TaskScheduler.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from _emerge.QueueScheduler import QueueScheduler @@ -11,13 +11,14 @@ class TaskScheduler(object): add tasks and call run(). The run() method returns when no tasks remain. """ - def __init__(self, max_jobs=None, max_load=None): + def __init__(self, main=True, max_jobs=None, max_load=None): self._queue = SequentialTaskQueue(max_jobs=max_jobs) - self._scheduler = QueueScheduler( + self._scheduler = QueueScheduler(main=main, max_jobs=max_jobs, max_load=max_load) self.sched_iface = self._scheduler.sched_iface self.run = self._scheduler.run self.clear = self._scheduler.clear + self.wait = self._queue.wait self._scheduler.add(self._queue) def add(self, task): diff --git a/portage_with_autodep/pym/_emerge/TaskScheduler.pyo b/portage_with_autodep/pym/_emerge/TaskScheduler.pyo Binary files differnew file mode 100644 index 0000000..8b84de7 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/TaskScheduler.pyo diff --git a/portage_with_autodep/pym/_emerge/TaskSequence.pyo b/portage_with_autodep/pym/_emerge/TaskSequence.pyo Binary files differnew file mode 100644 index 0000000..b98196e --- /dev/null +++ b/portage_with_autodep/pym/_emerge/TaskSequence.pyo diff --git a/portage_with_autodep/pym/_emerge/UninstallFailure.pyo b/portage_with_autodep/pym/_emerge/UninstallFailure.pyo Binary files differnew file mode 100644 index 0000000..9f1c88b --- /dev/null +++ b/portage_with_autodep/pym/_emerge/UninstallFailure.pyo diff --git a/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo Binary files differnew file mode 100644 index 0000000..b163ed7 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/UnmergeDepPriority.pyo diff --git a/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo b/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo Binary files differnew file mode 100644 index 0000000..005b007 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/UseFlagDisplay.pyo diff --git a/portage_with_autodep/pym/_emerge/__init__.pyo b/portage_with_autodep/pym/_emerge/__init__.pyo Binary files differnew file mode 100644 index 0000000..fba4ca5 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo b/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo Binary files differnew file mode 100644 index 0000000..8ad61b2 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/_find_deep_system_runtime_deps.pyo diff --git a/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.py b/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.py index eab4168..9ac65b8 100644 --- a/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.py +++ b/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.py @@ -8,7 +8,7 @@ def _flush_elog_mod_echo(): Dump the mod_echo output now so that our other notifications are shown last. @rtype: bool - @returns: True if messages were shown, False otherwise. + @return: True if messages were shown, False otherwise. """ messages_shown = bool(mod_echo._items) mod_echo.finalize() diff --git a/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo b/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo Binary files differnew file mode 100644 index 0000000..f211d41 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/_flush_elog_mod_echo.pyo diff --git a/portage_with_autodep/pym/_emerge/actions.py b/portage_with_autodep/pym/_emerge/actions.py index 2166963..eaf5a15 100644 --- a/portage_with_autodep/pym/_emerge/actions.py +++ b/portage_with_autodep/pym/_emerge/actions.py @@ -1,18 +1,19 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function import errno import logging +import operator import platform import pwd import random import re -import shutil import signal import socket import stat +import subprocess import sys import tempfile import textwrap @@ -20,20 +21,27 @@ import time from itertools import chain import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.news:count_unread_news,display_news_notifications', +) + +from portage.localization import _ from portage import os -from portage import subprocess_getstatusoutput -from portage import _unicode_decode +from portage import shutil +from portage import eapi_is_supported, _unicode_decode from portage.cache.cache_errors import CacheError -from portage.const import GLOBAL_CONFIG_PATH, NEWS_LIB_PATH +from portage.const import GLOBAL_CONFIG_PATH from portage.const import _ENABLE_DYN_LINK_MAP, _ENABLE_SET_CONFIG from portage.dbapi.dep_expand import dep_expand from portage.dbapi._expand_new_virt import expand_new_virt from portage.dep import Atom, extended_cp_match +from portage.eclass_cache import hashed_path from portage.exception import InvalidAtom from portage.output import blue, bold, colorize, create_color_func, darkgreen, \ red, yellow good = create_color_func("GOOD") bad = create_color_func("BAD") +warn = create_color_func("WARN") from portage.package.ebuild._ipc.QueryCommand import QueryCommand from portage.package.ebuild.doebuild import _check_temp_dir from portage._sets import load_default_config, SETPREFIX @@ -180,8 +188,7 @@ def action_build(settings, trees, mtimedb, " entire repository or category at once." prefix = bad(" * ") writemsg(prefix + "\n") - from textwrap import wrap - for line in wrap(msg, 72): + for line in textwrap.wrap(msg, 72): writemsg("%s%s\n" % (prefix, line)) writemsg(prefix + "\n") @@ -209,7 +216,6 @@ def action_build(settings, trees, mtimedb, if isinstance(e, depgraph.UnsatisfiedResumeDep): mydepgraph = e.depgraph - from textwrap import wrap from portage.output import EOutput out = EOutput() @@ -248,7 +254,7 @@ def action_build(settings, trees, mtimedb, "to skip the first package in the list and " + \ "any other packages that may be " + \ "masked or have missing dependencies." - for line in wrap(msg, 72): + for line in textwrap.wrap(msg, 72): out.eerror(line) elif isinstance(e, portage.exception.PackageNotFound): out.eerror("An expected package is " + \ @@ -258,7 +264,7 @@ def action_build(settings, trees, mtimedb, "packages that are no longer " + \ "available. Please restart/continue " + \ "the operation manually." - for line in wrap(msg, 72): + for line in textwrap.wrap(msg, 72): out.eerror(line) if success: @@ -291,7 +297,7 @@ def action_build(settings, trees, mtimedb, success, mydepgraph, favorites = backtrack_depgraph( settings, trees, myopts, myparams, myaction, myfiles, spinner) except portage.exception.PackageSetNotFound as e: - root_config = trees[settings["ROOT"]]["root_config"] + root_config = trees[settings['EROOT']]['root_config'] display_missing_pkg_set(root_config, e.value) return 1 @@ -329,7 +335,7 @@ def action_build(settings, trees, mtimedb, mergecount += 1 if mergecount==0: - sets = trees[settings["ROOT"]]["root_config"].sets + sets = trees[settings['EROOT']]['root_config'].sets world_candidates = None if "selective" in myparams and \ not oneshot and favorites: @@ -362,7 +368,7 @@ def action_build(settings, trees, mtimedb, print() print("Quitting.") print() - return os.EX_OK + return 128 + signal.SIGINT # Don't ask again (e.g. when auto-cleaning packages after merge) myopts.pop("--ask", None) @@ -439,7 +445,7 @@ def action_build(settings, trees, mtimedb, if retval == os.EX_OK and not (buildpkgonly or fetchonly or pretend): if "yes" == settings.get("AUTOCLEAN"): portage.writemsg_stdout(">>> Auto-cleaning packages...\n") - unmerge(trees[settings["ROOT"]]["root_config"], + unmerge(trees[settings['EROOT']]['root_config'], myopts, "clean", [], ldpath_mtimes, autoclean=1) else: @@ -454,7 +460,7 @@ def action_config(settings, trees, myopts, myfiles): if len(myfiles) != 1: print(red("!!! config can only take a single package atom at this time\n")) sys.exit(1) - if not is_valid_package_atom(myfiles[0]): + if not is_valid_package_atom(myfiles[0], allow_repo=True): portage.writemsg("!!! '%s' is not a valid package atom.\n" % myfiles[0], noiselevel=-1) portage.writemsg("!!! Please check ebuild(5) for full details.\n") @@ -462,7 +468,7 @@ def action_config(settings, trees, myopts, myfiles): sys.exit(1) print() try: - pkgs = trees[settings["ROOT"]]["vartree"].dbapi.match(myfiles[0]) + pkgs = trees[settings['EROOT']]['vartree'].dbapi.match(myfiles[0]) except portage.exception.AmbiguousPackageName as e: # Multiple matches thrown from cpv_expand pkgs = e.args[0] @@ -482,7 +488,7 @@ def action_config(settings, trees, myopts, myfiles): options.append("X") idx = userquery("Selection?", enter_invalid, responses=options) if idx == "X": - sys.exit(0) + sys.exit(128 + signal.SIGINT) pkg = pkgs[int(idx)-1] else: print("The following packages available:") @@ -496,21 +502,20 @@ def action_config(settings, trees, myopts, myfiles): print() if "--ask" in myopts: if userquery("Ready to configure %s?" % pkg, enter_invalid) == "No": - sys.exit(0) + sys.exit(128 + signal.SIGINT) else: print("Configuring pkg...") print() - ebuildpath = trees[settings["ROOT"]]["vartree"].dbapi.findname(pkg) + ebuildpath = trees[settings['EROOT']]['vartree'].dbapi.findname(pkg) mysettings = portage.config(clone=settings) - vardb = trees[mysettings["ROOT"]]["vartree"].dbapi + vardb = trees[mysettings['EROOT']]['vartree'].dbapi debug = mysettings.get("PORTAGE_DEBUG") == "1" - retval = portage.doebuild(ebuildpath, "config", mysettings["ROOT"], - mysettings, + retval = portage.doebuild(ebuildpath, "config", settings=mysettings, debug=(settings.get("PORTAGE_DEBUG", "") == 1), cleanup=True, - mydbapi=trees[settings["ROOT"]]["vartree"].dbapi, tree="vartree") + mydbapi = trees[settings['EROOT']]['vartree'].dbapi, tree="vartree") if retval == os.EX_OK: - portage.doebuild(ebuildpath, "clean", mysettings["ROOT"], - mysettings, debug=debug, mydbapi=vardb, tree="vartree") + portage.doebuild(ebuildpath, "clean", settings=mysettings, + debug=debug, mydbapi=vardb, tree="vartree") print() def action_depclean(settings, trees, ldpath_mtimes, @@ -550,7 +555,7 @@ def action_depclean(settings, trees, ldpath_mtimes, for x in msg: portage.writemsg_stdout(colorize("WARN", " * ") + x) - root_config = trees[settings['ROOT']]['root_config'] + root_config = trees[settings['EROOT']]['root_config'] vardb = root_config.trees['vartree'].dbapi args_set = InternalPackageSet(allow_repo=True) @@ -582,15 +587,15 @@ def action_depclean(settings, trees, ldpath_mtimes, return rval if cleanlist: - unmerge(root_config, myopts, "unmerge", + rval = unmerge(root_config, myopts, "unmerge", cleanlist, ldpath_mtimes, ordered=ordered, scheduler=scheduler) if action == "prune": - return + return rval if not cleanlist and "--quiet" in myopts: - return + return rval print("Packages installed: " + str(len(vardb.cpv_all()))) print("Packages in world: " + \ @@ -603,14 +608,17 @@ def action_depclean(settings, trees, ldpath_mtimes, else: print("Number removed: "+str(len(cleanlist))) + return rval + def calc_depclean(settings, trees, ldpath_mtimes, myopts, action, args_set, spinner): allow_missing_deps = bool(args_set) debug = '--debug' in myopts xterm_titles = "notitles" not in settings.features - myroot = settings["ROOT"] - root_config = trees[myroot]["root_config"] + root_len = len(settings["ROOT"]) + eroot = settings['EROOT'] + root_config = trees[eroot]["root_config"] psets = root_config.setconfig.psets deselect = myopts.get('--deselect') != 'n' required_sets = {} @@ -649,8 +657,8 @@ def calc_depclean(settings, trees, ldpath_mtimes, resolver_params = create_depgraph_params(myopts, "remove") resolver = depgraph(settings, trees, myopts, resolver_params, spinner) resolver._load_vdb() - vardb = resolver._frozen_config.trees[myroot]["vartree"].dbapi - real_vardb = trees[myroot]["vartree"].dbapi + vardb = resolver._frozen_config.trees[eroot]["vartree"].dbapi + real_vardb = trees[eroot]["vartree"].dbapi if action == "depclean": @@ -705,7 +713,8 @@ def calc_depclean(settings, trees, ldpath_mtimes, # that are also matched by argument atoms, but do not remove # them if they match the highest installed version. for pkg in vardb: - spinner.update() + if spinner is not None: + spinner.update() pkgs_for_cp = vardb.match_pkgs(pkg.cp) if not pkgs_for_cp or pkg not in pkgs_for_cp: raise AssertionError("package expected in matches: " + \ @@ -751,7 +760,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, del e required_sets['__excluded__'].add("=" + pkg.cpv) - success = resolver._complete_graph(required_sets={myroot:required_sets}) + success = resolver._complete_graph(required_sets={eroot:required_sets}) writemsg_level("\b\b... done!\n") resolver.display_problems() @@ -937,7 +946,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, consumers = {} for lib in pkg_dblink.getcontents(): - lib = lib[len(myroot):] + lib = lib[root_len:] lib_key = linkmap._obj_key(lib) lib_consumers = consumer_cache.get(lib_key) if lib_consumers is None: @@ -1053,9 +1062,8 @@ def calc_depclean(settings, trees, ldpath_mtimes, "the packages that pulled them in." prefix = bad(" * ") - from textwrap import wrap writemsg_level("".join(prefix + "%s\n" % line for \ - line in wrap(msg, 70)), level=logging.WARNING, noiselevel=-1) + line in textwrap.wrap(msg, 70)), level=logging.WARNING, noiselevel=-1) msg = [] for pkg in sorted(consumer_map, key=cmp_sort_key(cmp_pkg_cpv)): @@ -1095,7 +1103,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, writemsg_level("\nCalculating dependencies ") success = resolver._complete_graph( - required_sets={myroot:required_sets}) + required_sets={eroot:required_sets}) writemsg_level("\b\b... done!\n") resolver.display_problems() if not success: @@ -1137,7 +1145,6 @@ def calc_depclean(settings, trees, ldpath_mtimes, for node in clean_set: graph.add(node, None) - mydeps = [] for dep_type in dep_keys: depstr = node.metadata[dep_type] if not depstr: @@ -1153,7 +1160,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, % (priority,), noiselevel=-1, level=logging.DEBUG) try: - atoms = resolver._select_atoms(myroot, depstr, + atoms = resolver._select_atoms(eroot, depstr, myuse=node.use.enabled, parent=node, priority=priority)[node] except portage.exception.InvalidDependString: @@ -1226,7 +1233,7 @@ def calc_depclean(settings, trees, ldpath_mtimes, def action_deselect(settings, trees, opts, atoms): enter_invalid = '--ask-enter-invalid' in opts - root_config = trees[settings['ROOT']]['root_config'] + root_config = trees[settings['EROOT']]['root_config'] world_set = root_config.sets['selected'] if not hasattr(world_set, 'update'): writemsg_level("World @selected set does not appear to be mutable.\n", @@ -1291,7 +1298,7 @@ def action_deselect(settings, trees, opts, atoms): prompt = "Would you like to remove these " + \ "packages from your world favorites?" if userquery(prompt, enter_invalid) == 'No': - return os.EX_OK + return 128 + signal.SIGINT remaining = set(world_set) remaining.difference_update(discard_atoms) @@ -1325,11 +1332,12 @@ def action_info(settings, trees, myopts, myfiles): output_buffer = [] append = output_buffer.append - root_config = trees[settings['ROOT']]['root_config'] + root_config = trees[settings['EROOT']]['root_config'] + running_eroot = trees._running_eroot - append(getportageversion(settings["PORTDIR"], settings["ROOT"], + append(getportageversion(settings["PORTDIR"], None, settings.profile_path, settings["CHOST"], - trees[settings["ROOT"]]["vartree"].dbapi)) + trees[settings['EROOT']]["vartree"].dbapi)) header_width = 65 header_title = "System Settings" @@ -1347,7 +1355,14 @@ def action_info(settings, trees, myopts, myfiles): lastSync = "Unknown" append("Timestamp of tree: %s" % (lastSync,)) - output=subprocess_getstatusoutput("distcc --version") + try: + proc = subprocess.Popen(["distcc", "--version"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + output = (1, None) + else: + output = _unicode_decode(proc.communicate()[0]).rstrip("\n") + output = (proc.wait(), output) if output[0] == os.EX_OK: distcc_str = output[1].split("\n", 1)[0] if "distcc" in settings.features: @@ -1356,7 +1371,14 @@ def action_info(settings, trees, myopts, myfiles): distcc_str += " [disabled]" append(distcc_str) - output=subprocess_getstatusoutput("ccache -V") + try: + proc = subprocess.Popen(["ccache", "-V"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + output = (1, None) + else: + output = _unicode_decode(proc.communicate()[0]).rstrip("\n") + output = (proc.wait(), output) if output[0] == os.EX_OK: ccache_str = output[1].split("\n", 1)[0] if "ccache" in settings.features: @@ -1369,7 +1391,7 @@ def action_info(settings, trees, myopts, myfiles): "sys-devel/binutils", "sys-devel/libtool", "dev-lang/python"] myvars += portage.util.grabfile(settings["PORTDIR"]+"/profiles/info_pkgs") atoms = [] - vardb = trees["/"]["vartree"].dbapi + vardb = trees[running_eroot]['vartree'].dbapi for x in myvars: try: x = Atom(x) @@ -1382,7 +1404,7 @@ def action_info(settings, trees, myopts, myfiles): myvars = sorted(set(atoms)) - portdb = trees["/"]["porttree"].dbapi + portdb = trees[running_eroot]['porttree'].dbapi main_repo = portdb.getRepositoryName(portdb.porttree_root) cp_map = {} cp_max_len = 0 @@ -1425,8 +1447,6 @@ def action_info(settings, trees, myopts, myfiles): append("%s %s" % \ ((cp + ":").ljust(cp_max_len + 1), versions)) - libtool_vers = ",".join(trees["/"]["vartree"].dbapi.match("sys-devel/libtool")) - repos = portdb.settings.repositories if "--verbose" in myopts: append("Repositories:\n") @@ -1463,9 +1483,6 @@ def action_info(settings, trees, myopts, myfiles): myvars = portage.util.unique_array(myvars) use_expand = settings.get('USE_EXPAND', '').split() use_expand.sort() - use_expand_hidden = set( - settings.get('USE_EXPAND_HIDDEN', '').upper().split()) - alphabetical_use = '--alphabetical' in myopts unset_vars = [] myvars.sort() for k in myvars: @@ -1504,9 +1521,10 @@ def action_info(settings, trees, myopts, myfiles): # See if we can find any packages installed matching the strings # passed on the command line mypkgs = [] - vardb = trees[settings["ROOT"]]["vartree"].dbapi - portdb = trees[settings["ROOT"]]["porttree"].dbapi - bindb = trees[settings["ROOT"]]["bintree"].dbapi + eroot = settings['EROOT'] + vardb = trees[eroot]["vartree"].dbapi + portdb = trees[eroot]['porttree'].dbapi + bindb = trees[eroot]["bintree"].dbapi for x in myfiles: match_found = False installed_match = vardb.match(x) @@ -1541,7 +1559,6 @@ def action_info(settings, trees, myopts, myfiles): mydesiredvars = [ 'CHOST', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS' ] auxkeys = mydesiredvars + list(vardb._aux_cache_keys) auxkeys.append('DEFINED_PHASES') - global_vals = {} pkgsettings = portage.config(clone=settings) # Loop through each package @@ -1611,19 +1628,19 @@ def action_info(settings, trees, myopts, myfiles): continue if pkg_type == "installed": - portage.doebuild(ebuildpath, "info", pkgsettings["ROOT"], - pkgsettings, debug=(settings.get("PORTAGE_DEBUG", "") == 1), - mydbapi=trees[settings["ROOT"]]["vartree"].dbapi, + portage.doebuild(ebuildpath, "info", settings=pkgsettings, + debug=(settings.get("PORTAGE_DEBUG", "") == 1), + mydbapi=trees[settings['EROOT']]["vartree"].dbapi, tree="vartree") elif pkg_type == "ebuild": - portage.doebuild(ebuildpath, "info", pkgsettings["ROOT"], - pkgsettings, debug=(settings.get("PORTAGE_DEBUG", "") == 1), - mydbapi=trees[settings["ROOT"]]["porttree"].dbapi, + portage.doebuild(ebuildpath, "info", settings=pkgsettings, + debug=(settings.get("PORTAGE_DEBUG", "") == 1), + mydbapi=trees[settings['EROOT']]['porttree'].dbapi, tree="porttree") elif pkg_type == "binary": - portage.doebuild(ebuildpath, "info", pkgsettings["ROOT"], - pkgsettings, debug=(settings.get("PORTAGE_DEBUG", "") == 1), - mydbapi=trees[settings["ROOT"]]["bintree"].dbapi, + portage.doebuild(ebuildpath, "info", settings=pkgsettings, + debug=(settings.get("PORTAGE_DEBUG", "") == 1), + mydbapi=trees[settings['EROOT']]["bintree"].dbapi, tree="bintree") shutil.rmtree(tmpdir) @@ -1643,8 +1660,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None): if not os.path.exists(cachedir): os.makedirs(cachedir) - auxdbkeys = [x for x in portage.auxdbkeys if not x.startswith("UNUSED_0")] - auxdbkeys = tuple(auxdbkeys) + auxdbkeys = portdb._known_keys class TreeData(object): __slots__ = ('dest_db', 'eclass_db', 'path', 'src_db', 'valid_nodes') @@ -1658,18 +1674,14 @@ def action_metadata(settings, portdb, myopts, porttrees=None): porttrees_data = [] for path in porttrees: src_db = portdb._pregen_auxdb.get(path) - if src_db is None and \ - os.path.isdir(os.path.join(path, 'metadata', 'cache')): - src_db = portdb.metadbmodule( - path, 'metadata/cache', auxdbkeys, readonly=True) - try: - src_db.ec = portdb._repo_info[path].eclass_db - except AttributeError: - pass + if src_db is None: + # portdbapi does not populate _pregen_auxdb + # when FEATURES=metadata-transfer is enabled + src_db = portdb._create_pregen_cache(path) if src_db is not None: porttrees_data.append(TreeData(portdb.auxdb[path], - portdb._repo_info[path].eclass_db, path, src_db)) + portdb.repositories.get_repo_for_location(path).eclass_db, path, src_db)) porttrees = [tree_data.path for tree_data in porttrees_data] @@ -1704,42 +1716,45 @@ def action_metadata(settings, portdb, myopts, porttrees=None): if onProgress is not None: onProgress(maxval, curval) - from portage.cache.util import quiet_mirroring - from portage import eapi_is_supported, \ - _validate_cache_for_unsupported_eapis - # TODO: Display error messages, but do not interfere with the progress bar. # Here's how: # 1) erase the progress bar # 2) show the error message # 3) redraw the progress bar on a new line - noise = quiet_mirroring() for cp in cp_all: for tree_data in porttrees_data: + + src_chf = tree_data.src_db.validation_chf + dest_chf = tree_data.dest_db.validation_chf + dest_chf_key = '_%s_' % dest_chf + dest_chf_getter = operator.attrgetter(dest_chf) + for cpv in portdb.cp_list(cp, mytree=tree_data.path): tree_data.valid_nodes.add(cpv) try: src = tree_data.src_db[cpv] - except KeyError as e: - noise.missing_entry(cpv) - del e + except (CacheError, KeyError): continue - except CacheError as ce: - noise.exception(cpv, ce) - del ce + + ebuild_location = portdb.findname(cpv, mytree=tree_data.path) + if ebuild_location is None: + continue + ebuild_hash = hashed_path(ebuild_location) + + try: + if not tree_data.src_db.validate_entry(src, + ebuild_hash, tree_data.eclass_db): + continue + except CacheError: continue eapi = src.get('EAPI') if not eapi: eapi = '0' - eapi = eapi.lstrip('-') eapi_supported = eapi_is_supported(eapi) if not eapi_supported: - if not _validate_cache_for_unsupported_eapis: - noise.misc(cpv, "unable to validate " + \ - "cache for EAPI='%s'" % eapi) - continue + continue dest = None try: @@ -1751,18 +1766,30 @@ def action_metadata(settings, portdb, myopts, porttrees=None): if d is not None and d.get('EAPI') in ('', '0'): del d['EAPI'] + if src_chf != 'mtime': + # src may contain an irrelevant _mtime_ which corresponds + # to the time that the cache entry was written + src.pop('_mtime_', None) + + if src_chf != dest_chf: + # populate src entry with dest_chf_key + # (the validity of the dest_chf that we generate from the + # ebuild here relies on the fact that we already used + # validate_entry to validate the ebuild with src_chf) + src[dest_chf_key] = dest_chf_getter(ebuild_hash) + if dest is not None: - if not (dest['_mtime_'] == src['_mtime_'] and \ - tree_data.eclass_db.is_eclass_data_valid( - dest['_eclasses_']) and \ + if not (dest[dest_chf_key] == src[dest_chf_key] and \ + tree_data.eclass_db.validate_and_rewrite_cache( + dest['_eclasses_'], tree_data.dest_db.validation_chf, + tree_data.dest_db.store_eclass_paths) is not None and \ set(dest['_eclasses_']) == set(src['_eclasses_'])): dest = None else: # We don't want to skip the write unless we're really # sure that the existing cache is identical, so don't # trust _mtime_ and _eclasses_ alone. - for k in set(chain(src, dest)).difference( - ('_mtime_', '_eclasses_')): + for k in auxdbkeys: if dest.get(k, '') != src.get(k, ''): dest = None break @@ -1773,56 +1800,10 @@ def action_metadata(settings, portdb, myopts, porttrees=None): continue try: - inherited = src.get('INHERITED', '') - eclasses = src.get('_eclasses_') - except CacheError as ce: - noise.exception(cpv, ce) - del ce - continue - - if eclasses is not None: - if not tree_data.eclass_db.is_eclass_data_valid( - src['_eclasses_']): - noise.eclass_stale(cpv) - continue - inherited = eclasses - else: - inherited = inherited.split() - - if tree_data.src_db.complete_eclass_entries and \ - eclasses is None: - noise.corruption(cpv, "missing _eclasses_ field") - continue - - if inherited: - # Even if _eclasses_ already exists, replace it with data from - # eclass_cache, in order to insert local eclass paths. - try: - eclasses = tree_data.eclass_db.get_eclass_data(inherited) - except KeyError: - # INHERITED contains a non-existent eclass. - noise.eclass_stale(cpv) - continue - - if eclasses is None: - noise.eclass_stale(cpv) - continue - src['_eclasses_'] = eclasses - else: - src['_eclasses_'] = {} - - if not eapi_supported: - src = { - 'EAPI' : '-' + eapi, - '_mtime_' : src['_mtime_'], - '_eclasses_' : src['_eclasses_'], - } - - try: tree_data.dest_db[cpv] = src - except CacheError as ce: - noise.exception(cpv, ce) - del ce + except CacheError: + # ignore it; can't do anything about it. + pass curval += 1 if onProgress is not None: @@ -1860,12 +1841,6 @@ def action_regen(settings, portdb, max_jobs, max_load): xterm_titles = "notitles" not in settings.features emergelog(xterm_titles, " === regen") #regenerate cache entries - try: - os.close(sys.stdin.fileno()) - except SystemExit as e: - raise # Needed else can't exit - except: - pass sys.stdout.flush() regen = MetadataRegen(portdb, max_jobs=max_jobs, max_load=max_load) @@ -1921,7 +1896,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): enter_invalid = '--ask-enter-invalid' in myopts xterm_titles = "notitles" not in settings.features emergelog(xterm_titles, " === sync") - portdb = trees[settings["ROOT"]]["porttree"].dbapi + portdb = trees[settings['EROOT']]['porttree'].dbapi myportdir = portdb.porttree_root if not myportdir: myportdir = settings.get('PORTDIR', '') @@ -1993,6 +1968,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): os.umask(0o022) dosyncuri = syncuri updatecache_flg = False + git = False if myaction == "metadata": print("skipping sync") updatecache_flg = True @@ -2021,9 +1997,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): msg = ">>> Git pull in %s successful" % myportdir emergelog(xterm_titles, msg) writemsg_level(msg + "\n") - exitcode = git_sync_timestamps(settings, myportdir) - if exitcode == os.EX_OK: - updatecache_flg = True + git = True elif syncuri[:8]=="rsync://" or syncuri[:6]=="ssh://": for vcs_dir in vcs_dirs: writemsg_level(("!!! %s appears to be under revision " + \ @@ -2050,6 +2024,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): "--whole-file", # Don't do block transfers, only entire files "--delete", # Delete files that aren't in the master tree "--stats", # Show final statistics about what was transfered + "--human-readable", "--timeout="+str(mytimeout), # IO timeout if not done in X seconds "--exclude=/distfiles", # Exclude distfiles from consideration "--exclude=/local", # Exclude local from consideration @@ -2237,7 +2212,7 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): print() print("Quitting.") print() - sys.exit(0) + sys.exit(128 + signal.SIGINT) emergelog(xterm_titles, ">>> Starting rsync with " + dosyncuri) if "--quiet" not in myopts: print(">>> Starting rsync with "+dosyncuri+"...") @@ -2465,17 +2440,25 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): noiselevel=-1, level=logging.ERROR) return 1 + # Reload the whole config from scratch. + settings, trees, mtimedb = load_emerge_config(trees=trees) + adjust_configs(myopts, trees) + root_config = trees[settings['EROOT']]['root_config'] + portdb = trees[settings['EROOT']]['porttree'].dbapi + + if git: + # NOTE: Do this after reloading the config, in case + # it did not exist prior to sync, so that the config + # and portdb properly account for its existence. + exitcode = git_sync_timestamps(portdb, myportdir) + if exitcode == os.EX_OK: + updatecache_flg = True + if updatecache_flg and \ myaction != "metadata" and \ "metadata-transfer" not in settings.features: updatecache_flg = False - # Reload the whole config from scratch. - settings, trees, mtimedb = load_emerge_config(trees=trees) - adjust_configs(myopts, trees) - root_config = trees[settings["ROOT"]]["root_config"] - portdb = trees[settings["ROOT"]]["porttree"].dbapi - if updatecache_flg and \ os.path.exists(os.path.join(myportdir, 'metadata', 'cache')): @@ -2489,13 +2472,13 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): # Reload the whole config from scratch. settings, trees, mtimedb = load_emerge_config(trees=trees) adjust_configs(myopts, trees) - portdb = trees[settings["ROOT"]]["porttree"].dbapi - root_config = trees[settings["ROOT"]]["root_config"] + portdb = trees[settings['EROOT']]['porttree'].dbapi + root_config = trees[settings['EROOT']]['root_config'] mybestpv = portdb.xmatch("bestmatch-visible", portage.const.PORTAGE_PACKAGE_ATOM) mypvs = portage.best( - trees[settings["ROOT"]]["vartree"].dbapi.match( + trees[settings['EROOT']]['vartree'].dbapi.match( portage.const.PORTAGE_PACKAGE_ATOM)) chk_updated_cfg_files(settings["EROOT"], @@ -2514,10 +2497,10 @@ def action_sync(settings, trees, mtimedb, myopts, myaction): if(mybestpv != mypvs) and not "--quiet" in myopts: print() - print(red(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended") - print(red(" * ")+"that you update portage now, before any other packages are updated.") + print(warn(" * ")+bold("An update to portage is available.")+" It is _highly_ recommended") + print(warn(" * ")+"that you update portage now, before any other packages are updated.") print() - print(red(" * ")+"To update portage, run 'emerge portage' now.") + print(warn(" * ")+"To update portage, run 'emerge portage' now.") print() display_news_notification(root_config, myopts) @@ -2528,7 +2511,8 @@ def action_uninstall(settings, trees, ldpath_mtimes, # For backward compat, some actions do not require leading '='. ignore_missing_eq = action in ('clean', 'unmerge') root = settings['ROOT'] - vardb = trees[root]['vartree'].dbapi + eroot = settings['EROOT'] + vardb = trees[settings['EROOT']]['vartree'].dbapi valid_atoms = [] lookup_owners = [] @@ -2566,9 +2550,9 @@ def action_uninstall(settings, trees, ldpath_mtimes, valid_atoms.append(atom) elif x.startswith(os.sep): - if not x.startswith(root): + if not x.startswith(eroot): writemsg_level(("!!! '%s' does not start with" + \ - " $ROOT.\n") % x, level=logging.ERROR, noiselevel=-1) + " $EROOT.\n") % x, level=logging.ERROR, noiselevel=-1) return 1 # Queue these up since it's most efficient to handle # multiple files in a single iter_owners() call. @@ -2661,7 +2645,7 @@ def action_uninstall(settings, trees, ldpath_mtimes, # redirection of ebuild phase output to logs as required for # options such as --quiet. sched = Scheduler(settings, trees, None, opts, - spinner) + spinner, uninstall_only=True) sched._background = sched._background_mode() sched._status_display.quiet = True @@ -2670,16 +2654,15 @@ def action_uninstall(settings, trees, ldpath_mtimes, sched.settings["PORTAGE_BACKGROUND"] = "1" sched.settings.backup_changes("PORTAGE_BACKGROUND") sched.settings.lock() - sched.pkgsettings[root] = portage.config(clone=sched.settings) + sched.pkgsettings[eroot] = portage.config(clone=sched.settings) if action in ('clean', 'unmerge') or \ (action == 'prune' and "--nodeps" in opts): # When given a list of atoms, unmerge them in the order given. ordered = action == 'unmerge' - unmerge(trees[settings["ROOT"]]['root_config'], opts, action, + rval = unmerge(trees[settings['EROOT']]['root_config'], opts, action, valid_atoms, ldpath_mtimes, ordered=ordered, scheduler=sched._sched_iface) - rval = os.EX_OK else: rval = action_depclean(settings, trees, ldpath_mtimes, opts, action, valid_atoms, spinner, scheduler=sched._sched_iface) @@ -2730,7 +2713,13 @@ def adjust_config(myopts, settings): settings["EMERGE_WARNING_DELAY"] = str(EMERGE_WARNING_DELAY) settings.backup_changes("EMERGE_WARNING_DELAY") - if "--quiet" in myopts or "--quiet-build" in myopts: + buildpkg = myopts.get("--buildpkg") + if buildpkg is True: + settings.features.add("buildpkg") + elif buildpkg == 'n': + settings.features.discard("buildpkg") + + if "--quiet" in myopts: settings["PORTAGE_QUIET"]="1" settings.backup_changes("PORTAGE_QUIET") @@ -2766,8 +2755,8 @@ def adjust_config(myopts, settings): if settings.get("NOCOLOR") not in ("yes","true"): portage.output.havecolor = 1 - """The explicit --color < y | n > option overrides the NOCOLOR environment - variable and stdout auto-detection.""" + # The explicit --color < y | n > option overrides the NOCOLOR environment + # variable and stdout auto-detection. if "--color" in myopts: if "y" == myopts["--color"]: portage.output.havecolor = 1 @@ -2806,7 +2795,7 @@ def relative_profile_path(portdir, abs_profile): profilever = None return profilever -def getportageversion(portdir, target_root, profile, chost, vardb): +def getportageversion(portdir, _unused, profile, chost, vardb): profilever = None if profile: profilever = relative_profile_path(portdir, profile) @@ -2839,7 +2828,7 @@ def getportageversion(portdir, target_root, profile, chost, vardb): for cpv in sorted(libclist): libc_split = portage.catpkgsplit(cpv)[1:] if libc_split[-1] == "r0": - libc_split[:-1] + libc_split = libc_split[:-1] libcver.append("-".join(libc_split)) else: libcver = ["unavailable"] @@ -2850,27 +2839,35 @@ def getportageversion(portdir, target_root, profile, chost, vardb): return "Portage %s (%s, %s, %s, %s)" % \ (portage.VERSION, profilever, gccver, ",".join(libcver), unameout) -def git_sync_timestamps(settings, portdir): +def git_sync_timestamps(portdb, portdir): """ Since git doesn't preserve timestamps, synchronize timestamps between entries and ebuilds/eclasses. Assume the cache has the correct timestamp for a given file as long as the file in the working tree is not modified (relative to HEAD). """ - cache_dir = os.path.join(portdir, "metadata", "cache") - if not os.path.isdir(cache_dir): - return os.EX_OK - writemsg_level(">>> Synchronizing timestamps...\n") - from portage.cache.cache_errors import CacheError + cache_db = portdb._pregen_auxdb.get(portdir) + try: - cache_db = settings.load_best_module("portdbapi.metadbmodule")( - portdir, "metadata/cache", portage.auxdbkeys[:], readonly=True) + if cache_db is None: + # portdbapi does not populate _pregen_auxdb + # when FEATURES=metadata-transfer is enabled + cache_db = portdb._create_pregen_cache(portdir) except CacheError as e: writemsg_level("!!! Unable to instantiate cache: %s\n" % (e,), level=logging.ERROR, noiselevel=-1) return 1 + if cache_db is None: + return os.EX_OK + + if cache_db.validation_chf != 'mtime': + # newer formats like md5-dict do not require mtime sync + return os.EX_OK + + writemsg_level(">>> Synchronizing timestamps...\n") + ec_dir = os.path.join(portdir, "eclass") try: ec_names = set(f[:-7] for f in os.listdir(ec_dir) \ @@ -2883,10 +2880,10 @@ def git_sync_timestamps(settings, portdir): args = [portage.const.BASH_BINARY, "-c", "cd %s && git diff-index --name-only --diff-filter=M HEAD" % \ portage._shell_quote(portdir)] - import subprocess proc = subprocess.Popen(args, stdout=subprocess.PIPE) modified_files = set(_unicode_decode(l).rstrip("\n") for l in proc.stdout) rval = proc.wait() + proc.stdout.close() if rval != os.EX_OK: return rval @@ -2990,22 +2987,15 @@ def load_emerge_config(trees=None): kwargs[k] = v trees = portage.create_trees(trees=trees, **kwargs) - for root, root_trees in trees.items(): + for root_trees in trees.values(): settings = root_trees["vartree"].settings settings._init_dirs() setconfig = load_default_config(settings, root_trees) root_trees["root_config"] = RootConfig(settings, root_trees, setconfig) - settings = trees["/"]["vartree"].settings - - for myroot in trees: - if myroot != "/": - settings = trees[myroot]["vartree"].settings - break - + settings = trees[trees._target_eroot]['vartree'].settings mtimedbfile = os.path.join(settings['EROOT'], portage.CACHE_PATH, "mtimedb") mtimedb = portage.MtimeDB(mtimedbfile) - portage.output._init(config_root=settings['PORTAGE_CONFIGROOT']) QueryCommand._db = trees return settings, trees, mtimedb @@ -3015,55 +3005,35 @@ def chk_updated_cfg_files(eroot, config_protect): portage.util.find_updated_config_files(target_root, config_protect)) for x in result: - writemsg_level("\n %s " % (colorize("WARN", "* IMPORTANT:"),), + writemsg_level("\n %s " % (colorize("WARN", "* " + _("IMPORTANT:"))), level=logging.INFO, noiselevel=-1) if not x[1]: # it's a protected file - writemsg_level("config file '%s' needs updating.\n" % x[0], + writemsg_level( _("config file '%s' needs updating.\n") % x[0], level=logging.INFO, noiselevel=-1) else: # it's a protected dir if len(x[1]) == 1: head, tail = os.path.split(x[1][0]) tail = tail[len("._cfg0000_"):] fpath = os.path.join(head, tail) - writemsg_level("config file '%s' needs updating.\n" % fpath, + writemsg_level(_("config file '%s' needs updating.\n") % fpath, level=logging.INFO, noiselevel=-1) else: - writemsg_level("%d config files in '%s' need updating.\n" % \ + writemsg_level( _("%d config files in '%s' need updating.\n") % \ (len(x[1]), x[0]), level=logging.INFO, noiselevel=-1) if result: - print(" "+yellow("*")+" See the "+colorize("INFORM","CONFIGURATION FILES")\ - + " section of the " + bold("emerge")) - print(" "+yellow("*")+" man page to learn how to update config files.") + print(" "+yellow("*")+ " See the "+colorize("INFORM", _("CONFIGURATION FILES"))\ + + " " + _("section of the") + " " + bold("emerge")) + print(" "+yellow("*")+ " " + _("man page to learn how to update config files.")) + def display_news_notification(root_config, myopts): - target_root = root_config.settings['EROOT'] - trees = root_config.trees - settings = trees["vartree"].settings - portdb = trees["porttree"].dbapi - vardb = trees["vartree"].dbapi - NEWS_PATH = os.path.join("metadata", "news") - UNREAD_PATH = os.path.join(target_root, NEWS_LIB_PATH, "news") - newsReaderDisplay = False - update = "--pretend" not in myopts - if "news" not in settings.features: + if "news" not in root_config.settings.features: return - - for repo in portdb.getRepositories(): - unreadItems = checkUpdatedNewsItems( - portdb, vardb, NEWS_PATH, UNREAD_PATH, repo, update=update) - if unreadItems: - if not newsReaderDisplay: - newsReaderDisplay = True - print() - print(colorize("WARN", " * IMPORTANT:"), end=' ') - print("%s news items need reading for repository '%s'." % (unreadItems, repo)) - - - if newsReaderDisplay: - print(colorize("WARN", " *"), end=' ') - print("Use " + colorize("GOOD", "eselect news") + " to read news items.") - print() + portdb = root_config.trees["porttree"].dbapi + vardb = root_config.trees["vartree"].dbapi + news_counts = count_unread_news(portdb, vardb) + display_news_notifications(news_counts) def getgccversion(chost): """ @@ -3071,7 +3041,7 @@ def getgccversion(chost): return: the current in-use gcc version """ - gcc_ver_command = 'gcc -dumpversion' + gcc_ver_command = ['gcc', '-dumpversion'] gcc_ver_prefix = 'gcc-' gcc_not_found_error = red( @@ -3080,44 +3050,42 @@ def getgccversion(chost): "!!! other terminals also.\n" ) - mystatus, myoutput = subprocess_getstatusoutput("gcc-config -c") + try: + proc = subprocess.Popen(["gcc-config", "-c"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + myoutput = None + mystatus = 1 + else: + myoutput = _unicode_decode(proc.communicate()[0]).rstrip("\n") + mystatus = proc.wait() if mystatus == os.EX_OK and myoutput.startswith(chost + "-"): return myoutput.replace(chost + "-", gcc_ver_prefix, 1) - mystatus, myoutput = subprocess_getstatusoutput( - chost + "-" + gcc_ver_command) + try: + proc = subprocess.Popen( + [chost + "-" + gcc_ver_command[0]] + gcc_ver_command[1:], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + myoutput = None + mystatus = 1 + else: + myoutput = _unicode_decode(proc.communicate()[0]).rstrip("\n") + mystatus = proc.wait() if mystatus == os.EX_OK: return gcc_ver_prefix + myoutput - mystatus, myoutput = subprocess_getstatusoutput(gcc_ver_command) + try: + proc = subprocess.Popen(gcc_ver_command, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + myoutput = None + mystatus = 1 + else: + myoutput = _unicode_decode(proc.communicate()[0]).rstrip("\n") + mystatus = proc.wait() if mystatus == os.EX_OK: return gcc_ver_prefix + myoutput portage.writemsg(gcc_not_found_error, noiselevel=-1) return "[unavailable]" - -def checkUpdatedNewsItems(portdb, vardb, NEWS_PATH, UNREAD_PATH, repo_id, - update=False): - """ - Examines news items in repodir + '/' + NEWS_PATH and attempts to find unread items - Returns the number of unread (yet relevent) items. - - @param portdb: a portage tree database - @type portdb: pordbapi - @param vardb: an installed package database - @type vardb: vardbapi - @param NEWS_PATH: - @type NEWS_PATH: - @param UNREAD_PATH: - @type UNREAD_PATH: - @param repo_id: - @type repo_id: - @rtype: Integer - @returns: - 1. The number of unread but relevant news items. - - """ - from portage.news import NewsManager - manager = NewsManager(portdb, vardb, NEWS_PATH, UNREAD_PATH) - return manager.getUnreadItems( repo_id, update=update ) - diff --git a/portage_with_autodep/pym/_emerge/actions.pyo b/portage_with_autodep/pym/_emerge/actions.pyo Binary files differnew file mode 100644 index 0000000..4fbda01 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/actions.pyo diff --git a/portage_with_autodep/pym/_emerge/chk_updated_cfg_files.py b/portage_with_autodep/pym/_emerge/chk_updated_cfg_files.py new file mode 100644 index 0000000..9f2ab6f --- /dev/null +++ b/portage_with_autodep/pym/_emerge/chk_updated_cfg_files.py @@ -0,0 +1,42 @@ +# Copyright 1999-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + +import logging + +import portage +from portage import os +from portage.localization import _ +from portage.output import bold, colorize, yellow +from portage.util import writemsg_level + +def chk_updated_cfg_files(eroot, config_protect): + target_root = eroot + result = list( + portage.util.find_updated_config_files(target_root, config_protect)) + + for x in result: + writemsg_level("\n %s " % (colorize("WARN", "* " + _("IMPORTANT:"))), + level=logging.INFO, noiselevel=-1) + if not x[1]: # it's a protected file + writemsg_level( _("config file '%s' needs updating.\n") % x[0], + level=logging.INFO, noiselevel=-1) + else: # it's a protected dir + if len(x[1]) == 1: + head, tail = os.path.split(x[1][0]) + tail = tail[len("._cfg0000_"):] + fpath = os.path.join(head, tail) + writemsg_level(_("config file '%s' needs updating.\n") % fpath, + level=logging.INFO, noiselevel=-1) + else: + writemsg_level( + _("%d config files in '%s' need updating.\n") % \ + (len(x[1]), x[0]), level=logging.INFO, noiselevel=-1) + + if result: + print(" " + yellow("*") + " See the " + + colorize("INFORM", _("CONFIGURATION FILES")) + + " " + _("section of the") + " " + bold("emerge")) + print(" " + yellow("*") + " " + + _("man page to learn how to update config files.")) diff --git a/portage_with_autodep/pym/_emerge/clear_caches.pyo b/portage_with_autodep/pym/_emerge/clear_caches.pyo Binary files differnew file mode 100644 index 0000000..2e6f010 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/clear_caches.pyo diff --git a/portage_with_autodep/pym/_emerge/countdown.pyo b/portage_with_autodep/pym/_emerge/countdown.pyo Binary files differnew file mode 100644 index 0000000..537dd27 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/countdown.pyo diff --git a/portage_with_autodep/pym/_emerge/create_depgraph_params.py b/portage_with_autodep/pym/_emerge/create_depgraph_params.py index 44dceda..8f15c68 100644 --- a/portage_with_autodep/pym/_emerge/create_depgraph_params.py +++ b/portage_with_autodep/pym/_emerge/create_depgraph_params.py @@ -21,6 +21,10 @@ def create_depgraph_params(myopts, myaction): if bdeps is not None: myparams["bdeps"] = bdeps + dynamic_deps = myopts.get("--dynamic-deps") + if dynamic_deps is not None: + myparams["dynamic_deps"] = dynamic_deps + if myaction == "remove": myparams["remove"] = True myparams["complete"] = True @@ -37,6 +41,12 @@ def create_depgraph_params(myopts, myaction): deep = myopts.get("--deep") if deep is not None and deep != 0: myparams["deep"] = deep + + complete_if_new_ver = \ + myopts.get("--complete-graph-if-new-ver") + if complete_if_new_ver is not None: + myparams["complete_if_new_ver"] = complete_if_new_ver + if ("--complete-graph" in myopts or "--rebuild-if-new-rev" in myopts or "--rebuild-if-new-ver" in myopts or "--rebuild-if-unbuilt" in myopts): myparams["complete"] = True @@ -58,6 +68,16 @@ def create_depgraph_params(myopts, myaction): '--update' in myopts: myparams['rebuilt_binaries'] = True + binpkg_respect_use = myopts.get('--binpkg-respect-use') + if binpkg_respect_use is not None: + myparams['binpkg_respect_use'] = binpkg_respect_use + elif '--usepkgonly' not in myopts: + # If --binpkg-respect-use is not explicitly specified, we enable + # the behavior automatically (like requested in bug #297549), as + # long as it doesn't strongly conflict with other options that + # have been specified. + myparams['binpkg_respect_use'] = 'auto' + if myopts.get("--selective") == "n": # --selective=n can be used to remove selective # behavior that may have been implied by some diff --git a/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo b/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo Binary files differnew file mode 100644 index 0000000..834580a --- /dev/null +++ b/portage_with_autodep/pym/_emerge/create_depgraph_params.pyo diff --git a/portage_with_autodep/pym/_emerge/create_world_atom.py b/portage_with_autodep/pym/_emerge/create_world_atom.py index fa7cffc..35fb7c4 100644 --- a/portage_with_autodep/pym/_emerge/create_world_atom.py +++ b/portage_with_autodep/pym/_emerge/create_world_atom.py @@ -21,8 +21,25 @@ def create_world_atom(pkg, args_set, root_config): sets = root_config.sets portdb = root_config.trees["porttree"].dbapi vardb = root_config.trees["vartree"].dbapi - available_slots = set(portdb.aux_get(cpv, ["SLOT"])[0] \ - for cpv in portdb.match(cp)) + + if arg_atom.repo is not None: + repos = [arg_atom.repo] + else: + # Iterate over portdbapi.porttrees, since it's common to + # tweak this attribute in order to adjust match behavior. + repos = [] + for tree in portdb.porttrees: + repos.append(portdb.repositories.get_name_for_location(tree)) + + available_slots = set() + for cpv in portdb.match(cp): + for repo in repos: + try: + available_slots.add(portdb.aux_get(cpv, ["SLOT"], + myrepo=repo)[0]) + except KeyError: + pass + slotted = len(available_slots) > 1 or \ (len(available_slots) == 1 and "0" not in available_slots) if not slotted: @@ -64,8 +81,18 @@ def create_world_atom(pkg, args_set, root_config): # enough to identify a specific slot. matches = mydb.match(arg_atom) matched_slots = set() - for cpv in matches: - matched_slots.add(mydb.aux_get(cpv, ["SLOT"])[0]) + if mydb is vardb: + for cpv in matches: + matched_slots.add(mydb.aux_get(cpv, ["SLOT"])[0]) + else: + for cpv in matches: + for repo in repos: + try: + matched_slots.add(portdb.aux_get(cpv, ["SLOT"], + myrepo=repo)[0]) + except KeyError: + pass + if len(matched_slots) == 1: new_world_atom = slot_atom if arg_atom.repo: diff --git a/portage_with_autodep/pym/_emerge/create_world_atom.pyo b/portage_with_autodep/pym/_emerge/create_world_atom.pyo Binary files differnew file mode 100644 index 0000000..ac3fb5d --- /dev/null +++ b/portage_with_autodep/pym/_emerge/create_world_atom.pyo diff --git a/portage_with_autodep/pym/_emerge/depgraph.py b/portage_with_autodep/pym/_emerge/depgraph.py index 5b48aca..572cea7 100644 --- a/portage_with_autodep/pym/_emerge/depgraph.py +++ b/portage_with_autodep/pym/_emerge/depgraph.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -18,7 +18,10 @@ from portage import os, OrderedDict from portage import _unicode_decode, _unicode_encode, _encodings from portage.const import PORTAGE_PACKAGE_ATOM, USER_CONFIG_PATH from portage.dbapi import dbapi -from portage.dep import Atom, extract_affecting_use, check_required_use, human_readable_required_use, _repo_separator +from portage.dbapi.dep_expand import dep_expand +from portage.dep import Atom, best_match_to_list, extract_affecting_use, \ + check_required_use, human_readable_required_use, match_from_list, \ + _repo_separator from portage.eapi import eapi_has_strong_blocks, eapi_has_required_use from portage.exception import InvalidAtom, InvalidDependString, PortageException from portage.output import colorize, create_color_func, \ @@ -92,15 +95,13 @@ class _frozen_depgraph_config(object): def __init__(self, settings, trees, myopts, spinner): self.settings = settings - self.target_root = settings["ROOT"] + self.target_root = settings["EROOT"] self.myopts = myopts self.edebug = 0 if settings.get("PORTAGE_DEBUG", "") == "1": self.edebug = 1 self.spinner = spinner - self._running_root = trees["/"]["root_config"] - self._opts_no_restart = frozenset(["--buildpkgonly", - "--fetchonly", "--fetch-all-uri", "--pretend"]) + self._running_root = trees[trees._running_eroot]["root_config"] self.pkgsettings = {} self.trees = {} self._trees_orig = trees @@ -108,6 +109,7 @@ class _frozen_depgraph_config(object): # All Package instances self._pkg_cache = {} self._highest_license_masked = {} + dynamic_deps = myopts.get("--dynamic-deps", "y") != "n" for myroot in trees: self.trees[myroot] = {} # Create a RootConfig instance that references @@ -121,7 +123,8 @@ class _frozen_depgraph_config(object): self.trees[myroot]["vartree"] = \ FakeVartree(trees[myroot]["root_config"], pkg_cache=self._pkg_cache, - pkg_root_config=self.roots[myroot]) + pkg_root_config=self.roots[myroot], + dynamic_deps=dynamic_deps) self.pkgsettings[myroot] = portage.config( clone=self.trees[myroot]["vartree"].settings) @@ -174,7 +177,7 @@ class _rebuild_config(object): rebuild_exclude = self._frozen_config.rebuild_exclude rebuild_ignore = self._frozen_config.rebuild_ignore if (self.rebuild and isinstance(parent, Package) and - parent.built and (priority.buildtime or priority.runtime) and + parent.built and priority.buildtime and isinstance(dep_pkg, Package) and not rebuild_exclude.findAtomForPackage(parent) and not rebuild_ignore.findAtomForPackage(dep_pkg)): @@ -209,66 +212,63 @@ class _rebuild_config(object): return True - def _trigger_rebuild(self, parent, build_deps, runtime_deps): + def _trigger_rebuild(self, parent, build_deps): root_slot = (parent.root, parent.slot_atom) if root_slot in self.rebuild_list: return False trees = self._frozen_config.trees - children = set(build_deps).intersection(runtime_deps) reinstall = False - for slot_atom in children: - kids = set([build_deps[slot_atom], runtime_deps[slot_atom]]) - for dep_pkg in kids: - dep_root_slot = (dep_pkg.root, slot_atom) - if self._needs_rebuild(dep_pkg): + for slot_atom, dep_pkg in build_deps.items(): + dep_root_slot = (dep_pkg.root, slot_atom) + if self._needs_rebuild(dep_pkg): + self.rebuild_list.add(root_slot) + return True + elif ("--usepkg" in self._frozen_config.myopts and + (dep_root_slot in self.reinstall_list or + dep_root_slot in self.rebuild_list or + not dep_pkg.installed)): + + # A direct rebuild dependency is being installed. We + # should update the parent as well to the latest binary, + # if that binary is valid. + # + # To validate the binary, we check whether all of the + # rebuild dependencies are present on the same binhost. + # + # 1) If parent is present on the binhost, but one of its + # rebuild dependencies is not, then the parent should + # be rebuilt from source. + # 2) Otherwise, the parent binary is assumed to be valid, + # because all of its rebuild dependencies are + # consistent. + bintree = trees[parent.root]["bintree"] + uri = bintree.get_pkgindex_uri(parent.cpv) + dep_uri = bintree.get_pkgindex_uri(dep_pkg.cpv) + bindb = bintree.dbapi + if self.rebuild_if_new_ver and uri and uri != dep_uri: + cpv_norev = catpkgsplit(dep_pkg.cpv)[:-1] + for cpv in bindb.match(dep_pkg.slot_atom): + if cpv_norev == catpkgsplit(cpv)[:-1]: + dep_uri = bintree.get_pkgindex_uri(cpv) + if uri == dep_uri: + break + if uri and uri != dep_uri: + # 1) Remote binary package is invalid because it was + # built without dep_pkg. Force rebuild. self.rebuild_list.add(root_slot) return True - elif ("--usepkg" in self._frozen_config.myopts and - (dep_root_slot in self.reinstall_list or - dep_root_slot in self.rebuild_list or - not dep_pkg.installed)): - - # A direct rebuild dependency is being installed. We - # should update the parent as well to the latest binary, - # if that binary is valid. - # - # To validate the binary, we check whether all of the - # rebuild dependencies are present on the same binhost. - # - # 1) If parent is present on the binhost, but one of its - # rebuild dependencies is not, then the parent should - # be rebuilt from source. - # 2) Otherwise, the parent binary is assumed to be valid, - # because all of its rebuild dependencies are - # consistent. - bintree = trees[parent.root]["bintree"] - uri = bintree.get_pkgindex_uri(parent.cpv) - dep_uri = bintree.get_pkgindex_uri(dep_pkg.cpv) - bindb = bintree.dbapi - if self.rebuild_if_new_ver and uri and uri != dep_uri: - cpv_norev = catpkgsplit(dep_pkg.cpv)[:-1] - for cpv in bindb.match(dep_pkg.slot_atom): - if cpv_norev == catpkgsplit(cpv)[:-1]: - dep_uri = bintree.get_pkgindex_uri(cpv) - if uri == dep_uri: - break - if uri and uri != dep_uri: - # 1) Remote binary package is invalid because it was - # built without dep_pkg. Force rebuild. - self.rebuild_list.add(root_slot) - return True - elif (parent.installed and - root_slot not in self.reinstall_list): - inst_build_time = parent.metadata.get("BUILD_TIME") - try: - bin_build_time, = bindb.aux_get(parent.cpv, - ["BUILD_TIME"]) - except KeyError: - continue - if bin_build_time != inst_build_time: - # 2) Remote binary package is valid, and local package - # is not up to date. Force reinstall. - reinstall = True + elif (parent.installed and + root_slot not in self.reinstall_list): + inst_build_time = parent.metadata.get("BUILD_TIME") + try: + bin_build_time, = bindb.aux_get(parent.cpv, + ["BUILD_TIME"]) + except KeyError: + continue + if bin_build_time != inst_build_time: + # 2) Remote binary package is valid, and local package + # is not up to date. Force reinstall. + reinstall = True if reinstall: self.reinstall_list.add(root_slot) return reinstall @@ -282,31 +282,15 @@ class _rebuild_config(object): need_restart = False graph = self._graph build_deps = {} - runtime_deps = {} - leaf_nodes = deque(graph.leaf_nodes()) - - def ignore_non_runtime(priority): - return not priority.runtime - def ignore_non_buildtime(priority): - return not priority.buildtime + leaf_nodes = deque(graph.leaf_nodes()) # Trigger rebuilds bottom-up (starting with the leaves) so that parents # will always know which children are being rebuilt. while graph: if not leaf_nodes: - # We're interested in intersection of buildtime and runtime, - # so ignore edges that do not contain both. - leaf_nodes.extend(graph.leaf_nodes( - ignore_priority=ignore_non_runtime)) - if not leaf_nodes: - leaf_nodes.extend(graph.leaf_nodes( - ignore_priority=ignore_non_buildtime)) - if not leaf_nodes: - # We'll have to drop an edge that is both - # buildtime and runtime. This should be - # quite rare. - leaf_nodes.append(graph.order[-1]) + # We'll have to drop an edge. This should be quite rare. + leaf_nodes.append(graph.order[-1]) node = leaf_nodes.popleft() if node not in graph: @@ -315,32 +299,23 @@ class _rebuild_config(object): slot_atom = node.slot_atom # Remove our leaf node from the graph, keeping track of deps. - parents = graph.nodes[node][1].items() + parents = graph.parent_nodes(node) graph.remove(node) node_build_deps = build_deps.get(node, {}) - node_runtime_deps = runtime_deps.get(node, {}) - for parent, priorities in parents: + for parent in parents: if parent == node: # Ignore a direct cycle. continue parent_bdeps = build_deps.setdefault(parent, {}) - parent_rdeps = runtime_deps.setdefault(parent, {}) - for priority in priorities: - if priority.buildtime: - parent_bdeps[slot_atom] = node - if priority.runtime: - parent_rdeps[slot_atom] = node - if slot_atom in parent_bdeps and slot_atom in parent_rdeps: - parent_rdeps.update(node_runtime_deps) + parent_bdeps[slot_atom] = node if not graph.child_nodes(parent): leaf_nodes.append(parent) # Trigger rebuilds for our leaf node. Because all of our children - # have been processed, build_deps and runtime_deps will be - # completely filled in, and self.rebuild_list / self.reinstall_list - # will tell us whether any of our children need to be rebuilt or - # reinstalled. - if self._trigger_rebuild(node, node_build_deps, node_runtime_deps): + # have been processed, the build_deps will be completely filled in, + # and self.rebuild_list / self.reinstall_list will tell us whether + # any of our children need to be rebuilt or reinstalled. + if self._trigger_rebuild(node, node_build_deps): need_restart = True return need_restart @@ -416,6 +391,11 @@ class _dynamic_depgraph_config(object): self._ignored_deps = [] self._highest_pkg_cache = {} + # Binary packages that have been rejected because their USE + # didn't match the user's config. It maps packages to a set + # of flags causing the rejection. + self.ignored_binaries = {} + self._needed_unstable_keywords = backtrack_parameters.needed_unstable_keywords self._needed_p_mask_changes = backtrack_parameters.needed_p_mask_changes self._needed_license_changes = backtrack_parameters.needed_license_changes @@ -536,9 +516,15 @@ class depgraph(object): for myroot in self._frozen_config.trees: + dynamic_deps = self._dynamic_config.myparams.get( + "dynamic_deps", "y") != "n" preload_installed_pkgs = \ "--nodeps" not in self._frozen_config.myopts + if self._frozen_config.myopts.get("--root-deps") is not None and \ + myroot != self._frozen_config.target_root: + continue + fake_vartree = self._frozen_config.trees[myroot]["vartree"] if not fake_vartree.dbapi: # This needs to be called for the first depgraph, but not for @@ -557,8 +543,11 @@ class depgraph(object): for pkg in vardb: self._spinner_update() - # This triggers metadata updates via FakeVartree. - vardb.aux_get(pkg.cpv, []) + if dynamic_deps: + # This causes FakeVartree to update the + # Package instance dependencies via + # PackageVirtualDbapi.aux_update() + vardb.aux_get(pkg.cpv, []) fakedb.cpv_inject(pkg) self._dynamic_config._vdb_loaded = True @@ -567,6 +556,67 @@ class depgraph(object): if self._frozen_config.spinner: self._frozen_config.spinner.update() + def _show_ignored_binaries(self): + """ + Show binaries that have been ignored because their USE didn't + match the user's config. + """ + if not self._dynamic_config.ignored_binaries \ + or '--quiet' in self._frozen_config.myopts \ + or self._dynamic_config.myparams.get( + "binpkg_respect_use") in ("y", "n"): + return + + for pkg in list(self._dynamic_config.ignored_binaries): + + selected_pkg = self._dynamic_config.mydbapi[pkg.root + ].match_pkgs(pkg.slot_atom) + + if not selected_pkg: + continue + + selected_pkg = selected_pkg[-1] + if selected_pkg > pkg: + self._dynamic_config.ignored_binaries.pop(pkg) + continue + + if selected_pkg.installed and \ + selected_pkg.cpv == pkg.cpv and \ + selected_pkg.metadata.get('BUILD_TIME') == \ + pkg.metadata.get('BUILD_TIME'): + # We don't care about ignored binaries when an + # identical installed instance is selected to + # fill the slot. + self._dynamic_config.ignored_binaries.pop(pkg) + continue + + if not self._dynamic_config.ignored_binaries: + return + + self._show_merge_list() + + writemsg("\n!!! The following binary packages have been ignored " + \ + "due to non matching USE:\n\n", noiselevel=-1) + + for pkg, flags in self._dynamic_config.ignored_binaries.items(): + writemsg(" =%s" % pkg.cpv, noiselevel=-1) + if pkg.root_config.settings["ROOT"] != "/": + writemsg(" for %s" % (pkg.root,), noiselevel=-1) + writemsg("\n use flag(s): %s\n" % ", ".join(sorted(flags)), + noiselevel=-1) + + msg = [ + "", + "NOTE: The --binpkg-respect-use=n option will prevent emerge", + " from ignoring these binary packages if possible.", + " Using --binpkg-respect-use=y will silence this warning." + ] + + for line in msg: + if line: + line = colorize("INFORM", line) + writemsg(line + "\n", noiselevel=-1) + def _show_missed_update(self): # In order to minimize noise, show only the highest @@ -578,6 +628,10 @@ class depgraph(object): # Exclude installed here since we only # want to show available updates. continue + chosen_pkg = self._dynamic_config.mydbapi[pkg.root + ].match_pkgs(pkg.slot_atom) + if not chosen_pkg or chosen_pkg[-1] >= pkg: + continue k = (pkg.root, pkg.slot_atom) if k in missed_updates: other_pkg, mask_type, parent_atoms = missed_updates[k] @@ -613,6 +667,7 @@ class depgraph(object): if not missed_updates: return + self._show_merge_list() backtrack_masked = [] for pkg, parent_atoms in missed_updates: @@ -630,7 +685,7 @@ class depgraph(object): "due to unsatisfied dependencies:\n\n", noiselevel=-1) writemsg(str(pkg.slot_atom), noiselevel=-1) - if pkg.root != '/': + if pkg.root_config.settings["ROOT"] != "/": writemsg(" for %s" % (pkg.root,), noiselevel=-1) writemsg("\n", noiselevel=-1) @@ -646,7 +701,7 @@ class depgraph(object): "!!! triggered by backtracking:\n\n", noiselevel=-1) for pkg, parent_atoms in backtrack_masked: writemsg(str(pkg.slot_atom), noiselevel=-1) - if pkg.root != '/': + if pkg.root_config.settings["ROOT"] != "/": writemsg(" for %s" % (pkg.root,), noiselevel=-1) writemsg("\n", noiselevel=-1) @@ -655,6 +710,7 @@ class depgraph(object): if not missed_updates: return + self._show_merge_list() msg = [] msg.append("\nWARNING: One or more updates have been " + \ "skipped due to a dependency conflict:\n\n") @@ -662,7 +718,7 @@ class depgraph(object): indent = " " for pkg, parent_atoms in missed_updates: msg.append(str(pkg.slot_atom)) - if pkg.root != '/': + if pkg.root_config.settings["ROOT"] != "/": msg.append(" for %s" % (pkg.root,)) msg.append("\n\n") @@ -777,19 +833,28 @@ class depgraph(object): else: self._dynamic_config._slot_conflict_parent_atoms.add(parent_atom) - def _reinstall_for_flags(self, forced_flags, + def _reinstall_for_flags(self, pkg, forced_flags, orig_use, orig_iuse, cur_use, cur_iuse): """Return a set of flags that trigger reinstallation, or None if there are no such flags.""" - if "--newuse" in self._frozen_config.myopts or \ - "--binpkg-respect-use" in self._frozen_config.myopts: + + # binpkg_respect_use: Behave like newuse by default. If newuse is + # False and changed_use is True, then behave like changed_use. + binpkg_respect_use = (pkg.built and + self._dynamic_config.myparams.get("binpkg_respect_use") + in ("y", "auto")) + newuse = "--newuse" in self._frozen_config.myopts + changed_use = "changed-use" == self._frozen_config.myopts.get("--reinstall") + + if newuse or (binpkg_respect_use and not changed_use): flags = set(orig_iuse.symmetric_difference( cur_iuse).difference(forced_flags)) flags.update(orig_iuse.intersection(orig_use).symmetric_difference( cur_iuse.intersection(cur_use))) if flags: return flags - elif "changed-use" == self._frozen_config.myopts.get("--reinstall"): + + elif changed_use or binpkg_respect_use: flags = orig_iuse.intersection(orig_use).symmetric_difference( cur_iuse.intersection(cur_use)) if flags: @@ -827,7 +892,7 @@ class depgraph(object): relationships from nested sets @type add_to_digraph: Boolean @rtype: Iterable - @returns: All args given in the input together with additional + @return: All args given in the input together with additional SetArg instances that are generated from nested sets """ @@ -876,8 +941,6 @@ class depgraph(object): debug = "--debug" in self._frozen_config.myopts buildpkgonly = "--buildpkgonly" in self._frozen_config.myopts nodeps = "--nodeps" in self._frozen_config.myopts - deep = self._dynamic_config.myparams.get("deep", 0) - recurse = deep is True or dep.depth <= deep if dep.blocker: if not buildpkgonly and \ not nodeps and \ @@ -922,7 +985,7 @@ class depgraph(object): # infinite backtracking loop. if self._dynamic_config._allow_backtracking: if dep.parent in self._dynamic_config._runtime_pkg_mask: - if "--debug" in self._frozen_config.myopts: + if debug: writemsg( "!!! backtracking loop detected: %s %s\n" % \ (dep.parent, @@ -937,7 +1000,7 @@ class depgraph(object): if dep_pkg is None: self._dynamic_config._backtrack_infos["missing dependency"] = dep self._dynamic_config._need_restart = True - if "--debug" in self._frozen_config.myopts: + if debug: msg = [] msg.append("") msg.append("") @@ -1009,17 +1072,18 @@ class depgraph(object): else: # Display the specific atom from SetArg or # Package types. + uneval = "" + if dep.atom is not dep.atom.unevaluated_atom: + uneval = " (%s)" % (dep.atom.unevaluated_atom,) writemsg_level( - "%s%s required by %s\n" % - ("Parent Dep:".ljust(15), dep.atom, myparent), + "%s%s%s required by %s\n" % + ("Parent Dep:".ljust(15), dep.atom, uneval, myparent), level=logging.DEBUG, noiselevel=-1) # Ensure that the dependencies of the same package # are never processed more than once. previously_added = pkg in self._dynamic_config.digraph - # select the correct /var database that we'll be checking against - vardbapi = self._frozen_config.trees[pkg.root]["vartree"].dbapi pkgsettings = self._frozen_config.pkgsettings[pkg.root] arg_atoms = None @@ -1036,7 +1100,7 @@ class depgraph(object): # package selection, since we want to prompt the user # for USE adjustment rather than have REQUIRED_USE # affect package selection and || dep choices. - if not pkg.built and pkg.metadata["REQUIRED_USE"] and \ + if not pkg.built and pkg.metadata.get("REQUIRED_USE") and \ eapi_has_required_use(pkg.metadata["EAPI"]): required_use_is_sat = check_required_use( pkg.metadata["REQUIRED_USE"], @@ -1055,7 +1119,8 @@ class depgraph(object): if atom is None: atom = Atom("=" + pkg.cpv) self._dynamic_config._unsatisfied_deps_for_display.append( - ((pkg.root, atom), {"myparent":dep.parent})) + ((pkg.root, atom), + {"myparent" : dep.parent, "show_req_use" : pkg})) self._dynamic_config._skip_restart = True return 0 @@ -1146,11 +1211,6 @@ class depgraph(object): all_match = False break - if to_be_selected >= to_be_masked: - # We only care about the parent atoms - # when they trigger a downgrade. - parent_atoms = set() - fallback_data.append((to_be_masked, parent_atoms)) if all_match: @@ -1244,7 +1304,7 @@ class depgraph(object): settings.unlock() settings.setinst(pkg.cpv, pkg.metadata) settings.lock() - except portage.exception.InvalidDependString as e: + except portage.exception.InvalidDependString: if not pkg.installed: # should have been masked before it was selected raise @@ -1265,12 +1325,11 @@ class depgraph(object): self._dynamic_config.digraph.add(pkg, parent, priority=priority) self._add_parent_atom(pkg, parent_atom) - """ This section determines whether we go deeper into dependencies or not. - We want to go deeper on a few occasions: - Installing package A, we need to make sure package A's deps are met. - emerge --deep <pkgspec>; we need to recursively check dependencies of pkgspec - If we are in --nodeps (no recursion) mode, we obviously only check 1 level of dependencies. - """ + # This section determines whether we go deeper into dependencies or not. + # We want to go deeper on a few occasions: + # Installing package A, we need to make sure package A's deps are met. + # emerge --deep <pkgspec>; we need to recursively check dependencies of pkgspec + # If we are in --nodeps (no recursion) mode, we obviously only check 1 level of dependencies. if arg_atoms: depth = 0 pkg.depth = depth @@ -1318,13 +1377,8 @@ class depgraph(object): def _add_pkg_deps(self, pkg, allow_unsatisfied=False): - mytype = pkg.type_name myroot = pkg.root - mykey = pkg.cpv metadata = pkg.metadata - myuse = self._pkg_use_enabled(pkg) - jbigkey = pkg - depth = pkg.depth + 1 removal_action = "remove" in self._dynamic_config.myparams edepend={} @@ -1361,7 +1415,7 @@ class depgraph(object): if removal_action: depend_root = myroot else: - depend_root = "/" + depend_root = self._frozen_config._running_root.root root_deps = self._frozen_config.myopts.get("--root-deps") if root_deps is not None: if root_deps is True: @@ -1388,7 +1442,6 @@ class depgraph(object): ) debug = "--debug" in self._frozen_config.myopts - strict = mytype != "installed" for dep_root, dep_string, dep_priority in deps: if not dep_string: @@ -1481,7 +1534,7 @@ class depgraph(object): selected_atoms = self._select_atoms(dep_root, dep_string, myuse=self._pkg_use_enabled(pkg), parent=pkg, strict=strict, priority=dep_priority) - except portage.exception.InvalidDependString as e: + except portage.exception.InvalidDependString: if pkg.installed: self._dynamic_config._masked_installed.add(pkg) return 1 @@ -1731,7 +1784,7 @@ class depgraph(object): pkg_atom_map.setdefault(pkg, set()).add(atom) cp_pkg_map.setdefault(pkg.cp, set()).add(pkg) - for cp, pkgs in cp_pkg_map.items(): + for pkgs in cp_pkg_map.values(): if len(pkgs) < 2: for pkg in pkgs: for atom in pkg_atom_map[pkg]: @@ -1807,7 +1860,7 @@ class depgraph(object): i += 1 else: try: - x = portage.dep.Atom(x) + x = portage.dep.Atom(x, eapi=pkg.metadata["EAPI"]) except portage.exception.InvalidAtom: if not pkg.installed: raise portage.exception.InvalidDependString( @@ -1855,7 +1908,7 @@ class depgraph(object): @param atom_without_category: an atom without a category component @type atom_without_category: String @rtype: list - @returns: a list of atoms containing categories (possibly empty) + @return: a list of atoms containing categories (possibly empty) """ null_cp = portage.dep_getkey(insert_category_into_atom( atom_without_category, "null")) @@ -1886,7 +1939,6 @@ class depgraph(object): def _iter_atoms_for_pkg(self, pkg): depgraph_sets = self._dynamic_config.sets[pkg.root] atom_arg_map = depgraph_sets.atom_arg_map - root_config = self._frozen_config.roots[pkg.root] for atom in depgraph_sets.atoms.iterAtomsForPackage(pkg): if atom.cp != pkg.cp and \ self._have_new_virt(pkg.root, atom.cp): @@ -1923,13 +1975,13 @@ class depgraph(object): sets = root_config.sets depgraph_sets = self._dynamic_config.sets[root_config.root] myfavorites=[] - myroot = self._frozen_config.target_root - dbs = self._dynamic_config._filtered_trees[myroot]["dbs"] - vardb = self._frozen_config.trees[myroot]["vartree"].dbapi - real_vardb = self._frozen_config._trees_orig[myroot]["vartree"].dbapi - portdb = self._frozen_config.trees[myroot]["porttree"].dbapi - bindb = self._frozen_config.trees[myroot]["bintree"].dbapi - pkgsettings = self._frozen_config.pkgsettings[myroot] + eroot = root_config.root + root = root_config.settings['ROOT'] + vardb = self._frozen_config.trees[eroot]["vartree"].dbapi + real_vardb = self._frozen_config._trees_orig[eroot]["vartree"].dbapi + portdb = self._frozen_config.trees[eroot]["porttree"].dbapi + bindb = self._frozen_config.trees[eroot]["bintree"].dbapi + pkgsettings = self._frozen_config.pkgsettings[eroot] args = [] onlydeps = "--onlydeps" in self._frozen_config.myopts lookup_owners = [] @@ -1950,7 +2002,7 @@ class depgraph(object): mytbz2=portage.xpak.tbz2(x) mykey=mytbz2.getelements("CATEGORY")[0]+"/"+os.path.splitext(os.path.basename(x))[0] if os.path.realpath(x) != \ - os.path.realpath(self._frozen_config.trees[myroot]["bintree"].getname(mykey)): + os.path.realpath(bindb.bintree.getname(mykey)): writemsg(colorize("BAD", "\n*** You need to adjust PKGDIR to emerge this package.\n\n"), noiselevel=-1) self._dynamic_config._skip_restart = True return 0, myfavorites @@ -1996,9 +2048,9 @@ class depgraph(object): args.append(PackageArg(arg=x, package=pkg, root_config=root_config)) elif x.startswith(os.path.sep): - if not x.startswith(myroot): + if not x.startswith(eroot): portage.writemsg(("\n\n!!! '%s' does not start with" + \ - " $ROOT.\n") % x, noiselevel=-1) + " $EROOT.\n") % x, noiselevel=-1) self._dynamic_config._skip_restart = True return 0, [] # Queue these up since it's most efficient to handle @@ -2007,9 +2059,9 @@ class depgraph(object): elif x.startswith("." + os.sep) or \ x.startswith(".." + os.sep): f = os.path.abspath(x) - if not f.startswith(myroot): + if not f.startswith(eroot): portage.writemsg(("\n\n!!! '%s' (resolved from '%s') does not start with" + \ - " $ROOT.\n") % (f, x), noiselevel=-1) + " $EROOT.\n") % (f, x), noiselevel=-1) self._dynamic_config._skip_restart = True return 0, [] lookup_owners.append(f) @@ -2126,7 +2178,7 @@ class depgraph(object): for x in lookup_owners: if not search_for_multiple and os.path.isdir(x): search_for_multiple = True - relative_paths.append(x[len(myroot)-1:]) + relative_paths.append(x[len(root)-1:]) owners = set() for pkg, relative_path in \ @@ -2526,24 +2578,36 @@ class depgraph(object): # account for masking and USE settings. _autounmask_backup = self._dynamic_config._autounmask self._dynamic_config._autounmask = False - mytrees["pkg_use_enabled"] = self._pkg_use_enabled + # backup state for restoration, in case of recursive + # calls to this method + backup_state = mytrees.copy() try: + # clear state from previous call, in case this + # call is recursive (we have a backup, that we + # will use to restore it later) + mytrees.pop("pkg_use_enabled", None) + mytrees.pop("parent", None) + mytrees.pop("atom_graph", None) + mytrees.pop("priority", None) + + mytrees["pkg_use_enabled"] = self._pkg_use_enabled if parent is not None: - trees[root]["parent"] = parent - trees[root]["atom_graph"] = atom_graph + mytrees["parent"] = parent + mytrees["atom_graph"] = atom_graph if priority is not None: - trees[root]["priority"] = priority + mytrees["priority"] = priority + mycheck = portage.dep_check(depstring, None, pkgsettings, myuse=myuse, myroot=root, trees=trees) finally: + # restore state self._dynamic_config._autounmask = _autounmask_backup - del mytrees["pkg_use_enabled"] - if parent is not None: - trees[root].pop("parent") - trees[root].pop("atom_graph") - if priority is not None: - trees[root].pop("priority") + mytrees.pop("pkg_use_enabled", None) + mytrees.pop("parent", None) + mytrees.pop("atom_graph", None) + mytrees.pop("priority", None) + mytrees.update(backup_state) if not mycheck[0]: raise portage.exception.InvalidDependString(mycheck[1]) if parent is None: @@ -2637,6 +2701,38 @@ class depgraph(object): continue yield atom + def _virt_deps_visible(self, pkg, ignore_use=False): + """ + Assumes pkg is a virtual package. Traverses virtual deps recursively + and returns True if all deps are visible, False otherwise. This is + useful for checking if it will be necessary to expand virtual slots, + for cases like bug #382557. + """ + try: + rdepend = self._select_atoms( + pkg.root, pkg.metadata.get("RDEPEND", ""), + myuse=self._pkg_use_enabled(pkg), + parent=pkg, priority=self._priority(runtime=True)) + except InvalidDependString as e: + if not pkg.installed: + raise + writemsg_level("!!! Invalid RDEPEND in " + \ + "'%svar/db/pkg/%s/RDEPEND': %s\n" % \ + (pkg.root, pkg.cpv, e), + noiselevel=-1, level=logging.ERROR) + return False + + for atoms in rdepend.values(): + for atom in atoms: + if ignore_use: + atom = atom.without_use + pkg, existing = self._select_package( + pkg.root, atom) + if pkg is None or not self._pkg_visibility_check(pkg): + return False + + return True + def _get_dep_chain(self, start_node, target_atom=None, unsatisfied_dependency=False): """ @@ -2652,6 +2748,7 @@ class depgraph(object): node = start_node child = None all_parents = self._dynamic_config._parent_atoms + graph = self._dynamic_config.digraph if target_atom is not None and isinstance(node, Package): affecting_use = set() @@ -2676,11 +2773,46 @@ class depgraph(object): dep_chain.append((pkg_name, node.type_name)) + + # To build a dep chain for the given package we take + # "random" parents form the digraph, except for the + # first package, because we want a parent that forced + # the corresponding change (i.e '>=foo-2', instead 'foo'). + + traversed_nodes.add(start_node) + + start_node_parent_atoms = {} + for ppkg, patom in all_parents.get(node, []): + # Get a list of suitable atoms. For use deps + # (aka unsatisfied_dependency is not None) we + # need that the start_node doesn't match the atom. + if not unsatisfied_dependency or \ + not InternalPackageSet(initial_atoms=(patom,)).findAtomForPackage(start_node): + start_node_parent_atoms.setdefault(patom, []).append(ppkg) + + if start_node_parent_atoms: + # If there are parents in all_parents then use one of them. + # If not, then this package got pulled in by an Arg and + # will be correctly handled by the code that handles later + # packages in the dep chain. + best_match = best_match_to_list(node.cpv, start_node_parent_atoms) + + child = node + for ppkg in start_node_parent_atoms[best_match]: + node = ppkg + if ppkg in self._dynamic_config._initial_arg_list: + # Stop if reached the top level of the dep chain. + break + while node is not None: traversed_nodes.add(node) - if isinstance(node, DependencyArg): - if self._dynamic_config.digraph.parent_nodes(node): + if node not in graph: + # The parent is not in the graph due to backtracking. + break + + elif isinstance(node, DependencyArg): + if graph.parent_nodes(node): node_type = "set" else: node_type = "argument" @@ -2689,17 +2821,29 @@ class depgraph(object): elif node is not start_node: for ppkg, patom in all_parents[child]: if ppkg == node: + if child is start_node and unsatisfied_dependency and \ + InternalPackageSet(initial_atoms=(patom,)).findAtomForPackage(child): + # This atom is satisfied by child, there must be another atom. + continue atom = patom.unevaluated_atom break dep_strings = set() - for priority in self._dynamic_config.digraph.nodes[node][0][child]: - if priority.buildtime: - dep_strings.add(node.metadata["DEPEND"]) - if priority.runtime: - dep_strings.add(node.metadata["RDEPEND"]) - if priority.runtime_post: - dep_strings.add(node.metadata["PDEPEND"]) + priorities = graph.nodes[node][0].get(child) + if priorities is None: + # This edge comes from _parent_atoms and was not added to + # the graph, and _parent_atoms does not contain priorities. + dep_strings.add(node.metadata["DEPEND"]) + dep_strings.add(node.metadata["RDEPEND"]) + dep_strings.add(node.metadata["PDEPEND"]) + else: + for priority in priorities: + if priority.buildtime: + dep_strings.add(node.metadata["DEPEND"]) + if priority.runtime: + dep_strings.add(node.metadata["RDEPEND"]) + if priority.runtime_post: + dep_strings.add(node.metadata["PDEPEND"]) affecting_use = set() for dep_str in dep_strings: @@ -2726,10 +2870,6 @@ class depgraph(object): dep_chain.append((pkg_name, node.type_name)) - if node not in self._dynamic_config.digraph: - # The parent is not in the graph due to backtracking. - break - # When traversing to parents, prefer arguments over packages # since arguments are root nodes. Never traverse the same # package twice, in order to prevent an infinite loop. @@ -2791,7 +2931,7 @@ class depgraph(object): def _show_unsatisfied_dep(self, root, atom, myparent=None, arg=None, - check_backtrack=False, check_autounmask_breakage=False): + check_backtrack=False, check_autounmask_breakage=False, show_req_use=None): """ When check_backtrack=True, no output is produced and the method either returns or raises _backtrack_mask if @@ -2810,14 +2950,13 @@ class depgraph(object): xinfo = _unicode_decode('"%s"') % (myparent,) # Discard null/ from failed cpv_expand category expansion. xinfo = xinfo.replace("null/", "") - if root != "/": + if root != self._frozen_config._running_root.root: xinfo = "%s for %s" % (xinfo, root) masked_packages = [] missing_use = [] missing_use_adjustable = set() required_use_unsatisfied = [] masked_pkg_instances = set() - missing_licenses = [] have_eapi_mask = False pkgsettings = self._frozen_config.pkgsettings[root] root_config = self._frozen_config.roots[root] @@ -2828,7 +2967,6 @@ class depgraph(object): for db, pkg_type, built, installed, db_keys in dbs: if installed: continue - match = db.match if hasattr(db, "xmatch"): cpv_list = db.xmatch("match-all-cpv-only", atom.without_use) else: @@ -2854,12 +2992,20 @@ class depgraph(object): repo = metadata.get('repository') pkg = self._pkg(cpv, pkg_type, root_config, installed=installed, myrepo=repo) - if not atom_set.findAtomForPackage(pkg, - modified_use=self._pkg_use_enabled(pkg)): - continue # pkg.metadata contains calculated USE for ebuilds, # required later for getMissingLicenses. metadata = pkg.metadata + if pkg.invalid: + # Avoid doing any operations with packages that + # have invalid metadata. It would be unsafe at + # least because it could trigger unhandled + # exceptions in places like check_required_use(). + masked_packages.append( + (root_config, pkgsettings, cpv, repo, metadata, mreasons)) + continue + if not atom_set.findAtomForPackage(pkg, + modified_use=self._pkg_use_enabled(pkg)): + continue if pkg in self._dynamic_config._runtime_pkg_mask: backtrack_reasons = \ self._dynamic_config._runtime_pkg_mask[pkg] @@ -2887,7 +3033,7 @@ class depgraph(object): raise if not mreasons and \ not pkg.built and \ - pkg.metadata["REQUIRED_USE"] and \ + pkg.metadata.get("REQUIRED_USE") and \ eapi_has_required_use(pkg.metadata["EAPI"]): if not check_required_use( pkg.metadata["REQUIRED_USE"], @@ -2942,7 +3088,7 @@ class depgraph(object): continue missing_use_adjustable.add(pkg) - required_use = pkg.metadata["REQUIRED_USE"] + required_use = pkg.metadata.get("REQUIRED_USE") required_use_warning = "" if required_use: old_use = self._pkg_use_enabled(pkg) @@ -2990,7 +3136,7 @@ class depgraph(object): if untouchable_flags.intersection(involved_flags): continue - required_use = myparent.metadata["REQUIRED_USE"] + required_use = myparent.metadata.get("REQUIRED_USE") required_use_warning = "" if required_use: old_use = self._pkg_use_enabled(myparent) @@ -3066,62 +3212,66 @@ class depgraph(object): mask_docs = False - if required_use_unsatisfied: + if show_req_use is None and required_use_unsatisfied: # We have an unmasked package that only requires USE adjustment # in order to satisfy REQUIRED_USE, and nothing more. We assume # that the user wants the latest version, so only the first # instance is displayed. - pkg = required_use_unsatisfied[0] + show_req_use = required_use_unsatisfied[0] + + if show_req_use is not None: + + pkg = show_req_use output_cpv = pkg.cpv + _repo_separator + pkg.repo - writemsg_stdout("\n!!! " + \ + writemsg("\n!!! " + \ colorize("BAD", "The ebuild selected to satisfy ") + \ colorize("INFORM", xinfo) + \ colorize("BAD", " has unmet requirements.") + "\n", noiselevel=-1) use_display = pkg_use_display(pkg, self._frozen_config.myopts) - writemsg_stdout("- %s %s\n" % (output_cpv, use_display), + writemsg("- %s %s\n" % (output_cpv, use_display), noiselevel=-1) - writemsg_stdout("\n The following REQUIRED_USE flag constraints " + \ + writemsg("\n The following REQUIRED_USE flag constraints " + \ "are unsatisfied:\n", noiselevel=-1) reduced_noise = check_required_use( pkg.metadata["REQUIRED_USE"], self._pkg_use_enabled(pkg), pkg.iuse.is_valid_flag).tounicode() - writemsg_stdout(" %s\n" % \ + writemsg(" %s\n" % \ human_readable_required_use(reduced_noise), noiselevel=-1) normalized_required_use = \ " ".join(pkg.metadata["REQUIRED_USE"].split()) if reduced_noise != normalized_required_use: - writemsg_stdout("\n The above constraints " + \ + writemsg("\n The above constraints " + \ "are a subset of the following complete expression:\n", noiselevel=-1) - writemsg_stdout(" %s\n" % \ + writemsg(" %s\n" % \ human_readable_required_use(normalized_required_use), noiselevel=-1) - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n", noiselevel=-1) elif show_missing_use: - writemsg_stdout("\nemerge: there are no ebuilds built with USE flags to satisfy "+green(xinfo)+".\n", noiselevel=-1) - writemsg_stdout("!!! One of the following packages is required to complete your request:\n", noiselevel=-1) + writemsg("\nemerge: there are no ebuilds built with USE flags to satisfy "+green(xinfo)+".\n", noiselevel=-1) + writemsg("!!! One of the following packages is required to complete your request:\n", noiselevel=-1) for pkg, mreasons in show_missing_use: - writemsg_stdout("- "+pkg.cpv+_repo_separator+pkg.repo+" ("+", ".join(mreasons)+")\n", noiselevel=-1) + writemsg("- "+pkg.cpv+_repo_separator+pkg.repo+" ("+", ".join(mreasons)+")\n", noiselevel=-1) elif masked_packages: - writemsg_stdout("\n!!! " + \ + writemsg("\n!!! " + \ colorize("BAD", "All ebuilds that could satisfy ") + \ colorize("INFORM", xinfo) + \ colorize("BAD", " have been masked.") + "\n", noiselevel=-1) - writemsg_stdout("!!! One of the following masked packages is required to complete your request:\n", noiselevel=-1) + writemsg("!!! One of the following masked packages is required to complete your request:\n", noiselevel=-1) have_eapi_mask = show_masked_packages(masked_packages) if have_eapi_mask: - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n", noiselevel=-1) msg = ("The current version of portage supports " + \ "EAPI '%s'. You must upgrade to a newer version" + \ " of portage before EAPI masked packages can" + \ " be installed.") % portage.const.EAPI - writemsg_stdout("\n".join(textwrap.wrap(msg, 75)), noiselevel=-1) - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n".join(textwrap.wrap(msg, 75)), noiselevel=-1) + writemsg("\n", noiselevel=-1) mask_docs = True else: cp_exists = False @@ -3131,7 +3281,7 @@ class depgraph(object): cp_exists = True break - writemsg_stdout("\nemerge: there are no ebuilds to satisfy "+green(xinfo)+".\n", noiselevel=-1) + writemsg("\nemerge: there are no ebuilds to satisfy "+green(xinfo)+".\n", noiselevel=-1) if isinstance(myparent, AtomArg) and \ not cp_exists and \ self._frozen_config.myopts.get( @@ -3141,12 +3291,13 @@ class depgraph(object): if cat == "null": cat = None - writemsg_stdout("\nemerge: searching for similar names..." + writemsg("\nemerge: searching for similar names..." , noiselevel=-1) all_cp = set() all_cp.update(vardb.cp_all()) - all_cp.update(portdb.cp_all()) + if "--usepkgonly" not in self._frozen_config.myopts: + all_cp.update(portdb.cp_all()) if "--usepkg" in self._frozen_config.myopts: all_cp.update(bindb.cp_all()) # discard dir containing no ebuilds @@ -3164,9 +3315,18 @@ class depgraph(object): for other_cp in list(all_cp): other_pkg = portage.catsplit(other_cp)[1] if other_pkg == pkg: - # discard dir containing no ebuilds - all_cp.discard(other_cp) - continue + # Check for non-identical package that + # differs only by upper/lower case. + identical = True + for cp_orig in orig_cp_map[other_cp]: + if portage.catsplit(cp_orig)[1] != \ + portage.catsplit(atom.cp)[1]: + identical = False + break + if identical: + # discard dir containing no ebuilds + all_cp.discard(other_cp) + continue pkg_to_cp.setdefault(other_pkg, set()).add(other_cp) pkg_matches = difflib.get_close_matches(pkg, pkg_to_cp) matches = [] @@ -3179,16 +3339,16 @@ class depgraph(object): matches = matches_orig_case if len(matches) == 1: - writemsg_stdout("\nemerge: Maybe you meant " + matches[0] + "?\n" + writemsg("\nemerge: Maybe you meant " + matches[0] + "?\n" , noiselevel=-1) elif len(matches) > 1: - writemsg_stdout( + writemsg( "\nemerge: Maybe you meant any of these: %s?\n" % \ (", ".join(matches),), noiselevel=-1) else: # Generally, this would only happen if # all dbapis are empty. - writemsg_stdout(" nothing similar found.\n" + writemsg(" nothing similar found.\n" , noiselevel=-1) msg = [] if not isinstance(myparent, AtomArg): @@ -3201,12 +3361,12 @@ class depgraph(object): (node)), node_type)) if msg: - writemsg_stdout("\n".join(msg), noiselevel=-1) - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n".join(msg), noiselevel=-1) + writemsg("\n", noiselevel=-1) if mask_docs: show_mask_docs() - writemsg_stdout("\n", noiselevel=-1) + writemsg("\n", noiselevel=-1) def _iter_match_pkgs_any(self, root_config, atom, onlydeps=False): for db, pkg_type, built, installed, db_keys in \ @@ -3224,51 +3384,12 @@ class depgraph(object): """ db = root_config.trees[self.pkg_tree_map[pkg_type]].dbapi - - if hasattr(db, "xmatch"): - # For portdbapi we match only against the cpv, in order - # to bypass unnecessary cache access for things like IUSE - # and SLOT. Later, we cache the metadata in a Package - # instance, and use that for further matching. This - # optimization is especially relevant since - # pordbapi.aux_get() does not cache calls that have - # myrepo or mytree arguments. - cpv_list = db.xmatch("match-all-cpv-only", atom) - else: - cpv_list = db.match(atom) - - # USE=multislot can make an installed package appear as if - # it doesn't satisfy a slot dependency. Rebuilding the ebuild - # won't do any good as long as USE=multislot is enabled since - # the newly built package still won't have the expected slot. - # Therefore, assume that such SLOT dependencies are already - # satisfied rather than forcing a rebuild. + atom_exp = dep_expand(atom, mydb=db, settings=root_config.settings) + cp_list = db.cp_list(atom_exp.cp) + matched_something = False installed = pkg_type == 'installed' - if installed and not cpv_list and atom.slot: - for cpv in db.match(atom.cp): - slot_available = False - for other_db, other_type, other_built, \ - other_installed, other_keys in \ - self._dynamic_config._filtered_trees[root_config.root]["dbs"]: - try: - if atom.slot == \ - other_db.aux_get(cpv, ["SLOT"])[0]: - slot_available = True - break - except KeyError: - pass - if not slot_available: - continue - inst_pkg = self._pkg(cpv, "installed", - root_config, installed=installed, myrepo = atom.repo) - # Remove the slot from the atom and verify that - # the package matches the resulting atom. - if portage.match_from_list( - atom.without_slot, [inst_pkg]): - yield inst_pkg - return - - if cpv_list: + + if cp_list: atom_set = InternalPackageSet(initial_atoms=(atom,), allow_repo=True) if atom.repo is None and hasattr(db, "getRepositories"): @@ -3277,8 +3398,13 @@ class depgraph(object): repo_list = [atom.repo] # descending order - cpv_list.reverse() - for cpv in cpv_list: + cp_list.reverse() + for cpv in cp_list: + # Call match_from_list on one cpv at a time, in order + # to avoid unnecessary match_from_list comparisons on + # versions that are never yielded from this method. + if not match_from_list(atom_exp, [cpv]): + continue for repo in repo_list: try: @@ -3295,16 +3421,63 @@ class depgraph(object): # Make sure that cpv from the current repo satisfies the atom. # This might not be the case if there are several repos with # the same cpv, but different metadata keys, like SLOT. - # Also, for portdbapi, parts of the match that require - # metadata access are deferred until we have cached the - # metadata in a Package instance. + # Also, parts of the match that require metadata access + # are deferred until we have cached the metadata in a + # Package instance. if not atom_set.findAtomForPackage(pkg, modified_use=self._pkg_use_enabled(pkg)): continue + matched_something = True yield pkg + # USE=multislot can make an installed package appear as if + # it doesn't satisfy a slot dependency. Rebuilding the ebuild + # won't do any good as long as USE=multislot is enabled since + # the newly built package still won't have the expected slot. + # Therefore, assume that such SLOT dependencies are already + # satisfied rather than forcing a rebuild. + if not matched_something and installed and atom.slot is not None: + + if "remove" in self._dynamic_config.myparams: + # We need to search the portdbapi, which is not in our + # normal dbs list, in order to find the real SLOT. + portdb = self._frozen_config.trees[root_config.root]["porttree"].dbapi + db_keys = list(portdb._aux_cache_keys) + dbs = [(portdb, "ebuild", False, False, db_keys)] + else: + dbs = self._dynamic_config._filtered_trees[root_config.root]["dbs"] + + cp_list = db.cp_list(atom_exp.cp) + if cp_list: + atom_set = InternalPackageSet( + initial_atoms=(atom.without_slot,), allow_repo=True) + atom_exp_without_slot = atom_exp.without_slot + cp_list.reverse() + for cpv in cp_list: + if not match_from_list(atom_exp_without_slot, [cpv]): + continue + slot_available = False + for other_db, other_type, other_built, \ + other_installed, other_keys in dbs: + try: + if atom.slot == \ + other_db.aux_get(cpv, ["SLOT"])[0]: + slot_available = True + break + except KeyError: + pass + if not slot_available: + continue + inst_pkg = self._pkg(cpv, "installed", + root_config, installed=installed, myrepo=atom.repo) + # Remove the slot from the atom and verify that + # the package matches the resulting atom. + if atom_set.findAtomForPackage(inst_pkg): + yield inst_pkg + return + def _select_pkg_highest_available(self, root, atom, onlydeps=False): - cache_key = (root, atom, onlydeps) + cache_key = (root, atom, atom.unevaluated_atom, onlydeps) ret = self._dynamic_config._highest_pkg_cache.get(cache_key) if ret is not None: pkg, existing = ret @@ -3320,7 +3493,6 @@ class depgraph(object): self._dynamic_config._highest_pkg_cache[cache_key] = ret pkg, existing = ret if pkg is not None: - settings = pkg.root_config.settings if self._pkg_visibility_check(pkg) and \ not (pkg.installed and pkg.masks): self._dynamic_config._visible_pkgs[pkg.root].cpv_inject(pkg) @@ -3347,40 +3519,81 @@ class depgraph(object): return False return True + class _AutounmaskLevel(object): + __slots__ = ("allow_use_changes", "allow_unstable_keywords", "allow_license_changes", \ + "allow_missing_keywords", "allow_unmasks") + + def __init__(self): + self.allow_use_changes = False + self.allow_license_changes = False + self.allow_unstable_keywords = False + self.allow_missing_keywords = False + self.allow_unmasks = False + + def _autounmask_levels(self): + """ + Iterate over the different allowed things to unmask. + + 1. USE + 2. USE + ~arch + license + 3. USE + ~arch + license + missing keywords + 4. USE + ~arch + license + masks + 5. USE + ~arch + license + missing keywords + masks + + Some thoughts: + * Do least invasive changes first. + * Try unmasking alone before unmasking + missing keywords + to avoid -9999 versions if possible + """ + + if self._dynamic_config._autounmask is not True: + return + + autounmask_keep_masks = self._frozen_config.myopts.get("--autounmask-keep-masks", "n") != "n" + autounmask_level = self._AutounmaskLevel() + + autounmask_level.allow_use_changes = True + + for only_use_changes in (True, False): + + autounmask_level.allow_unstable_keywords = (not only_use_changes) + autounmask_level.allow_license_changes = (not only_use_changes) + + for missing_keyword, unmask in ((False,False), (True, False), (False, True), (True, True)): + + if (only_use_changes or autounmask_keep_masks) and (missing_keyword or unmask): + break + + autounmask_level.allow_missing_keywords = missing_keyword + autounmask_level.allow_unmasks = unmask + + yield autounmask_level + + def _select_pkg_highest_available_imp(self, root, atom, onlydeps=False): pkg, existing = self._wrapped_select_pkg_highest_available_imp(root, atom, onlydeps=onlydeps) default_selection = (pkg, existing) - if self._dynamic_config._autounmask is True: + def reset_pkg(pkg): if pkg is not None and \ pkg.installed and \ not self._want_installed_pkg(pkg): pkg = None - for only_use_changes in True, False: + if self._dynamic_config._autounmask is True: + reset_pkg(pkg) + + for autounmask_level in self._autounmask_levels(): if pkg is not None: break - for allow_unmasks in (False, True): - if only_use_changes and allow_unmasks: - continue + pkg, existing = \ + self._wrapped_select_pkg_highest_available_imp( + root, atom, onlydeps=onlydeps, + autounmask_level=autounmask_level) - if pkg is not None: - break - - pkg, existing = \ - self._wrapped_select_pkg_highest_available_imp( - root, atom, onlydeps=onlydeps, - allow_use_changes=True, - allow_unstable_keywords=(not only_use_changes), - allow_license_changes=(not only_use_changes), - allow_unmasks=allow_unmasks) - - if pkg is not None and \ - pkg.installed and \ - not self._want_installed_pkg(pkg): - pkg = None + reset_pkg(pkg) if self._dynamic_config._need_restart: return None, None @@ -3392,21 +3605,20 @@ class depgraph(object): return pkg, existing - def _pkg_visibility_check(self, pkg, allow_unstable_keywords=False, allow_license_changes=False, allow_unmasks=False): + def _pkg_visibility_check(self, pkg, autounmask_level=None, trust_graph=True): if pkg.visible: return True - if pkg in self._dynamic_config.digraph: + if trust_graph and pkg in self._dynamic_config.digraph: # Sometimes we need to temporarily disable # dynamic_config._autounmask, but for overall - # consistency in dependency resolution, in any - # case we want to respect autounmask visibity - # for packages that have already been added to - # the dependency graph. + # consistency in dependency resolution, in most + # cases we want to treat packages in the graph + # as though they are visible. return True - if not self._dynamic_config._autounmask: + if not self._dynamic_config._autounmask or autounmask_level is None: return False pkgsettings = self._frozen_config.pkgsettings[pkg.root] @@ -3455,11 +3667,10 @@ class depgraph(object): #Package has already been unmasked. return True - #We treat missing keywords in the same way as masks. - if (masked_by_unstable_keywords and not allow_unstable_keywords) or \ - (masked_by_missing_keywords and not allow_unmasks) or \ - (masked_by_p_mask and not allow_unmasks) or \ - (missing_licenses and not allow_license_changes): + if (masked_by_unstable_keywords and not autounmask_level.allow_unstable_keywords) or \ + (masked_by_missing_keywords and not autounmask_level.allow_missing_keywords) or \ + (masked_by_p_mask and not autounmask_level.allow_unmasks) or \ + (missing_licenses and not autounmask_level.allow_license_changes): #We are not allowed to do the needed changes. return False @@ -3556,7 +3767,7 @@ class depgraph(object): if new_changes != old_changes: #Don't do the change if it violates REQUIRED_USE. - required_use = pkg.metadata["REQUIRED_USE"] + required_use = pkg.metadata.get("REQUIRED_USE") if required_use and check_required_use(required_use, old_use, pkg.iuse.is_valid_flag) and \ not check_required_use(required_use, new_use, pkg.iuse.is_valid_flag): return old_use @@ -3574,13 +3785,11 @@ class depgraph(object): self._dynamic_config._need_restart = True return new_use - def _wrapped_select_pkg_highest_available_imp(self, root, atom, onlydeps=False, \ - allow_use_changes=False, allow_unstable_keywords=False, allow_license_changes=False, allow_unmasks=False): + def _wrapped_select_pkg_highest_available_imp(self, root, atom, onlydeps=False, autounmask_level=None): root_config = self._frozen_config.roots[root] pkgsettings = self._frozen_config.pkgsettings[root] dbs = self._dynamic_config._filtered_trees[root]["dbs"] vardb = self._frozen_config.roots[root].trees["vartree"].dbapi - portdb = self._frozen_config.roots[root].trees["porttree"].dbapi # List of acceptable packages, ordered by type preference. matched_packages = [] matched_pkgs_ignore_use = [] @@ -3588,6 +3797,8 @@ class depgraph(object): if not isinstance(atom, portage.dep.Atom): atom = portage.dep.Atom(atom) atom_cp = atom.cp + have_new_virt = atom_cp.startswith("virtual/") and \ + self._have_new_virt(root, atom_cp) atom_set = InternalPackageSet(initial_atoms=(atom,), allow_repo=True) existing_node = None myeb = None @@ -3635,6 +3846,9 @@ class depgraph(object): # USE configuration. for pkg in self._iter_match_pkgs(root_config, pkg_type, atom.without_use, onlydeps=onlydeps): + if pkg.cp != atom_cp and have_new_virt: + # pull in a new-style virtual instead + continue if pkg in self._dynamic_config._runtime_pkg_mask: # The package has been masked by the backtracking logic continue @@ -3698,10 +3912,7 @@ class depgraph(object): # _dep_check_composite_db, in order to prevent # incorrect choices in || deps like bug #351828. - if not self._pkg_visibility_check(pkg, \ - allow_unstable_keywords=allow_unstable_keywords, - allow_license_changes=allow_license_changes, - allow_unmasks=allow_unmasks): + if not self._pkg_visibility_check(pkg, autounmask_level): continue # Enable upgrade or downgrade to a version @@ -3741,19 +3952,13 @@ class depgraph(object): pkg_eb_visible = False for pkg_eb in self._iter_match_pkgs(pkg.root_config, "ebuild", Atom("=%s" % (pkg.cpv,))): - if self._pkg_visibility_check(pkg_eb, \ - allow_unstable_keywords=allow_unstable_keywords, - allow_license_changes=allow_license_changes, - allow_unmasks=allow_unmasks): + if self._pkg_visibility_check(pkg_eb, autounmask_level): pkg_eb_visible = True break if not pkg_eb_visible: continue else: - if not self._pkg_visibility_check(pkg_eb, \ - allow_unstable_keywords=allow_unstable_keywords, - allow_license_changes=allow_license_changes, - allow_unmasks=allow_unmasks): + if not self._pkg_visibility_check(pkg_eb, autounmask_level): continue # Calculation of USE for unbuilt ebuilds is relatively @@ -3783,7 +3988,7 @@ class depgraph(object): if atom.use: matched_pkgs_ignore_use.append(pkg) - if allow_use_changes and not pkg.built: + if autounmask_level and autounmask_level.allow_use_changes and not pkg.built: target_use = {} for flag in atom.use.enabled: target_use[flag] = True @@ -3852,6 +4057,7 @@ class depgraph(object): e_pkg = self._dynamic_config._slot_pkg_map[root].get(pkg.slot_atom) if not e_pkg: break + # Use PackageSet.findAtomForPackage() # for PROVIDE support. if atom_set.findAtomForPackage(e_pkg, modified_use=self._pkg_use_enabled(e_pkg)): @@ -3872,7 +4078,8 @@ class depgraph(object): if built and not useoldpkg and (not installed or matched_pkgs_ignore_use) and \ ("--newuse" in self._frozen_config.myopts or \ "--reinstall" in self._frozen_config.myopts or \ - "--binpkg-respect-use" in self._frozen_config.myopts): + (not installed and self._dynamic_config.myparams.get( + "binpkg_respect_use") in ("y", "auto"))): iuses = pkg.iuse.all old_use = self._pkg_use_enabled(pkg) if myeb: @@ -3886,9 +4093,11 @@ class depgraph(object): cur_iuse = iuses if myeb and not usepkgonly and not useoldpkg: cur_iuse = myeb.iuse.all - if self._reinstall_for_flags(forced_flags, - old_use, iuses, - now_use, cur_iuse): + reinstall_for_flags = self._reinstall_for_flags(pkg, + forced_flags, old_use, iuses, now_use, cur_iuse) + if reinstall_for_flags: + if not pkg.installed: + self._dynamic_config.ignored_binaries.setdefault(pkg, set()).update(reinstall_for_flags) break # Compare current config to installed package # and do not reinstall if possible. @@ -3905,7 +4114,7 @@ class depgraph(object): cur_use = self._pkg_use_enabled(pkg) cur_iuse = pkg.iuse.all reinstall_for_flags = \ - self._reinstall_for_flags( + self._reinstall_for_flags(pkg, forced_flags, old_use, old_iuse, cur_use, cur_iuse) if reinstall_for_flags: @@ -4002,21 +4211,16 @@ class depgraph(object): if avoid_update: for pkg in matched_packages: - if pkg.installed and self._pkg_visibility_check(pkg, \ - allow_unstable_keywords=allow_unstable_keywords, - allow_license_changes=allow_license_changes, - allow_unmasks=allow_unmasks): + if pkg.installed and self._pkg_visibility_check(pkg, autounmask_level): return pkg, existing_node visible_matches = [] if matched_oldpkg: visible_matches = [pkg.cpv for pkg in matched_oldpkg \ - if self._pkg_visibility_check(pkg, allow_unstable_keywords=allow_unstable_keywords, - allow_license_changes=allow_license_changes, allow_unmasks=allow_unmasks)] + if self._pkg_visibility_check(pkg, autounmask_level)] if not visible_matches: visible_matches = [pkg.cpv for pkg in matched_packages \ - if self._pkg_visibility_check(pkg, allow_unstable_keywords=allow_unstable_keywords, - allow_license_changes=allow_license_changes, allow_unmasks=allow_unmasks)] + if self._pkg_visibility_check(pkg, autounmask_level)] if visible_matches: bestmatch = portage.best(visible_matches) else: @@ -4046,11 +4250,12 @@ class depgraph(object): """ Select packages that are installed. """ - vardb = self._dynamic_config._graph_trees[root]["vartree"].dbapi - matches = vardb.match_pkgs(atom) + matches = list(self._iter_match_pkgs(self._frozen_config.roots[root], + "installed", atom)) if not matches: return None, None if len(matches) > 1: + matches.reverse() # ascending order unmasked = [pkg for pkg in matches if \ self._pkg_visibility_check(pkg)] if unmasked: @@ -4088,11 +4293,10 @@ class depgraph(object): "recurse" not in self._dynamic_config.myparams: return 1 - if "complete" not in self._dynamic_config.myparams: - # Automatically enable complete mode if there are any - # downgrades, since they often break dependencies - # (like in bug #353613). - have_downgrade = False + if "complete" not in self._dynamic_config.myparams and \ + self._dynamic_config.myparams.get("complete_if_new_ver", "y") == "y": + # Enable complete mode if an installed package version will change. + version_change = False for node in self._dynamic_config.digraph: if not isinstance(node, Package) or \ node.operation != "merge": @@ -4100,16 +4304,15 @@ class depgraph(object): vardb = self._frozen_config.roots[ node.root].trees["vartree"].dbapi inst_pkg = vardb.match_pkgs(node.slot_atom) - if inst_pkg and inst_pkg[0] > node: - have_downgrade = True + if inst_pkg and (inst_pkg[0] > node or inst_pkg[0] < node): + version_change = True break - if have_downgrade: + if version_change: self._dynamic_config.myparams["complete"] = True - else: - # Skip complete graph mode, in order to avoid consuming - # enough time to disturb users. - return 1 + + if "complete" not in self._dynamic_config.myparams: + return 1 self._load_vdb() @@ -4137,7 +4340,8 @@ class depgraph(object): args = self._dynamic_config._initial_arg_list[:] for root in self._frozen_config.roots: if root != self._frozen_config.target_root and \ - "remove" in self._dynamic_config.myparams: + ("remove" in self._dynamic_config.myparams or + self._frozen_config.myopts.get("--root-deps") is not None): # Only pull in deps for the relevant root. continue depgraph_sets = self._dynamic_config.sets[root] @@ -4265,9 +4469,6 @@ class depgraph(object): "--nodeps" in self._frozen_config.myopts: return True - complete = "complete" in self._dynamic_config.myparams - deep = "deep" in self._dynamic_config.myparams - if True: # Pull in blockers from all installed packages that haven't already # been pulled into the depgraph, in order to ensure that they are @@ -4281,11 +4482,14 @@ class depgraph(object): # are already built. dep_keys = ["RDEPEND", "PDEPEND"] for myroot in self._frozen_config.trees: + + if self._frozen_config.myopts.get("--root-deps") is not None and \ + myroot != self._frozen_config.target_root: + continue + vardb = self._frozen_config.trees[myroot]["vartree"].dbapi - portdb = self._frozen_config.trees[myroot]["porttree"].dbapi pkgsettings = self._frozen_config.pkgsettings[myroot] root_config = self._frozen_config.roots[myroot] - dbs = self._dynamic_config._filtered_trees[myroot]["dbs"] final_db = self._dynamic_config.mydbapi[myroot] blocker_cache = BlockerCache(myroot, vardb) @@ -4304,7 +4508,8 @@ class depgraph(object): # packages masked by license, since the user likely wants # to adjust ACCEPT_LICENSE. if pkg in final_db: - if not self._pkg_visibility_check(pkg) and \ + if not self._pkg_visibility_check(pkg, + trust_graph=False) and \ (pkg_in_graph or 'LICENSE' in pkg.masks): self._dynamic_config._masked_installed.add(pkg) else: @@ -4381,7 +4586,7 @@ class depgraph(object): # matches (this can happen if an atom lacks a # category). show_invalid_depstring_notice( - pkg, depstr, str(e)) + pkg, depstr, _unicode_decode("%s") % (e,)) del e raise if not success: @@ -4412,7 +4617,8 @@ class depgraph(object): except portage.exception.InvalidAtom as e: depstr = " ".join(vardb.aux_get(pkg.cpv, dep_keys)) show_invalid_depstring_notice( - pkg, depstr, "Invalid Atom: %s" % (e,)) + pkg, depstr, + _unicode_decode("Invalid Atom: %s") % (e,)) return False for cpv in stale_cache: del blocker_cache[cpv] @@ -4852,15 +5058,6 @@ class depgraph(object): if replacement_portage == running_portage: replacement_portage = None - if replacement_portage is not None and \ - (running_portage is None or \ - running_portage.cpv != replacement_portage.cpv or \ - '9999' in replacement_portage.cpv or \ - 'git' in replacement_portage.inherited or \ - 'git-2' in replacement_portage.inherited): - # update from running_portage to replacement_portage asap - asap_nodes.append(replacement_portage) - if running_portage is not None: try: portage_rdepend = self._select_atoms_highest_available( @@ -5668,6 +5865,8 @@ class depgraph(object): """ autounmask_write = self._frozen_config.myopts.get("--autounmask-write", "n") == True + autounmask_unrestricted_atoms = \ + self._frozen_config.myopts.get("--autounmask-unrestricted-atoms", "n") == True quiet = "--quiet" in self._frozen_config.myopts pretend = "--pretend" in self._frozen_config.myopts ask = "--ask" in self._frozen_config.myopts @@ -5703,6 +5902,7 @@ class depgraph(object): #Set of roots we have autounmask changes for. roots = set() + masked_by_missing_keywords = False unstable_keyword_msg = {} for pkg in self._dynamic_config._needed_unstable_keywords: self._show_merge_list() @@ -5718,12 +5918,17 @@ class depgraph(object): if reason.unmask_hint and \ reason.unmask_hint.key == 'unstable keyword': keyword = reason.unmask_hint.value + if keyword == "**": + masked_by_missing_keywords = True unstable_keyword_msg[root].append(self._get_dep_chain_as_comment(pkg)) - if is_latest: - unstable_keyword_msg[root].append(">=%s %s\n" % (pkg.cpv, keyword)) - elif is_latest_in_slot: - unstable_keyword_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.metadata["SLOT"], keyword)) + if autounmask_unrestricted_atoms: + if is_latest: + unstable_keyword_msg[root].append(">=%s %s\n" % (pkg.cpv, keyword)) + elif is_latest_in_slot: + unstable_keyword_msg[root].append(">=%s:%s %s\n" % (pkg.cpv, pkg.metadata["SLOT"], keyword)) + else: + unstable_keyword_msg[root].append("=%s %s\n" % (pkg.cpv, keyword)) else: unstable_keyword_msg[root].append("=%s %s\n" % (pkg.cpv, keyword)) @@ -5757,10 +5962,13 @@ class depgraph(object): comment.splitlines() if line] for line in comment: p_mask_change_msg[root].append("%s\n" % line) - if is_latest: - p_mask_change_msg[root].append(">=%s\n" % pkg.cpv) - elif is_latest_in_slot: - p_mask_change_msg[root].append(">=%s:%s\n" % (pkg.cpv, pkg.metadata["SLOT"])) + if autounmask_unrestricted_atoms: + if is_latest: + p_mask_change_msg[root].append(">=%s\n" % pkg.cpv) + elif is_latest_in_slot: + p_mask_change_msg[root].append(">=%s:%s\n" % (pkg.cpv, pkg.metadata["SLOT"])) + else: + p_mask_change_msg[root].append("=%s\n" % pkg.cpv) else: p_mask_change_msg[root].append("=%s\n" % pkg.cpv) @@ -5893,33 +6101,41 @@ class depgraph(object): write_to_file = not problems + def format_msg(lines): + lines = lines[:] + for i, line in enumerate(lines): + if line.startswith("#"): + continue + lines[i] = colorize("INFORM", line.rstrip()) + "\n" + return "".join(lines) + for root in roots: settings = self._frozen_config.roots[root].settings abs_user_config = os.path.join( settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH) if len(roots) > 1: - writemsg_stdout("\nFor %s:\n" % abs_user_config, noiselevel=-1) + writemsg("\nFor %s:\n" % abs_user_config, noiselevel=-1) if root in unstable_keyword_msg: - writemsg_stdout("\nThe following " + colorize("BAD", "keyword changes") + \ + writemsg("\nThe following " + colorize("BAD", "keyword changes") + \ " are necessary to proceed:\n", noiselevel=-1) - writemsg_stdout("".join(unstable_keyword_msg[root]), noiselevel=-1) + writemsg(format_msg(unstable_keyword_msg[root]), noiselevel=-1) if root in p_mask_change_msg: - writemsg_stdout("\nThe following " + colorize("BAD", "mask changes") + \ + writemsg("\nThe following " + colorize("BAD", "mask changes") + \ " are necessary to proceed:\n", noiselevel=-1) - writemsg_stdout("".join(p_mask_change_msg[root]), noiselevel=-1) + writemsg(format_msg(p_mask_change_msg[root]), noiselevel=-1) if root in use_changes_msg: - writemsg_stdout("\nThe following " + colorize("BAD", "USE changes") + \ + writemsg("\nThe following " + colorize("BAD", "USE changes") + \ " are necessary to proceed:\n", noiselevel=-1) - writemsg_stdout("".join(use_changes_msg[root]), noiselevel=-1) + writemsg(format_msg(use_changes_msg[root]), noiselevel=-1) if root in license_msg: - writemsg_stdout("\nThe following " + colorize("BAD", "license changes") + \ + writemsg("\nThe following " + colorize("BAD", "license changes") + \ " are necessary to proceed:\n", noiselevel=-1) - writemsg_stdout("".join(license_msg[root]), noiselevel=-1) + writemsg(format_msg(license_msg[root]), noiselevel=-1) protect_obj = {} if write_to_file: @@ -5948,7 +6164,7 @@ class depgraph(object): if protect_obj[root].isprotected(file_to_write_to): # We want to force new_protect_filename to ensure # that the user will see all our changes via - # etc-update, even if file_to_write_to doesn't + # dispatch-conf, even if file_to_write_to doesn't # exist yet, so we specify force=True. file_to_write_to = new_protect_filename(file_to_write_to, force=True) @@ -5957,20 +6173,16 @@ class depgraph(object): except PortageException: problems.append("!!! Failed to write '%s'\n" % file_to_write_to) - if not quiet and \ - (unstable_keyword_msg or \ - p_mask_change_msg or \ - use_changes_msg or \ - license_msg): + if not quiet and (p_mask_change_msg or masked_by_missing_keywords): msg = [ "", - "NOTE: This --autounmask behavior can be disabled by setting", - " EMERGE_DEFAULT_OPTS=\"--autounmask=n\" in make.conf." + "NOTE: The --autounmask-keep-masks option will prevent emerge", + " from creating package.unmask or ** keyword changes." ] for line in msg: if line: line = colorize("INFORM", line) - writemsg_stdout(line + "\n", noiselevel=-1) + writemsg(line + "\n", noiselevel=-1) if ask and write_to_file and file_to_write_to: prompt = "\nWould you like to add these " + \ @@ -6002,14 +6214,14 @@ class depgraph(object): file_to_write_to.get((abs_user_config, "package.license"))) if problems: - writemsg_stdout("\nThe following problems occurred while writing autounmask changes:\n", \ + writemsg("\nThe following problems occurred while writing autounmask changes:\n", \ noiselevel=-1) - writemsg_stdout("".join(problems), noiselevel=-1) + writemsg("".join(problems), noiselevel=-1) elif write_to_file and roots: - writemsg_stdout("\nAutounmask changes successfully written. Remember to run etc-update.\n", \ + writemsg("\nAutounmask changes successfully written. Remember to run dispatch-conf.\n", \ noiselevel=-1) elif not pretend and not autounmask_write and roots: - writemsg_stdout("\nUse --autounmask-write to write changes to config files (honoring CONFIG_PROTECT).\n", \ + writemsg("\nUse --autounmask-write to write changes to config files (honoring CONFIG_PROTECT).\n", \ noiselevel=-1) @@ -6020,49 +6232,25 @@ class depgraph(object): the merge list where it is most likely to be seen, but if display() is not going to be called then this method should be called explicitly to ensure that the user is notified of problems with the graph. - - All output goes to stderr, except for unsatisfied dependencies which - go to stdout for parsing by programs such as autounmask. """ - # Note that show_masked_packages() sends its output to - # stdout, and some programs such as autounmask parse the - # output in cases when emerge bails out. However, when - # show_masked_packages() is called for installed packages - # here, the message is a warning that is more appropriate - # to send to stderr, so temporarily redirect stdout to - # stderr. TODO: Fix output code so there's a cleaner way - # to redirect everything to stderr. - sys.stdout.flush() - sys.stderr.flush() - stdout = sys.stdout - try: - sys.stdout = sys.stderr - self._display_problems() - finally: - sys.stdout = stdout - sys.stdout.flush() - sys.stderr.flush() - - # This goes to stdout for parsing by programs like autounmask. - for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: - self._show_unsatisfied_dep(*pargs, **kwargs) - - def _display_problems(self): if self._dynamic_config._circular_deps_for_display is not None: self._show_circular_deps( self._dynamic_config._circular_deps_for_display) - # The user is only notified of a slot conflict if - # there are no unresolvable blocker conflicts. - if self._dynamic_config._unsatisfied_blockers_for_display is not None: + # The slot conflict display has better noise reduction than + # the unsatisfied blockers display, so skip unsatisfied blockers + # display if there are slot conflicts (see bug #385391). + if self._dynamic_config._slot_collision_info: + self._show_slot_collision_notice() + elif self._dynamic_config._unsatisfied_blockers_for_display is not None: self._show_unsatisfied_blockers( self._dynamic_config._unsatisfied_blockers_for_display) - elif self._dynamic_config._slot_collision_info: - self._show_slot_collision_notice() else: self._show_missed_update() + self._show_ignored_binaries() + self._display_autounmask() # TODO: Add generic support for "set problem" handlers so that @@ -6164,6 +6352,9 @@ class depgraph(object): show_mask_docs() writemsg("\n", noiselevel=-1) + for pargs, kwargs in self._dynamic_config._unsatisfied_deps_for_display: + self._show_unsatisfied_dep(*pargs, **kwargs) + def saveNomergeFavorites(self): """Find atoms in favorites that are not in the mergelist and add them to the world file if necessary.""" @@ -6184,7 +6375,6 @@ class depgraph(object): args_set = self._dynamic_config.sets[ self._frozen_config.target_root].sets['__non_set_args__'] - portdb = self._frozen_config.trees[self._frozen_config.target_root]["porttree"].dbapi added_favorites = set() for x in self._dynamic_config._set_nodes: if x.operation != "nomerge": @@ -6222,7 +6412,8 @@ class depgraph(object): all_added.extend(added_favorites) all_added.sort() for a in all_added: - writemsg(">>> Recording %s in \"world\" favorites file...\n" % \ + writemsg_stdout( + ">>> Recording %s in \"world\" favorites file...\n" % \ colorize("INFORM", str(a)), noiselevel=-1) if all_added: world_set.update(all_added) @@ -6247,15 +6438,12 @@ class depgraph(object): mergelist = [] favorites = resume_data.get("favorites") - args_set = self._dynamic_config.sets[ - self._frozen_config.target_root].sets['__non_set_args__'] if isinstance(favorites, list): args = self._load_favorites(favorites) else: args = [] fakedb = self._dynamic_config.mydbapi - trees = self._frozen_config.trees serialized_tasks = [] masked_tasks = [] for x in mergelist: @@ -6552,38 +6740,43 @@ class _dep_check_composite_db(dbapi): return ret def match(self, atom): - ret = self._match_cache.get(atom) + cache_key = (atom, atom.unevaluated_atom) + ret = self._match_cache.get(cache_key) if ret is not None: return ret[:] + + ret = [] pkg, existing = self._depgraph._select_package(self._root, atom) - if not pkg: - ret = [] - else: - # Return the highest available from select_package() as well as - # any matching slots in the graph db. + + if pkg is not None and self._visible(pkg): + self._cpv_pkg_map[pkg.cpv] = pkg + ret.append(pkg.cpv) + + if pkg is not None and \ + atom.slot is None and \ + pkg.cp.startswith("virtual/") and \ + (("remove" not in self._depgraph._dynamic_config.myparams and + "--update" not in self._depgraph._frozen_config.myopts) or + not ret or + not self._depgraph._virt_deps_visible(pkg, ignore_use=True)): + # For new-style virtual lookahead that occurs inside dep_check() + # for bug #141118, examine all slots. This is needed so that newer + # slots will not unnecessarily be pulled in when a satisfying lower + # slot is already installed. For example, if virtual/jdk-1.5 is + # satisfied via gcj-jdk then there's no need to pull in a newer + # slot to satisfy a virtual/jdk dependency, unless --update is + # enabled. slots = set() - slots.add(pkg.metadata["SLOT"]) - if pkg.cp.startswith("virtual/"): - # For new-style virtual lookahead that occurs inside - # dep_check(), examine all slots. This is needed - # so that newer slots will not unnecessarily be pulled in - # when a satisfying lower slot is already installed. For - # example, if virtual/jdk-1.4 is satisfied via kaffe then - # there's no need to pull in a newer slot to satisfy a - # virtual/jdk dependency. - for db, pkg_type, built, installed, db_keys in \ - self._depgraph._dynamic_config._filtered_trees[self._root]["dbs"]: - for cpv in db.match(atom): - if portage.cpv_getkey(cpv) != pkg.cp: - continue - slots.add(db.aux_get(cpv, ["SLOT"])[0]) - ret = [] - if self._visible(pkg): - self._cpv_pkg_map[pkg.cpv] = pkg - ret.append(pkg.cpv) - slots.remove(pkg.metadata["SLOT"]) + slots.add(pkg.slot) + for virt_pkg in self._depgraph._iter_match_pkgs_any( + self._depgraph._frozen_config.roots[self._root], atom): + if virt_pkg.cp != pkg.cp: + continue + slots.add(virt_pkg.slot) + + slots.remove(pkg.slot) while slots: - slot_atom = Atom("%s:%s" % (atom.cp, slots.pop())) + slot_atom = atom.with_slot(slots.pop()) pkg, existing = self._depgraph._select_package( self._root, slot_atom) if not pkg: @@ -6592,9 +6785,11 @@ class _dep_check_composite_db(dbapi): continue self._cpv_pkg_map[pkg.cpv] = pkg ret.append(pkg.cpv) - if ret: + + if len(ret) > 1: self._cpv_sort_ascending(ret) - self._match_cache[atom] = ret + + self._match_cache[cache_key] = ret return ret[:] def _visible(self, pkg): @@ -6650,7 +6845,7 @@ class _dep_check_composite_db(dbapi): # Note: highest_visible is not necessarily the real highest # visible, especially when --update is not enabled, so use # < operator instead of !=. - if pkg < highest_visible: + if highest_visible is not None and pkg < highest_visible: return False elif in_graph != pkg: # Mask choices for packages that would trigger a slot @@ -6832,7 +7027,7 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): PackageNotFound or depgraph.UnsatisfiedResumeDep when necessary. TODO: Return reasons for dropped_tasks, for display/logging. @rtype: tuple - @returns: (success, depgraph, dropped_tasks) + @return: (success, depgraph, dropped_tasks) """ skip_masked = True skip_unsatisfied = True @@ -6869,12 +7064,12 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): if not isinstance(parent_node, Package) \ or parent_node.operation not in ("merge", "nomerge"): continue - unsatisfied = \ - graph.child_nodes(parent_node, - ignore_priority=DepPrioritySatisfiedRange.ignore_soft) - if pkg in unsatisfied: - unsatisfied_parents[parent_node] = parent_node - unsatisfied_stack.append(parent_node) + # We need to traverse all priorities here, in order to + # ensure that a package with an unsatisfied depenedency + # won't get pulled in, even indirectly via a soft + # dependency. + unsatisfied_parents[parent_node] = parent_node + unsatisfied_stack.append(parent_node) unsatisfied_tuples = frozenset(tuple(parent_node) for parent_node in unsatisfied_parents @@ -6907,7 +7102,6 @@ def _resume_depgraph(settings, trees, mtimedb, myopts, myparams, spinner): def get_mask_info(root_config, cpv, pkgsettings, db, pkg_type, built, installed, db_keys, myrepo = None, _pkg_use_enabled=None): - eapi_masked = False try: metadata = dict(zip(db_keys, db.aux_get(cpv, db_keys, myrepo=myrepo))) @@ -6918,8 +7112,6 @@ def get_mask_info(root_config, cpv, pkgsettings, mreasons = ["corruption"] else: eapi = metadata['EAPI'] - if eapi[:1] == '-': - eapi = eapi[1:] if not portage.eapi_is_supported(eapi): mreasons = ['EAPI %s' % eapi] else: @@ -6976,10 +7168,11 @@ def show_masked_packages(masked_packages): # above via mreasons. pass - writemsg_stdout("- "+output_cpv+" (masked by: "+", ".join(mreasons)+")\n", noiselevel=-1) + writemsg("- "+output_cpv+" (masked by: "+", ".join(mreasons)+")\n", + noiselevel=-1) if comment and comment not in shown_comments: - writemsg_stdout(filename + ":\n" + comment + "\n", + writemsg(filename + ":\n" + comment + "\n", noiselevel=-1) shown_comments.add(comment) portdb = root_config.trees["porttree"].dbapi @@ -6989,13 +7182,14 @@ def show_masked_packages(masked_packages): continue msg = ("A copy of the '%s' license" + \ " is located at '%s'.\n\n") % (l, l_path) - writemsg_stdout(msg, noiselevel=-1) + writemsg(msg, noiselevel=-1) shown_licenses.add(l) return have_eapi_mask def show_mask_docs(): - writemsg_stdout("For more information, see the MASKED PACKAGES section in the emerge\n", noiselevel=-1) - writemsg_stdout("man page or refer to the Gentoo Handbook.\n", noiselevel=-1) + writemsg("For more information, see the MASKED PACKAGES " + "section in the emerge\n", noiselevel=-1) + writemsg("man page or refer to the Gentoo Handbook.\n", noiselevel=-1) def show_blocker_docs_link(): writemsg("\nFor more information about " + bad("Blocked Packages") + ", please refer to the following\n", noiselevel=-1) @@ -7017,7 +7211,7 @@ def _get_masking_status(pkg, pkgsettings, root_config, myrepo=None, use=None): pkg.metadata["CHOST"])) if pkg.invalid: - for msg_type, msgs in pkg.invalid.items(): + for msgs in pkg.invalid.values(): for msg in msgs: mreasons.append( _MaskReason("invalid", "invalid: %s" % (msg,))) diff --git a/portage_with_autodep/pym/_emerge/depgraph.pyo b/portage_with_autodep/pym/_emerge/depgraph.pyo Binary files differnew file mode 100644 index 0000000..ba00a11 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/depgraph.pyo diff --git a/portage_with_autodep/pym/_emerge/emergelog.py b/portage_with_autodep/pym/_emerge/emergelog.py index d6ef1b4..b1b093f 100644 --- a/portage_with_autodep/pym/_emerge/emergelog.py +++ b/portage_with_autodep/pym/_emerge/emergelog.py @@ -49,15 +49,12 @@ def emergelog(xterm_titles, mystr, short_msg=None): portage.util.apply_secpass_permissions(file_path, uid=portage.portage_uid, gid=portage.portage_gid, mode=0o660) - mylock = None + mylock = portage.locks.lockfile(file_path) try: - mylock = portage.locks.lockfile(mylogfile) mylogfile.write(_log_fmt % (time.time(), mystr)) - mylogfile.flush() - finally: - if mylock: - portage.locks.unlockfile(mylock) mylogfile.close() + finally: + portage.locks.unlockfile(mylock) except (IOError,OSError,portage.exception.PortageException) as e: if secpass >= 1: print("emergelog():",e, file=sys.stderr) diff --git a/portage_with_autodep/pym/_emerge/emergelog.pyo b/portage_with_autodep/pym/_emerge/emergelog.pyo Binary files differnew file mode 100644 index 0000000..7e67bd3 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/emergelog.pyo diff --git a/portage_with_autodep/pym/_emerge/getloadavg.pyo b/portage_with_autodep/pym/_emerge/getloadavg.pyo Binary files differnew file mode 100644 index 0000000..56bda8c --- /dev/null +++ b/portage_with_autodep/pym/_emerge/getloadavg.pyo diff --git a/portage_with_autodep/pym/_emerge/help.py b/portage_with_autodep/pym/_emerge/help.py index c978ce2..a1dbb37 100644 --- a/portage_with_autodep/pym/_emerge/help.py +++ b/portage_with_autodep/pym/_emerge/help.py @@ -3,10 +3,9 @@ from __future__ import print_function -from portage.const import _ENABLE_DYN_LINK_MAP from portage.output import bold, turquoise, green -def shorthelp(): +def help(): print(bold("emerge:")+" the other white meat (command-line interface to the Portage system)") print(bold("Usage:")) print(" "+turquoise("emerge")+" [ "+green("options")+" ] [ "+green("action")+" ] [ "+turquoise("ebuild")+" | "+turquoise("tbz2")+" | "+turquoise("file")+" | "+turquoise("@set")+" | "+turquoise("atom")+" ] [ ... ]") @@ -19,797 +18,8 @@ def shorthelp(): print(" [ "+green("--complete-graph")+" ] [ "+green("--deep")+" ]") print(" [ "+green("--jobs") + " " + turquoise("JOBS")+" ] [ "+green("--keep-going")+" ] [ " + green("--load-average")+" " + turquoise("LOAD") + " ]") print(" [ "+green("--newuse")+" ] [ "+green("--noconfmem")+" ] [ "+green("--nospinner")+" ]") - print(" [ "+green("--oneshot")+" ] [ "+green("--onlydeps")+" ]") + print(" [ "+green("--oneshot")+" ] [ "+green("--onlydeps")+" ] [ "+ green("--quiet-build")+" [ " + turquoise("y") + " | "+ turquoise("n")+" ] ]") print(" [ "+green("--reinstall ")+turquoise("changed-use")+" ] [ " + green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" > ]") print(bold("Actions:")+" [ "+green("--depclean")+" | "+green("--list-sets")+" | "+green("--search")+" | "+green("--sync")+" | "+green("--version")+" ]") - -def help(myopts, havecolor=1): - # TODO: Implement a wrap() that accounts for console color escape codes. - from textwrap import wrap - desc_left_margin = 14 - desc_indent = desc_left_margin * " " - desc_width = 80 - desc_left_margin - 5 - if "--verbose" not in myopts: - shorthelp() - print() - print(" For more help try 'emerge --help --verbose' or consult the man page.") - else: - shorthelp() - print() - print(turquoise("Help (this screen):")) - print(" "+green("--help")+" ("+green("-h")+" short option)") - print(" Displays this help; an additional argument (see above) will tell") - print(" emerge to display detailed help.") - print() - print(turquoise("Actions:")) - print(" "+green("--clean")) - print(" Cleans the system by removing outdated packages which will not") - print(" remove functionalities or prevent your system from working.") - print(" The arguments can be in several different formats :") - print(" * world ") - print(" * system or") - print(" * 'dependency specification' (in single quotes is best.)") - print(" Here are a few examples of the dependency specification format:") - print(" "+bold("binutils")+" matches") - print(" binutils-2.11.90.0.7 and binutils-2.11.92.0.12.3-r1") - print(" "+bold("sys-devel/binutils")+" matches") - print(" binutils-2.11.90.0.7 and binutils-2.11.92.0.12.3-r1") - print(" "+bold(">sys-devel/binutils-2.11.90.0.7")+" matches") - print(" binutils-2.11.92.0.12.3-r1") - print(" "+bold(">=sys-devel/binutils-2.11.90.0.7")+" matches") - print(" binutils-2.11.90.0.7 and binutils-2.11.92.0.12.3-r1") - print(" "+bold("<=sys-devel/binutils-2.11.92.0.12.3-r1")+" matches") - print(" binutils-2.11.90.0.7 and binutils-2.11.92.0.12.3-r1") - print() - print(" "+green("--config")) - print(" Runs package-specific operations that must be executed after an") - print(" emerge process has completed. This usually entails configuration") - print(" file setup or other similar setups that the user may wish to run.") - print() - print(" "+green("--depclean")+" ("+green("-c")+" short option)") - - paragraph = "Cleans the system by removing packages that are " + \ - "not associated with explicitly merged packages. Depclean works " + \ - "by creating the full dependency tree from the " + \ - "@world set, then comparing it to installed packages. Packages " + \ - "installed, but not part of the dependency tree, will be " + \ - "uninstalled by depclean. See --with-bdeps for behavior with " + \ - "respect to build time dependencies that are not strictly " + \ - "required. Packages that are part of the world set will " + \ - "always be kept. They can be manually added to this set with " + \ - "emerge --noreplace <atom>. As a safety measure, depclean " + \ - "will not remove any packages unless *all* required dependencies " + \ - "have been resolved. As a consequence, it is often necessary to " + \ - "run emerge --update --newuse --deep @world " + \ - "prior to depclean." - - for line in wrap(paragraph, desc_width): - print(desc_indent + line) - print() - - paragraph = "WARNING: Inexperienced users are advised to use " + \ - "--pretend with this option in order to see a preview of which " + \ - "packages will be uninstalled. Always study the list of packages " + \ - "to be cleaned for any obvious mistakes. Note that packages " + \ - "listed in package.provided (see portage(5)) may be removed by " + \ - "depclean, even if they are part of the world set." - - paragraph += " Also note that " + \ - "depclean may break link level dependencies" - - if _ENABLE_DYN_LINK_MAP: - paragraph += ", especially when the " + \ - "--depclean-lib-check option is disabled" - - paragraph += ". Thus, it is " + \ - "recommended to use a tool such as revdep-rebuild(1) " + \ - "in order to detect such breakage." - - for line in wrap(paragraph, desc_width): - print(desc_indent + line) - print() - - paragraph = "Depclean serves as a dependency aware version of " + \ - "--unmerge. When given one or more atoms, it will unmerge " + \ - "matched packages that have no reverse dependencies. Use " + \ - "--depclean together with --verbose to show reverse dependencies." - - for line in wrap(paragraph, desc_width): - print(desc_indent + line) - print() - print(" " + green("--deselect") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - - paragraph = \ - "Remove atoms and/or sets from the world file. This action is implied " + \ - "by uninstall actions, including --depclean, " + \ - "--prune and --unmerge. Use --deselect=n " + \ - "in order to prevent uninstall actions from removing " + \ - "atoms from the world file." - - for line in wrap(paragraph, desc_width): - print(desc_indent + line) - print() - print(" " + green("--ignore-default-opts")) - - paragraph = \ - "Causes EMERGE_DEFAULT_OPTS (see make.conf(5)) to be ignored." - - for line in wrap(paragraph, desc_width): - print(desc_indent + line) - print() - print(" "+green("--info")) - print(" Displays important portage variables that will be exported to") - print(" ebuild.sh when performing merges. This information is useful") - print(" for bug reports and verification of settings. All settings in") - print(" make.{conf,globals,defaults} and the environment show up if") - print(" run with the '--verbose' flag.") - print() - print(" " + green("--list-sets")) - paragraph = "Displays a list of available package sets." - - for line in wrap(paragraph, desc_width): - print(desc_indent + line) - print() - print(" "+green("--metadata")) - print(" Transfers metadata cache from ${PORTDIR}/metadata/cache/ to") - print(" /var/cache/edb/dep/ as is normally done on the tail end of an") - print(" rsync update using " + bold("emerge --sync") + ". This process populates the") - print(" cache database that portage uses for pre-parsed lookups of") - print(" package data. It does not populate cache for the overlays") - print(" listed in PORTDIR_OVERLAY. In order to generate cache for") - print(" overlays, use " + bold("--regen") + ".") - print() - print(" "+green("--prune")+" ("+green("-P")+" short option)") - print(" "+turquoise("WARNING: This action can remove important packages!")) - paragraph = "Removes all but the highest installed version of a " + \ - "package from your system. Use --prune together with " + \ - "--verbose to show reverse dependencies or with --nodeps " + \ - "to ignore all dependencies. " - - for line in wrap(paragraph, desc_width): - print(desc_indent + line) - print() - print(" "+green("--regen")) - print(" Causes portage to check and update the dependency cache of all") - print(" ebuilds in the portage tree. This is not recommended for rsync") - print(" users as rsync updates the cache using server-side caches.") - print(" Rsync users should simply 'emerge --sync' to regenerate.") - desc = "In order to specify parallel --regen behavior, use "+ \ - "the ---jobs and --load-average options. If you would like to " + \ - "generate and distribute cache for use by others, use egencache(1)." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--resume")+" ("+green("-r")+" short option)") - print(" Resumes the most recent merge list that has been aborted due to an") - print(" error. Please note that this operation will only return an error") - print(" on failure. If there is nothing for portage to do, then portage") - print(" will exit with a message and a success condition. A resume list") - print(" will persist until it has been completed in entirety or until") - print(" another aborted merge list replaces it. The resume history is") - print(" capable of storing two merge lists. After one resume list") - print(" completes, it is possible to invoke --resume once again in order") - print(" to resume an older list.") - print() - print(" "+green("--search")+" ("+green("-s")+" short option)") - print(" Searches for matches of the supplied string in the current local") - print(" portage tree. By default emerge uses a case-insensitive simple ") - print(" search, but you can enable a regular expression search by ") - print(" prefixing the search string with %.") - print(" Prepending the expression with a '@' will cause the category to") - print(" be included in the search.") - print(" A few examples:") - print(" "+bold("emerge --search libc")) - print(" list all packages that contain libc in their name") - print(" "+bold("emerge --search '%^kde'")) - print(" list all packages starting with kde") - print(" "+bold("emerge --search '%gcc$'")) - print(" list all packages ending with gcc") - print(" "+bold("emerge --search '%@^dev-java.*jdk'")) - print(" list all available Java JDKs") - print() - print(" "+green("--searchdesc")+" ("+green("-S")+" short option)") - print(" Matches the search string against the description field as well") - print(" the package's name. Take caution as the descriptions are also") - print(" matched as regular expressions.") - print(" emerge -S html") - print(" emerge -S applet") - print(" emerge -S 'perl.*module'") - print() - print(" "+green("--sync")) - desc = "This updates the portage tree that is located in the " + \ - "directory that the PORTDIR variable refers to (default " + \ - "location is /usr/portage). The SYNC variable specifies " + \ - "the remote URI from which files will be synchronized. " + \ - "The PORTAGE_SYNC_STALE variable configures " + \ - "warnings that are shown when emerge --sync has not " + \ - "been executed recently." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(desc_indent + turquoise("WARNING:")) - desc = "The emerge --sync action will modify and/or delete " + \ - "files located inside the directory that the PORTDIR " + \ - "variable refers to (default location is /usr/portage). " + \ - "For more information, see the PORTDIR documentation in " + \ - "the make.conf(5) man page." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(desc_indent + green("NOTE:")) - desc = "The emerge-webrsync program will download the entire " + \ - "portage tree as a tarball, which is much faster than emerge " + \ - "--sync for first time syncs." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--unmerge")+" ("+green("-C")+" short option)") - print(" "+turquoise("WARNING: This action can remove important packages!")) - print(" Removes all matching packages. This does no checking of") - print(" dependencies, so it may remove packages necessary for the proper") - print(" operation of your system. Its arguments can be atoms or") - print(" ebuilds. For a dependency aware version of --unmerge, use") - print(" --depclean or --prune.") - print() - print(" "+green("--version")+" ("+green("-V")+" short option)") - print(" Displays the currently installed version of portage along with") - print(" other information useful for quick reference on a system. See") - print(" "+bold("emerge info")+" for more advanced information.") - print() - print(turquoise("Options:")) - print(" "+green("--accept-properties=ACCEPT_PROPERTIES")) - desc = "This option temporarily overrides the ACCEPT_PROPERTIES " + \ - "variable. The ACCEPT_PROPERTIES variable is incremental, " + \ - "which means that the specified setting is appended to the " + \ - "existing value from your configuration. The special -* " + \ - "token can be used to discard the existing configuration " + \ - "value and start fresh. See the MASKED PACKAGES section " + \ - "and make.conf(5) for more information about " + \ - "ACCEPT_PROPERTIES. A typical usage example for this option " + \ - "would be to use --accept-properties=-interactive to " + \ - "temporarily mask interactive packages. With default " + \ - "configuration, this would result in an effective " + \ - "ACCEPT_PROPERTIES value of \"* -interactive\"." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--alphabetical")) - print(" When displaying USE and other flag output, combines the enabled") - print(" and disabled flags into a single list and sorts it alphabetically.") - print(" With this option, output such as USE=\"dar -bar -foo\" will instead") - print(" be displayed as USE=\"-bar dar -foo\"") - print() - print(" " + green("--ask") + \ - " [ %s | %s ] (%s short option)" % \ - (turquoise("y"), turquoise("n"), green("-a"))) - desc = "Before performing the action, display what will take place (server info for " + \ - "--sync, --pretend output for merge, and so forth), then ask " + \ - "whether to proceed with the action or abort. Using --ask is more " + \ - "efficient than using --pretend and then executing the same command " + \ - "without --pretend, as dependencies will only need to be calculated once. " + \ - "WARNING: If the \"Enter\" key is pressed at the prompt (with no other input), " + \ - "it is interpreted as acceptance of the first choice. Note that the input " + \ - "buffer is not cleared prior to the prompt, so an accidental press of the " + \ - "\"Enter\" key at any time prior to the prompt will be interpreted as a choice! " + \ - "Use the --ask-enter-invalid option if you want a single \"Enter\" key " + \ - "press to be interpreted as invalid input." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--ask-enter-invalid")) - desc = "When used together with the --ask option, " + \ - "interpret a single \"Enter\" key press as " + \ - "invalid input. This helps prevent accidental " + \ - "acceptance of the first choice. This option is " + \ - "intended to be set in the make.conf(5) " + \ - "EMERGE_DEFAULT_OPTS variable." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--autounmask") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Automatically unmask packages and generate package.use " + \ - "settings as necessary to satisfy dependencies. This " + \ - "option is enabled by default. If any configuration " + \ - "changes are required, then they will be displayed " + \ - "after the merge list and emerge will immediately " + \ - "abort. If the displayed configuration changes are " + \ - "satisfactory, you should copy and paste them into " + \ - "the specified configuration file(s), or enable the " + \ - "--autounmask-write option. The " + \ - "EMERGE_DEFAULT_OPTS variable may be used to " + \ - "disable this option by default in make.conf(5)." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--autounmask-write") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "If --autounmask is enabled, changes are written " + \ - "to config files, respecting CONFIG_PROTECT and --ask." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--backtrack") + " " + turquoise("COUNT")) - desc = "Specifies an integer number of times to backtrack if " + \ - "dependency calculation fails due to a conflict or an " + \ - "unsatisfied dependency (default: '10')." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--binpkg-respect-use") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Tells emerge to ignore binary packages if their use flags" + \ - " don't match the current configuration. (default: 'n')" - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--buildpkg") + \ - " [ %s | %s ] (%s short option)" % \ - (turquoise("y"), turquoise("n"), green("-b"))) - desc = "Tells emerge to build binary packages for all ebuilds processed in" + \ - " addition to actually merging the packages. Useful for maintainers" + \ - " or if you administrate multiple Gentoo Linux systems (build once," + \ - " emerge tbz2s everywhere) as well as disaster recovery. The package" + \ - " will be created in the" + \ - " ${PKGDIR}/All directory. An alternative for already-merged" + \ - " packages is to use quickpkg(1) which creates a tbz2 from the" + \ - " live filesystem." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--buildpkgonly")+" ("+green("-B")+" short option)") - print(" Creates a binary package, but does not merge it to the") - print(" system. This has the restriction that unsatisfied dependencies") - print(" must not exist for the desired package as they cannot be used if") - print(" they do not exist on the system.") - print() - print(" " + green("--changed-use")) - desc = "This is an alias for --reinstall=changed-use." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--changelog")+" ("+green("-l")+" short option)") - print(" When pretending, also display the ChangeLog entries for packages") - print(" that will be upgraded.") - print() - print(" "+green("--color") + " < " + turquoise("y") + " | "+ turquoise("n")+" >") - print(" Enable or disable color output. This option will override NOCOLOR") - print(" (see make.conf(5)) and may also be used to force color output when") - print(" stdout is not a tty (by default, color is disabled unless stdout") - print(" is a tty).") - print() - print(" "+green("--columns")) - print(" Display the pretend output in a tabular form. Versions are") - print(" aligned vertically.") - print() - print(" "+green("--complete-graph") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "This causes emerge to consider the deep dependencies of all" + \ - " packages from the world set. With this option enabled," + \ - " emerge will bail out if it determines that the given operation will" + \ - " break any dependencies of the packages that have been added to the" + \ - " graph. Like the --deep option, the --complete-graph" + \ - " option will significantly increase the time taken for dependency" + \ - " calculations. Note that, unlike the --deep option, the" + \ - " --complete-graph option does not cause any more packages to" + \ - " be updated than would have otherwise " + \ - "been updated with the option disabled. " + \ - "Using --with-bdeps=y together with --complete-graph makes " + \ - "the graph as complete as possible." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--config-root=DIR")) - desc = "Set the PORTAGE_CONFIGROOT environment variable " + \ - "which is documented in the emerge(1) man page." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--debug")+" ("+green("-d")+" short option)") - print(" Tell emerge to run the ebuild command in --debug mode. In this") - print(" mode, the bash build environment will run with the -x option,") - print(" causing it to output verbose debug information print to stdout.") - print(" --debug is great for finding bash syntax errors as providing") - print(" very verbose information about the dependency and build process.") - print() - print(" "+green("--deep") + " " + turquoise("[DEPTH]") + \ - " (" + green("-D") + " short option)") - print(" This flag forces emerge to consider the entire dependency tree of") - print(" packages, instead of checking only the immediate dependencies of") - print(" the packages. As an example, this catches updates in libraries") - print(" that are not directly listed in the dependencies of a package.") - print(" Also see --with-bdeps for behavior with respect to build time") - print(" dependencies that are not strictly required.") - print() - - if _ENABLE_DYN_LINK_MAP: - print(" " + green("--depclean-lib-check") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Account for library link-level dependencies during " + \ - "--depclean and --prune actions. This " + \ - "option is enabled by default. In some cases this can " + \ - "be somewhat time-consuming. This option is ignored " + \ - "when FEATURES=\"preserve-libs\" is enabled in " + \ - "make.conf(5), since any libraries that have " + \ - "consumers will simply be preserved." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - - print(" "+green("--emptytree")+" ("+green("-e")+" short option)") - desc = "Reinstalls target atoms and their entire deep " + \ - "dependency tree, as though no packages are currently " + \ - "installed. You should run this with --pretend " + \ - "first to make sure the result is what you expect." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--exclude") + " " + turquoise("ATOMS")) - desc = "A space separated list of package names or slot atoms. " + \ - "Emerge won't install any ebuild or binary package that " + \ - "matches any of the given package atoms." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--fail-clean") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Clean up temporary files after a build failure. This is " + \ - "particularly useful if you have PORTAGE_TMPDIR on " + \ - "tmpfs. If this option is enabled, you probably also want " + \ - "to enable PORT_LOGDIR (see make.conf(5)) in " + \ - "order to save the build log." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--fetchonly")+" ("+green("-f")+" short option)") - print(" Instead of doing any package building, just perform fetches for") - print(" all packages (main package as well as all dependencies.) When") - print(" used in combination with --pretend all the SRC_URIs will be") - print(" displayed multiple mirrors per line, one line per file.") - print() - print(" "+green("--fetch-all-uri")+" ("+green("-F")+" short option)") - print(" Same as --fetchonly except that all package files, including those") - print(" not required to build the package, will be processed.") - print() - print(" " + green("--getbinpkg") + \ - " [ %s | %s ] (%s short option)" % \ - (turquoise("y"), turquoise("n"), green("-g"))) - print(" Using the server and location defined in PORTAGE_BINHOST, portage") - print(" will download the information from each binary file there and it") - print(" will use that information to help build the dependency list. This") - print(" option implies '-k'. (Use -gK for binary-only merging.)") - print() - print(" " + green("--getbinpkgonly") + \ - " [ %s | %s ] (%s short option)" % \ - (turquoise("y"), turquoise("n"), green("-G"))) - print(" This option is identical to -g, as above, except it will not use") - print(" ANY information from the local machine. All binaries will be") - print(" downloaded from the remote server without consulting packages") - print(" existing in the packages directory.") - print() - print(" " + green("--jobs") + " " + turquoise("[JOBS]") + " ("+green("-j")+" short option)") - desc = "Specifies the number of packages " + \ - "to build simultaneously. If this option is " + \ - "given without an argument, emerge will not " + \ - "limit the number of jobs that " + \ - "can run simultaneously. Also see " + \ - "the related --load-average option. " + \ - "Note that interactive packages currently force a setting " + \ - "of --jobs=1. This issue can be temporarily avoided " + \ - "by specifying --accept-properties=-interactive." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--keep-going") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Continue as much as possible after " + \ - "an error. When an error occurs, " + \ - "dependencies are recalculated for " + \ - "remaining packages and any with " + \ - "unsatisfied dependencies are " + \ - "automatically dropped. Also see " + \ - "the related --skipfirst option." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--load-average") + " " + turquoise("LOAD")) - desc = "Specifies that no new builds should " + \ - "be started if there are other builds " + \ - "running and the load average is at " + \ - "least LOAD (a floating-point number). " + \ - "This option is recommended for use " + \ - "in combination with --jobs in " + \ - "order to avoid excess load. See " + \ - "make(1) for information about " + \ - "analogous options that should be " + \ - "configured via MAKEOPTS in " + \ - "make.conf(5)." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--misspell-suggestions") + " < %s | %s >" % \ - (turquoise("y"), turquoise("n"))) - desc = "Enable or disable misspell suggestions. By default, " + \ - "emerge will show a list of packages with similar names " + \ - "when a package doesn't exist. The EMERGE_DEFAULT_OPTS " + \ - "variable may be used to disable this option by default" - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--newuse")+" ("+green("-N")+" short option)") - desc = "Tells emerge to include installed packages where USE " + \ - "flags have changed since compilation. This option " + \ - "also implies the --selective option. If you would " + \ - "like to skip rebuilds for which disabled flags have " + \ - "been added to or removed from IUSE, see the related " + \ - "--reinstall=changed-use option." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--noconfmem")) - print(" Portage keeps track of files that have been placed into") - print(" CONFIG_PROTECT directories, and normally it will not merge the") - print(" same file more than once, as that would become annoying. This") - print(" can lead to problems when the user wants the file in the case") - print(" of accidental deletion. With this option, files will always be") - print(" merged to the live fs instead of silently dropped.") - print() - print(" "+green("--nodeps")+" ("+green("-O")+" short option)") - print(" Merge specified packages, but don't merge any dependencies.") - print(" Note that the build may fail if deps aren't satisfied.") - print() - print(" "+green("--noreplace")+" ("+green("-n")+" short option)") - print(" Skip the packages specified on the command-line that have") - print(" already been installed. Without this option, any packages,") - print(" ebuilds, or deps you specify on the command-line *will* cause") - print(" Portage to remerge the package, even if it is already installed.") - print(" Note that Portage won't remerge dependencies by default.") - print() - print(" "+green("--nospinner")) - print(" Disables the spinner regardless of terminal type.") - print() - print(" " + green("--usepkg-exclude") + " " + turquoise("ATOMS")) - desc = "A space separated list of package names or slot atoms." + \ - " Emerge will ignore matching binary packages." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--rebuild-exclude") + " " + turquoise("ATOMS")) - desc = "A space separated list of package names or slot atoms." + \ - " Emerge will not rebuild matching packages due to --rebuild." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--rebuild-ignore") + " " + turquoise("ATOMS")) - desc = "A space separated list of package names or slot atoms." + \ - " Emerge will not rebuild packages that depend on matching " + \ - " packages due to --rebuild." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--oneshot")+" ("+green("-1")+" short option)") - print(" Emerge as normal, but don't add packages to the world profile.") - print(" This package will only be updated if it is depended upon by") - print(" another package.") - print() - print(" "+green("--onlydeps")+" ("+green("-o")+" short option)") - print(" Only merge (or pretend to merge) the dependencies of the") - print(" specified packages, not the packages themselves.") - print() - print(" " + green("--package-moves") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Perform package moves when necessary. This option " + \ - "is enabled by default. WARNING: This option " + \ - "should remain enabled under normal circumstances. " + \ - "Do not disable it unless you know what you are " + \ - "doing." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--pretend")+" ("+green("-p")+" short option)") - print(" Instead of actually performing the merge, simply display what") - print(" ebuilds and tbz2s *would* have been installed if --pretend") - print(" weren't used. Using --pretend is strongly recommended before") - print(" installing an unfamiliar package. In the printout, N = new,") - print(" U = updating, R = replacing, F = fetch restricted, B = blocked") - print(" by an already installed package, D = possible downgrading,") - print(" S = slotted install. --verbose causes affecting use flags to be") - print(" printed out accompanied by a '+' for enabled and a '-' for") - print(" disabled USE flags.") - print() - print(" " + green("--quiet") + \ - " [ %s | %s ] (%s short option)" % \ - (turquoise("y"), turquoise("n"), green("-q"))) - print(" Effects vary, but the general outcome is a reduced or condensed") - print(" output from portage's displays.") - print() - print(" " + green("--quiet-build") + \ - " [ %s | %s ]" % (turquoise("y"), turquoise("n"))) - desc = "Redirect all build output to logs alone, and do not " + \ - "display it on stdout." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--quiet-unmerge-warn")) - desc = "Disable the warning message that's shown prior to " + \ - "--unmerge actions. This option is intended " + \ - "to be set in the make.conf(5) " + \ - "EMERGE_DEFAULT_OPTS variable." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--rebuild-if-new-rev") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Rebuild packages when dependencies that are " + \ - "used at both build-time and run-time are built, " + \ - "if the dependency is not already installed with the " + \ - "same version and revision." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--rebuild-if-new-ver") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Rebuild packages when dependencies that are " + \ - "used at both build-time and run-time are built, " + \ - "if the dependency is not already installed with the " + \ - "same version. Revision numbers are ignored." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--rebuild-if-unbuilt") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Rebuild packages when dependencies that are " + \ - "used at both build-time and run-time are built." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--rebuilt-binaries") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Replace installed packages with binary packages that have " + \ - "been rebuilt. Rebuilds are detected by comparison of " + \ - "BUILD_TIME package metadata. This option is enabled " + \ - "automatically when using binary packages " + \ - "(--usepkgonly or --getbinpkgonly) together with " + \ - "--update and --deep." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--rebuilt-binaries-timestamp") + "=%s" % turquoise("TIMESTAMP")) - desc = "This option modifies emerge's behaviour only if " + \ - "--rebuilt-binaries is given. Only binaries that " + \ - "have a BUILD_TIME that is larger than the given TIMESTAMP " + \ - "and that is larger than that of the installed package will " + \ - "be considered by the rebuilt-binaries logic." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--reinstall ") + turquoise("changed-use")) - print(" Tells emerge to include installed packages where USE flags have") - print(" changed since installation. Unlike --newuse, this option does") - print(" not trigger reinstallation when flags that the user has not") - print(" enabled are added or removed.") - print() - print(" " + green("--reinstall-atoms") + " " + turquoise("ATOMS")) - desc = "A space separated list of package names or slot atoms. " + \ - "Emerge will treat matching packages as if they are not " + \ - "installed, and reinstall them if necessary." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--root=DIR")) - desc = "Set the ROOT environment variable " + \ - "which is documented in the emerge(1) man page." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--root-deps[=rdeps]")) - desc = "If no argument is given then build-time dependencies of packages for " + \ - "ROOT are installed to " + \ - "ROOT instead of /. If the rdeps argument is given then discard " + \ - "all build-time dependencies of packages for ROOT. This option is " + \ - "only meaningful when used together with ROOT and it should not " + \ - "be enabled under normal circumstances. For currently supported " + \ - "EAPI values, the build-time dependencies are specified in the " + \ - "DEPEND variable. However, behavior may change for new " + \ - "EAPIs when related extensions are added in the future." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--select") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Add specified packages to the world set (inverse of " + \ - "--oneshot). This is useful if you want to " + \ - "use EMERGE_DEFAULT_OPTS to make " + \ - "--oneshot behavior default." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--selective") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "This identical to the --noreplace option. " + \ - "Some options, such as --update, imply --selective. " + \ - "Use --selective=n if you want to forcefully disable " + \ - "--selective, regardless of options like --update." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--skipfirst")) - desc = "This option is only valid when " + \ - "used with --resume. It removes the " + \ - "first package in the resume list. " + \ - "Dependencies are recalculated for " + \ - "remaining packages and any that " + \ - "have unsatisfied dependencies or are " + \ - "masked will be automatically dropped. " + \ - "Also see the related " + \ - "--keep-going option." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--tree")+" ("+green("-t")+" short option)") - print(" Shows the dependency tree using indentation for dependencies.") - print(" The packages are also listed in reverse merge order so that") - print(" a package's dependencies follow the package. Only really useful") - print(" in combination with --emptytree, --update or --deep.") - print() - print(" " + green("--unordered-display")) - desc = "By default the displayed merge list is sorted using the " + \ - "order in which the packages will be merged. When " + \ - "--tree is used together with this option, this " + \ - "constraint is removed, hopefully leading to a more " + \ - "readable dependency tree." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" "+green("--update")+" ("+green("-u")+" short option)") - desc = "Updates packages to the best version available, which may " + \ - "not always be the highest version number due to masking " + \ - "for testing and development. Package atoms specified on " + \ - "the command line are greedy, meaning that unspecific " + \ - "atoms may match multiple versions of slotted packages." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--use-ebuild-visibility") + " [ %s | %s ]" % \ - (turquoise("y"), turquoise("n"))) - desc = "Use unbuilt ebuild metadata for visibility " + \ - "checks on built packages." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--useoldpkg-atoms") + " " + turquoise("ATOMS")) - desc = "A space separated list of package names or slot atoms." + \ - " Emerge will prefer matching binary packages over newer" + \ - " unbuilt packages." - for line in wrap(desc, desc_width): - print(desc_indent + line) - print() - print(" " + green("--usepkg") + \ - " [ %s | %s ] (%s short option)" % \ - (turquoise("y"), turquoise("n"), green("-k"))) - print(" Tell emerge to use binary packages (from $PKGDIR) if they are") - print(" available, thus possibly avoiding some time-consuming compiles.") - print(" This option is useful for CD installs; you can export") - print(" PKGDIR=/mnt/cdrom/packages and then use this option to have") - print(" emerge \"pull\" binary packages from the CD in order to satisfy") - print(" dependencies.") - print() - print(" " + green("--usepkgonly") + \ - " [ %s | %s ] (%s short option)" % \ - (turquoise("y"), turquoise("n"), green("-K"))) - print(" Like --usepkg above, except this only allows the use of binary") - print(" packages, and it will abort the emerge if the package is not") - print(" available at the time of dependency calculation.") - print() - print(" "+green("--verbose")+" ("+green("-v")+" short option)") - print(" Effects vary, but the general outcome is an increased or expanded") - print(" display of content in portage's displays.") - print() - print(" "+green("--with-bdeps")+" < " + turquoise("y") + " | "+ turquoise("n")+" >") - print(" In dependency calculations, pull in build time dependencies that") - print(" are not strictly required. This defaults to 'n' for installation") - print(" actions and 'y' for the --depclean action. This setting can be") - print(" added to EMERGE_DEFAULT_OPTS (see make.conf(5)) and later") - print(" overridden via the command line.") - print() + print() + print(" For more help consult the man page.") diff --git a/portage_with_autodep/pym/_emerge/help.pyo b/portage_with_autodep/pym/_emerge/help.pyo Binary files differnew file mode 100644 index 0000000..f6fea4e --- /dev/null +++ b/portage_with_autodep/pym/_emerge/help.pyo diff --git a/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo b/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo Binary files differnew file mode 100644 index 0000000..20edc85 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/is_valid_package_atom.pyo diff --git a/portage_with_autodep/pym/_emerge/main.py b/portage_with_autodep/pym/_emerge/main.py index 2830214..c52a3ea 100644 --- a/portage_with_autodep/pym/_emerge/main.py +++ b/portage_with_autodep/pym/_emerge/main.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -6,10 +6,14 @@ from __future__ import print_function import logging import signal import stat +import subprocess import sys import textwrap import platform import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.news:count_unread_news,display_news_notifications', +) from portage import os from portage import _encodings from portage import _unicode_decode @@ -28,7 +32,8 @@ import portage.exception from portage.data import secpass from portage.dbapi.dep_expand import dep_expand from portage.util import normalize_path as normpath -from portage.util import shlex_split, writemsg_level, writemsg_stdout +from portage.util import (shlex_split, varexpand, + writemsg_level, writemsg_stdout) from portage._sets import SETPREFIX from portage._global_updates import _global_updates @@ -62,6 +67,7 @@ options=[ "--nodeps", "--noreplace", "--nospinner", "--oneshot", "--onlydeps", "--pretend", +"--quiet-repo-display", "--quiet-unmerge-warn", "--resume", "--searchdesc", @@ -70,6 +76,7 @@ options=[ "--unordered-display", "--update", "--verbose", +"--verbose-main-repo-display", ] shortmapping={ @@ -92,6 +99,21 @@ shortmapping={ "v":"--verbose", "V":"--version" } +COWSAY_MOO = """ + + Larry loves Gentoo (%s) + + _______________________ +< Have you mooed today? > + ----------------------- + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || + +""" + def chk_updated_info_files(root, infodirs, prev_mtimes, retval): if os.path.exists("/usr/bin/install-info"): @@ -158,11 +180,21 @@ def chk_updated_info_files(root, infodirs, prev_mtimes, retval): raise del e processed_count += 1 - myso = portage.subprocess_getstatusoutput( - "LANG=C LANGUAGE=C /usr/bin/install-info " + - "--dir-file=%s/dir %s/%s" % (inforoot, inforoot, x))[1] + try: + proc = subprocess.Popen( + ['/usr/bin/install-info', + '--dir-file=%s' % os.path.join(inforoot, "dir"), + os.path.join(inforoot, x)], + env=dict(os.environ, LANG="C", LANGUAGE="C"), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + myso = None + else: + myso = _unicode_decode( + proc.communicate()[0]).rstrip("\n") + proc.wait() existsstr="already exists, for file `" - if myso!="": + if myso: if re.search(existsstr,myso): # Already exists... Don't increment the count for this. pass @@ -233,7 +265,6 @@ def display_preserved_libs(vardbapi, myopts): linkmap = vardbapi._linkmap consumer_map = {} owners = {} - linkmap_broken = False try: linkmap.rebuild() @@ -241,7 +272,6 @@ def display_preserved_libs(vardbapi, myopts): writemsg_level("!!! Command Not Found: %s\n" % (e,), level=logging.ERROR, noiselevel=-1) del e - linkmap_broken = True else: search_for_owners = set() for cpv in plibdata: @@ -315,7 +345,7 @@ def post_emerge(myaction, myopts, myfiles, @type myopts: dict @param myfiles: emerge arguments @type myfiles: list - @param target_root: The target ROOT for myaction + @param target_root: The target EROOT for myaction @type target_root: String @param trees: A dictionary mapping each ROOT to it's package databases @type trees: dict @@ -326,7 +356,7 @@ def post_emerge(myaction, myopts, myfiles, """ root_config = trees[target_root]["root_config"] - vardbapi = trees[target_root]["vartree"].dbapi + vardbapi = trees[target_root]['vartree'].dbapi settings = vardbapi.settings info_mtimes = mtimedb["info"] @@ -351,7 +381,9 @@ def post_emerge(myaction, myopts, myfiles, _flush_elog_mod_echo() if not vardbapi._pkgs_changed: - display_news_notification(root_config, myopts) + # GLEP 42 says to display news *after* an emerge --pretend + if "--pretend" in myopts: + display_news_notification(root_config, myopts) # If vdb state has not changed then there's nothing else to do. return @@ -372,11 +404,10 @@ def post_emerge(myaction, myopts, myfiles, if vdb_lock: vardbapi.unlock() + display_preserved_libs(vardbapi, myopts) chk_updated_cfg_files(settings['EROOT'], config_protect) display_news_notification(root_config, myopts) - if retval in (None, os.EX_OK) or (not "--pretend" in myopts): - display_preserved_libs(vardbapi, myopts) postemerge = os.path.join(settings["PORTAGE_CONFIGROOT"], portage.USER_CONFIG_PATH, "bin", "post_emerge") @@ -388,6 +419,8 @@ def post_emerge(myaction, myopts, myfiles, " %s spawn failed of %s\n" % (bad("*"), postemerge,), level=logging.ERROR, noiselevel=-1) + clean_logs(settings) + if "--quiet" not in myopts and \ myaction is None and "@world" in myfiles: show_depclean_suggestion() @@ -428,6 +461,8 @@ def insert_optional_args(args): default_arg_opts = { '--ask' : y_or_n, '--autounmask' : y_or_n, + '--autounmask-keep-masks': y_or_n, + '--autounmask-unrestricted-atoms' : y_or_n, '--autounmask-write' : y_or_n, '--buildpkg' : y_or_n, '--complete-graph' : y_or_n, @@ -551,19 +586,25 @@ def insert_optional_args(args): return new_args -def _find_bad_atoms(atoms): +def _find_bad_atoms(atoms, less_strict=False): + """ + Declares all atoms as invalid that have an operator, + a use dependency, a blocker or a repo spec. + It accepts atoms with wildcards. + In less_strict mode it accepts operators and repo specs. + """ bad_atoms = [] for x in ' '.join(atoms).split(): bad_atom = False try: - atom = portage.dep.Atom(x, allow_wildcard=True) + atom = portage.dep.Atom(x, allow_wildcard=True, allow_repo=less_strict) except portage.exception.InvalidAtom: try: - atom = portage.dep.Atom("*/"+x, allow_wildcard=True) + atom = portage.dep.Atom("*/"+x, allow_wildcard=True, allow_repo=less_strict) except portage.exception.InvalidAtom: bad_atom = True - if bad_atom or atom.operator or atom.blocker or atom.use: + if bad_atom or (atom.operator and not less_strict) or atom.blocker or atom.use: bad_atoms.append(x) return bad_atoms @@ -573,16 +614,15 @@ def parse_opts(tmpcmdline, silent=False): myopts = {} myfiles=[] - global options, shortmapping - actions = frozenset([ - "clean", "config", "depclean", "help", - "info", "list-sets", "metadata", + "clean", "check-news", "config", "depclean", "help", + "info", "list-sets", "metadata", "moo", "prune", "regen", "search", "sync", "unmerge", "version", ]) longopt_aliases = {"--cols":"--columns", "--skip-first":"--skipfirst"} + y_or_n = ("y", "n") true_y_or_n = ("True", "y", "n") true_y = ("True", "y") argument_options = { @@ -600,6 +640,18 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n }, + "--autounmask-unrestricted-atoms": { + "help" : "write autounmask changes with >= atoms if possible", + "type" : "choice", + "choices" : true_y_or_n + }, + + "--autounmask-keep-masks": { + "help" : "don't add package.unmask entries", + "type" : "choice", + "choices" : true_y_or_n + }, + "--autounmask-write": { "help" : "write changes made by --autounmask to disk", "type" : "choice", @@ -626,6 +678,14 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n }, + "--buildpkg-exclude": { + "help" :"A space separated list of package atoms for which " + \ + "no binary packages should be built. This option overrides all " + \ + "possible ways to enable building of binary packages.", + + "action" : "append" + }, + "--config-root": { "help":"specify the location for portage configuration files", "action":"store" @@ -642,6 +702,12 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n }, + "--complete-graph-if-new-ver": { + "help" : "trigger --complete-graph behavior if an installed package version will change (upgrade or downgrade)", + "type" : "choice", + "choices" : y_or_n + }, + "--deep": { "shortopt" : "-D", @@ -660,6 +726,12 @@ def parse_opts(tmpcmdline, silent=False): "choices" : true_y_or_n }, + "--dynamic-deps": { + "help": "substitute the dependencies of installed packages with the dependencies of unbuilt ebuilds", + "type": "choice", + "choices": y_or_n + }, + "--exclude": { "help" :"A space separated list of package names or slot atoms. " + \ "Emerge won't install any ebuild or binary package that " + \ @@ -784,7 +856,7 @@ def parse_opts(tmpcmdline, silent=False): "--quiet-build": { "help" : "redirect build output to logs", "type" : "choice", - "choices" : true_y_or_n + "choices" : true_y_or_n, }, "--rebuild-if-new-rev": { @@ -923,13 +995,23 @@ def parse_opts(tmpcmdline, silent=False): if myoptions.autounmask in true_y: myoptions.autounmask = True + if myoptions.autounmask_unrestricted_atoms in true_y: + myoptions.autounmask_unrestricted_atoms = True + + if myoptions.autounmask_keep_masks in true_y: + myoptions.autounmask_keep_masks = True + if myoptions.autounmask_write in true_y: myoptions.autounmask_write = True if myoptions.buildpkg in true_y: myoptions.buildpkg = True - else: - myoptions.buildpkg = None + + if myoptions.buildpkg_exclude: + bad_atoms = _find_bad_atoms(myoptions.buildpkg_exclude, less_strict=True) + if bad_atoms and not silent: + parser.error("Invalid Atom(s) in --buildpkg-exclude parameter: '%s'\n" % \ + (",".join(bad_atoms),)) if myoptions.changed_use is not False: myoptions.reinstall = "changed-use" @@ -938,10 +1020,11 @@ def parse_opts(tmpcmdline, silent=False): if myoptions.deselect in true_y: myoptions.deselect = True - if myoptions.binpkg_respect_use in true_y: - myoptions.binpkg_respect_use = True - else: - myoptions.binpkg_respect_use = None + if myoptions.binpkg_respect_use is not None: + if myoptions.binpkg_respect_use in true_y: + myoptions.binpkg_respect_use = 'y' + else: + myoptions.binpkg_respect_use = 'n' if myoptions.complete_graph in true_y: myoptions.complete_graph = True @@ -1015,9 +1098,7 @@ def parse_opts(tmpcmdline, silent=False): myoptions.quiet = None if myoptions.quiet_build in true_y: - myoptions.quiet_build = True - else: - myoptions.quiet_build = None + myoptions.quiet_build = 'y' if myoptions.rebuild_if_new_ver in true_y: myoptions.rebuild_if_new_ver = True @@ -1172,8 +1253,7 @@ def parse_opts(tmpcmdline, silent=False): if myaction is None and myoptions.deselect is True: myaction = 'deselect' - if myargs and sys.hexversion < 0x3000000 and \ - not isinstance(myargs[0], unicode): + if myargs and isinstance(myargs[0], bytes): for i in range(len(myargs)): myargs[i] = portage._unicode_decode(myargs[i]) @@ -1222,7 +1302,6 @@ def ionice(settings): if not ionice_cmd: return - from portage.util import varexpand variables = {"PID" : str(os.getpid())} cmd = [varexpand(x, mydict=variables) for x in ionice_cmd] @@ -1238,6 +1317,35 @@ def ionice(settings): out.eerror("PORTAGE_IONICE_COMMAND returned %d" % (rval,)) out.eerror("See the make.conf(5) man page for PORTAGE_IONICE_COMMAND usage instructions.") +def clean_logs(settings): + + if "clean-logs" not in settings.features: + return + + clean_cmd = settings.get("PORT_LOGDIR_CLEAN") + if clean_cmd: + clean_cmd = shlex_split(clean_cmd) + if not clean_cmd: + return + + logdir = settings.get("PORT_LOGDIR") + if logdir is None or not os.path.isdir(logdir): + return + + variables = {"PORT_LOGDIR" : logdir} + cmd = [varexpand(x, mydict=variables) for x in clean_cmd] + + try: + rval = portage.process.spawn(cmd, env=os.environ) + except portage.exception.CommandNotFound: + rval = 127 + + if rval != os.EX_OK: + out = portage.output.EOutput() + out.eerror("PORT_LOGDIR_CLEAN returned %d" % (rval,)) + out.eerror("See the make.conf(5) man page for " + "PORT_LOGDIR_CLEAN usage instructions.") + def setconfig_fallback(root_config): from portage._sets.base import DummyPackageSet from portage._sets.files import WorldSelectedSet @@ -1451,25 +1559,26 @@ def repo_name_duplicate_check(trees): def config_protect_check(trees): for root, root_trees in trees.items(): - if not root_trees["root_config"].settings.get("CONFIG_PROTECT"): + settings = root_trees["root_config"].settings + if not settings.get("CONFIG_PROTECT"): msg = "!!! CONFIG_PROTECT is empty" - if root != "/": + if settings["ROOT"] != "/": msg += " for '%s'" % root msg += "\n" writemsg_level(msg, level=logging.WARN, noiselevel=-1) def profile_check(trees, myaction): - if myaction in ("help", "info", "sync", "version"): + if myaction in ("help", "info", "search", "sync", "version"): return os.EX_OK - for root, root_trees in trees.items(): + for root_trees in trees.values(): if root_trees["root_config"].settings.profiles: continue # generate some profile related warning messages validate_ebuild_environment(trees) - msg = "If you have just changed your profile configuration, you " + \ - "should revert back to the previous configuration. Due to " + \ - "your current profile being invalid, allowed actions are " + \ - "limited to --help, --info, --sync, and --version." + msg = ("Your current profile is invalid. If you have just changed " + "your profile configuration, you should revert back to the " + "previous configuration. Allowed actions are limited to " + "--help, --info, --search, --sync, and --version.") writemsg_level("".join("!!! %s\n" % l for l in textwrap.wrap(msg, 70)), level=logging.ERROR, noiselevel=-1) return 1 @@ -1515,7 +1624,7 @@ def emerge_main(args=None): # Portage needs to ensure a sane umask for the files it creates. os.umask(0o22) settings, trees, mtimedb = load_emerge_config() - portdb = trees[settings["ROOT"]]["porttree"].dbapi + portdb = trees[settings['EROOT']]['porttree'].dbapi rval = profile_check(trees, myaction) if rval != os.EX_OK: return rval @@ -1526,13 +1635,14 @@ def emerge_main(args=None): tmpcmdline.extend(args) myaction, myopts, myfiles = parse_opts(tmpcmdline) - if myaction not in ('help', 'info', 'version') and \ + # skip global updates prior to sync, since it's called after sync + if myaction not in ('help', 'info', 'sync', 'version') and \ myopts.get('--package-moves') != 'n' and \ _global_updates(trees, mtimedb["updates"], quiet=("--quiet" in myopts)): mtimedb.commit() # Reload the whole config from scratch. settings, trees, mtimedb = load_emerge_config(trees=trees) - portdb = trees[settings["ROOT"]]["porttree"].dbapi + portdb = trees[settings['EROOT']]['porttree'].dbapi xterm_titles = "notitles" not in settings.features if xterm_titles: @@ -1543,19 +1653,24 @@ def emerge_main(args=None): # Reload the whole config from scratch so that the portdbapi internal # config is updated with new FEATURES. settings, trees, mtimedb = load_emerge_config(trees=trees) - portdb = trees[settings["ROOT"]]["porttree"].dbapi + portdb = trees[settings['EROOT']]['porttree'].dbapi + + # NOTE: adjust_configs() can map options to FEATURES, so any relevant + # options adjustments should be made prior to calling adjust_configs(). + if "--buildpkgonly" in myopts: + myopts["--buildpkg"] = True adjust_configs(myopts, trees) apply_priorities(settings) if myaction == 'version': writemsg_stdout(getportageversion( - settings["PORTDIR"], settings["ROOT"], + settings["PORTDIR"], None, settings.profile_path, settings["CHOST"], - trees[settings["ROOT"]]["vartree"].dbapi) + '\n', noiselevel=-1) + trees[settings['EROOT']]['vartree'].dbapi) + '\n', noiselevel=-1) return 0 elif myaction == 'help': - _emerge.help.help(myopts, portage.output.havecolor) + _emerge.help.help() return 0 spinner = stdout_spinner() @@ -1587,9 +1702,6 @@ def emerge_main(args=None): if "--usepkgonly" in myopts: myopts["--usepkg"] = True - if "buildpkg" in settings.features or "--buildpkgonly" in myopts: - myopts["--buildpkg"] = True - if "--buildpkgonly" in myopts: # --buildpkgonly will not merge anything, so # it cancels all binary package options. @@ -1613,20 +1725,11 @@ def emerge_main(args=None): del mytrees, mydb if "moo" in myfiles: - print(""" - - Larry loves Gentoo (""" + platform.system() + """) - - _______________________ -< Have you mooed today? > - ----------------------- - \ ^__^ - \ (oo)\_______ - (__)\ )\/\ - ||----w | - || || - -""") + print(COWSAY_MOO % platform.system()) + msg = ("The above `emerge moo` display is deprecated. " + "Please use `emerge --moo` instead.") + for line in textwrap.wrap(msg, 50): + print(" %s %s" % (colorize("WARN", "*"), line)) for x in myfiles: ext = os.path.splitext(x)[1] @@ -1634,10 +1737,22 @@ def emerge_main(args=None): print(colorize("BAD", "\n*** emerging by path is broken and may not always work!!!\n")) break - root_config = trees[settings["ROOT"]]["root_config"] - if myaction == "list-sets": + root_config = trees[settings['EROOT']]['root_config'] + if myaction == "moo": + print(COWSAY_MOO % platform.system()) + return os.EX_OK + elif myaction == "list-sets": writemsg_stdout("".join("%s\n" % s for s in sorted(root_config.sets))) return os.EX_OK + elif myaction == "check-news": + news_counts = count_unread_news( + root_config.trees["porttree"].dbapi, + root_config.trees["vartree"].dbapi) + if any(news_counts.values()): + display_news_notifications(news_counts) + elif "--quiet" not in myopts: + print("", colorize("GOOD", "*"), "No news items were found.") + return os.EX_OK ensure_required_sets(trees) @@ -1703,7 +1818,7 @@ def emerge_main(args=None): print("myopts", myopts) if not myaction and not myfiles and "--resume" not in myopts: - _emerge.help.help(myopts, portage.output.havecolor) + _emerge.help.help() return 1 pretend = "--pretend" in myopts @@ -1735,7 +1850,7 @@ def emerge_main(args=None): portage_group_warning() if userquery("Would you like to add --pretend to options?", "--ask-enter-invalid" in myopts) == "No": - return 1 + return 128 + signal.SIGINT myopts["--pretend"] = True del myopts["--ask"] else: @@ -1753,7 +1868,11 @@ def emerge_main(args=None): if x in myopts: disable_emergelog = True break - if myaction in ("search", "info"): + if disable_emergelog: + pass + elif myaction in ("search", "info"): + disable_emergelog = True + elif portage.data.secpass < 1: disable_emergelog = True _emerge.emergelog._disable = disable_emergelog @@ -1768,8 +1887,13 @@ def emerge_main(args=None): "EMERGE_LOG_DIR='%s':\n!!! %s\n" % \ (settings['EMERGE_LOG_DIR'], e), noiselevel=-1, level=logging.ERROR) + portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) else: _emerge.emergelog._emerge_log_dir = settings["EMERGE_LOG_DIR"] + else: + _emerge.emergelog._emerge_log_dir = os.path.join(os.sep, + settings["EPREFIX"].lstrip(os.sep), "var", "log") + portage.util.ensure_dirs(_emerge.emergelog._emerge_log_dir) if not "--pretend" in myopts: emergelog(xterm_titles, "Started emerge on: "+\ @@ -1778,9 +1902,19 @@ def emerge_main(args=None): encoding=_encodings['content'], errors='replace')) myelogstr="" if myopts: - myelogstr=" ".join(myopts) + opt_list = [] + for opt, arg in myopts.items(): + if arg is True: + opt_list.append(opt) + elif isinstance(arg, list): + # arguments like --exclude that use 'append' action + for x in arg: + opt_list.append("%s=%s" % (opt, x)) + else: + opt_list.append("%s=%s" % (opt, arg)) + myelogstr=" ".join(opt_list) if myaction: - myelogstr+=" "+myaction + myelogstr += " --" + myaction if myfiles: myelogstr += " " + " ".join(oldargs) emergelog(xterm_titles, " *** emerge " + myelogstr) @@ -1824,7 +1958,7 @@ def emerge_main(args=None): # SEARCH action elif "search"==myaction: validate_ebuild_environment(trees) - action_search(trees[settings["ROOT"]]["root_config"], + action_search(trees[settings['EROOT']]['root_config'], myopts, myfiles, spinner) elif myaction in ('clean', 'depclean', 'deselect', 'prune', 'unmerge'): @@ -1832,19 +1966,19 @@ def emerge_main(args=None): rval = action_uninstall(settings, trees, mtimedb["ldpath"], myopts, myaction, myfiles, spinner) if not (myaction == 'deselect' or buildpkgonly or fetchonly or pretend): - post_emerge(myaction, myopts, myfiles, settings["ROOT"], + post_emerge(myaction, myopts, myfiles, settings['EROOT'], trees, mtimedb, rval) return rval elif myaction == 'info': # Ensure atoms are valid before calling unmerge(). - vardb = trees[settings["ROOT"]]["vartree"].dbapi - portdb = trees[settings["ROOT"]]["porttree"].dbapi - bindb = trees[settings["ROOT"]]["bintree"].dbapi + vardb = trees[settings['EROOT']]['vartree'].dbapi + portdb = trees[settings['EROOT']]['porttree'].dbapi + bindb = trees[settings['EROOT']]["bintree"].dbapi valid_atoms = [] for x in myfiles: - if is_valid_package_atom(x): + if is_valid_package_atom(x, allow_repo=True): try: #look at the installed files first, if there is no match #look at the ebuilds, since EAPI 4 allows running pkg_info @@ -1900,11 +2034,12 @@ def emerge_main(args=None): level=logging.ERROR, noiselevel=-1) return 1 + # GLEP 42 says to display news *after* an emerge --pretend if "--pretend" not in myopts: display_news_notification(root_config, myopts) retval = action_build(settings, trees, mtimedb, myopts, myaction, myfiles, spinner) - post_emerge(myaction, myopts, myfiles, settings["ROOT"], + post_emerge(myaction, myopts, myfiles, settings['EROOT'], trees, mtimedb, retval) return retval diff --git a/portage_with_autodep/pym/_emerge/main.pyo b/portage_with_autodep/pym/_emerge/main.pyo Binary files differnew file mode 100644 index 0000000..aaeb5b9 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/main.pyo diff --git a/portage_with_autodep/pym/_emerge/post_emerge.py b/portage_with_autodep/pym/_emerge/post_emerge.py new file mode 100644 index 0000000..d5f1ba5 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/post_emerge.py @@ -0,0 +1,165 @@ +# Copyright 1999-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + +import logging +import textwrap + +import portage +from portage import os +from portage.emaint.modules.logs.logs import CleanLogs +from portage.news import count_unread_news, display_news_notifications +from portage.output import colorize +from portage.util._dyn_libs.display_preserved_libs import \ + display_preserved_libs +from portage.util._info_files import chk_updated_info_files + +from .chk_updated_cfg_files import chk_updated_cfg_files +from .emergelog import emergelog +from ._flush_elog_mod_echo import _flush_elog_mod_echo + +def clean_logs(settings): + + if "clean-logs" not in settings.features: + return + + logdir = settings.get("PORT_LOGDIR") + if logdir is None or not os.path.isdir(logdir): + return + + cleanlogs = CleanLogs() + errors = cleanlogs.clean(settings=settings) + if errors: + out = portage.output.EOutput() + for msg in errors: + out.eerror(msg) + +def display_news_notification(root_config, myopts): + if "news" not in root_config.settings.features: + return + portdb = root_config.trees["porttree"].dbapi + vardb = root_config.trees["vartree"].dbapi + news_counts = count_unread_news(portdb, vardb) + display_news_notifications(news_counts) + +def show_depclean_suggestion(): + out = portage.output.EOutput() + msg = "After world updates, it is important to remove " + \ + "obsolete packages with emerge --depclean. Refer " + \ + "to `man emerge` for more information." + for line in textwrap.wrap(msg, 72): + out.ewarn(line) + +def post_emerge(myaction, myopts, myfiles, + target_root, trees, mtimedb, retval): + """ + Misc. things to run at the end of a merge session. + + Update Info Files + Update Config Files + Update News Items + Commit mtimeDB + Display preserved libs warnings + + @param myaction: The action returned from parse_opts() + @type myaction: String + @param myopts: emerge options + @type myopts: dict + @param myfiles: emerge arguments + @type myfiles: list + @param target_root: The target EROOT for myaction + @type target_root: String + @param trees: A dictionary mapping each ROOT to it's package databases + @type trees: dict + @param mtimedb: The mtimeDB to store data needed across merge invocations + @type mtimedb: MtimeDB class instance + @param retval: Emerge's return value + @type retval: Int + """ + + root_config = trees[target_root]["root_config"] + vardbapi = trees[target_root]['vartree'].dbapi + settings = vardbapi.settings + info_mtimes = mtimedb["info"] + + # Load the most current variables from ${ROOT}/etc/profile.env + settings.unlock() + settings.reload() + settings.regenerate() + settings.lock() + + config_protect = portage.util.shlex_split( + settings.get("CONFIG_PROTECT", "")) + infodirs = settings.get("INFOPATH","").split(":") + \ + settings.get("INFODIR","").split(":") + + os.chdir("/") + + if retval == os.EX_OK: + exit_msg = " *** exiting successfully." + else: + exit_msg = " *** exiting unsuccessfully with status '%s'." % retval + emergelog("notitles" not in settings.features, exit_msg) + + _flush_elog_mod_echo() + + if not vardbapi._pkgs_changed: + # GLEP 42 says to display news *after* an emerge --pretend + if "--pretend" in myopts: + display_news_notification(root_config, myopts) + # If vdb state has not changed then there's nothing else to do. + return + + vdb_path = os.path.join(root_config.settings['EROOT'], portage.VDB_PATH) + portage.util.ensure_dirs(vdb_path) + vdb_lock = None + if os.access(vdb_path, os.W_OK) and not "--pretend" in myopts: + vardbapi.lock() + vdb_lock = True + + if vdb_lock: + try: + if "noinfo" not in settings.features: + chk_updated_info_files(target_root, + infodirs, info_mtimes) + mtimedb.commit() + finally: + if vdb_lock: + vardbapi.unlock() + + # Explicitly load and prune the PreservedLibsRegistry in order + # to ensure that we do not display stale data. + vardbapi._plib_registry.load() + + if vardbapi._plib_registry.hasEntries(): + if "--quiet" in myopts: + print() + print(colorize("WARN", "!!!") + " existing preserved libs found") + else: + print() + print(colorize("WARN", "!!!") + " existing preserved libs:") + display_preserved_libs(vardbapi) + print("Use " + colorize("GOOD", "emerge @preserved-rebuild") + + " to rebuild packages using these libraries") + + chk_updated_cfg_files(settings['EROOT'], config_protect) + + display_news_notification(root_config, myopts) + + postemerge = os.path.join(settings["PORTAGE_CONFIGROOT"], + portage.USER_CONFIG_PATH, "bin", "post_emerge") + if os.access(postemerge, os.X_OK): + hook_retval = portage.process.spawn( + [postemerge], env=settings.environ()) + if hook_retval != os.EX_OK: + portage.util.writemsg_level( + " %s spawn failed of %s\n" % + (colorize("BAD", "*"), postemerge,), + level=logging.ERROR, noiselevel=-1) + + clean_logs(settings) + + if "--quiet" not in myopts and \ + myaction is None and "@world" in myfiles: + show_depclean_suggestion() diff --git a/portage_with_autodep/pym/_emerge/resolver/__init__.pyo b/portage_with_autodep/pym/_emerge/resolver/__init__.pyo Binary files differnew file mode 100644 index 0000000..5c1b374 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/resolver/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/backtracking.py b/portage_with_autodep/pym/_emerge/resolver/backtracking.py index dcdaee0..f2857b0 100644 --- a/portage_with_autodep/pym/_emerge/resolver/backtracking.py +++ b/portage_with_autodep/pym/_emerge/resolver/backtracking.py @@ -47,7 +47,7 @@ class BacktrackParameter(object): self.reinstall_list == other.reinstall_list -class _BacktrackNode: +class _BacktrackNode(object): __slots__ = ( "parameter", "depth", "mask_steps", "terminal", @@ -84,6 +84,9 @@ class Backtracker(object): Adds a newly computed backtrack parameter. Makes sure that it doesn't already exist and that we don't backtrack deeper than we are allowed by --backtrack. """ + if not self._check_runtime_pkg_mask(node.parameter.runtime_pkg_mask): + return + if node.mask_steps <= self._max_depth and node not in self._nodes: if explore: self._unexplored_nodes.append(node) @@ -105,6 +108,28 @@ class Backtracker(object): def __len__(self): return len(self._unexplored_nodes) + def _check_runtime_pkg_mask(self, runtime_pkg_mask): + """ + If a package gets masked that caused other packages to be masked + before, we revert the mask for other packages (bug 375573). + """ + + for pkg in runtime_pkg_mask: + + if "missing dependency" in runtime_pkg_mask[pkg]: + continue + + entry_is_valid = False + + for ppkg, patom in runtime_pkg_mask[pkg].get("slot conflict", set()): + if ppkg not in runtime_pkg_mask: + entry_is_valid = True + break + + if not entry_is_valid: + return False + + return True def _feedback_slot_conflict(self, conflict_data): for pkg, parent_atoms in conflict_data: diff --git a/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo b/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo Binary files differnew file mode 100644 index 0000000..d989c15 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/resolver/backtracking.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py index d113c5e..aca81fa 100644 --- a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py +++ b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.py @@ -143,7 +143,8 @@ class circular_dependency_handler(object): #If any of the flags we're going to touch is in REQUIRED_USE, add all #other flags in REQUIRED_USE to affecting_use, to not lose any solution. - required_use_flags = get_required_use_flags(parent.metadata["REQUIRED_USE"]) + required_use_flags = get_required_use_flags( + parent.metadata.get("REQUIRED_USE", "")) if affecting_use.intersection(required_use_flags): # TODO: Find out exactly which REQUIRED_USE flags are @@ -185,7 +186,7 @@ class circular_dependency_handler(object): parent_atom not in reduced_dep: #We found an assignment that removes the atom from 'dep'. #Make sure it doesn't conflict with REQUIRED_USE. - required_use = parent.metadata["REQUIRED_USE"] + required_use = parent.metadata.get("REQUIRED_USE", "") if check_required_use(required_use, current_use, parent.iuse.is_valid_flag): use = self.depgraph._pkg_use_enabled(parent) diff --git a/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo Binary files differnew file mode 100644 index 0000000..c1f95dc --- /dev/null +++ b/portage_with_autodep/pym/_emerge/resolver/circular_dependency.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/output.py b/portage_with_autodep/pym/_emerge/resolver/output.py index 05e316a..1208bf9 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output.py +++ b/portage_with_autodep/pym/_emerge/resolver/output.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Resolver output display operation. @@ -13,14 +13,14 @@ import sys from portage import os from portage import _unicode_decode from portage.dbapi.dep_expand import dep_expand -from portage.const import PORTAGE_PACKAGE_ATOM -from portage.dep import cpvequal, match_from_list -from portage.exception import InvalidDependString -from portage.output import ( blue, bold, colorize, create_color_func, +from portage.dep import cpvequal, _repo_separator +from portage.exception import InvalidDependString, SignatureException +from portage.package.ebuild._spawn_nofetch import spawn_nofetch +from portage.output import ( blue, colorize, create_color_func, darkblue, darkgreen, green, nc_len, red, teal, turquoise, yellow ) bad = create_color_func("BAD") -from portage.util import writemsg_stdout, writemsg_level -from portage.versions import best, catpkgsplit, cpv_getkey +from portage.util import writemsg_stdout +from portage.versions import best, catpkgsplit from _emerge.Blocker import Blocker from _emerge.create_world_atom import create_world_atom @@ -72,7 +72,7 @@ class Display(object): """Processes pkg for blockers and adds colorized strings to self.print_msg and self.blockers - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @param fetch_symbol: string @rtype: bool Modifies class globals: self.blocker_style, self.resolved, @@ -121,7 +121,7 @@ class Display(object): def _display_use(self, pkg, myoldbest, myinslotlist): """ USE flag display - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @param myoldbest: list of installed versions @param myinslotlist: list of installed slots Modifies class globals: self.forced_flags, self.cur_iuse, @@ -161,7 +161,7 @@ class Display(object): def gen_mask_str(self, pkg): """ - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance """ hardmasked = pkg.isHardMasked() mask_str = " " @@ -223,7 +223,7 @@ class Display(object): """ Prevent USE_EXPAND_HIDDEN flags from being hidden if they are the only thing that triggered reinstallation. - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance Modifies self.use_expand_hidden, self.use_expand, self.verboseadd """ reinst_flags_map = {} @@ -302,68 +302,78 @@ class Display(object): def verbose_size(self, pkg, repoadd_set, pkg_info): """Determines the size of the downloads required - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @param repoadd_set: set of repos to add @param pkg_info: dictionary Modifies class globals: self.myfetchlist, self.counters.totalsize, self.verboseadd, repoadd_set. """ mysize = 0 - if pkg.type_name == "ebuild" and pkg_info.merge: + if pkg.type_name in ("binary", "ebuild") and pkg_info.merge: + db = pkg.root_config.trees[ + pkg.root_config.pkg_tree_map[pkg.type_name]].dbapi + kwargs = {} + if pkg.type_name == "ebuild": + kwargs["useflags"] = pkg_info.use + kwargs["myrepo"] = pkg.repo + myfilesdict = None try: - myfilesdict = self.portdb.getfetchsizes(pkg.cpv, - useflags=pkg_info.use, myrepo=pkg.repo) + myfilesdict = db.getfetchsizes(pkg.cpv, **kwargs) except InvalidDependString as e: # FIXME: validate SRC_URI earlier - depstr, = self.portdb.aux_get(pkg.cpv, + depstr, = db.aux_get(pkg.cpv, ["SRC_URI"], myrepo=pkg.repo) show_invalid_depstring_notice( pkg, depstr, str(e)) raise + except SignatureException: + # missing/invalid binary package SIZE signature + pass if myfilesdict is None: myfilesdict = "[empty/missing/bad digest]" else: for myfetchfile in myfilesdict: if myfetchfile not in self.myfetchlist: mysize += myfilesdict[myfetchfile] - self.myfetchlist.append(myfetchfile) + self.myfetchlist.add(myfetchfile) if pkg_info.ordered: self.counters.totalsize += mysize self.verboseadd += _format_size(mysize) - # overlay verbose - # assign index for a previous version in the same slot - slot_matches = self.vardb.match(pkg.slot_atom) - if slot_matches: - repo_name_prev = self.vardb.aux_get(slot_matches[0], - ["repository"])[0] - else: - repo_name_prev = None + if self.quiet_repo_display: + # overlay verbose + # assign index for a previous version in the same slot + slot_matches = self.vardb.match(pkg.slot_atom) + if slot_matches: + repo_name_prev = self.vardb.aux_get(slot_matches[0], + ["repository"])[0] + else: + repo_name_prev = None - # now use the data to generate output - if pkg.installed or not slot_matches: - self.repoadd = self.conf.repo_display.repoStr( - pkg_info.repo_path_real) - else: - repo_path_prev = None - if repo_name_prev: - repo_path_prev = self.portdb.getRepositoryPath( - repo_name_prev) - if repo_path_prev == pkg_info.repo_path_real: + # now use the data to generate output + if pkg.installed or not slot_matches: self.repoadd = self.conf.repo_display.repoStr( pkg_info.repo_path_real) else: - self.repoadd = "%s=>%s" % ( - self.conf.repo_display.repoStr(repo_path_prev), - self.conf.repo_display.repoStr(pkg_info.repo_path_real)) - if self.repoadd: - repoadd_set.add(self.repoadd) + repo_path_prev = None + if repo_name_prev: + repo_path_prev = self.portdb.getRepositoryPath( + repo_name_prev) + if repo_path_prev == pkg_info.repo_path_real: + self.repoadd = self.conf.repo_display.repoStr( + pkg_info.repo_path_real) + else: + self.repoadd = "%s=>%s" % ( + self.conf.repo_display.repoStr(repo_path_prev), + self.conf.repo_display.repoStr(pkg_info.repo_path_real)) + if self.repoadd: + repoadd_set.add(self.repoadd) - @staticmethod - def convert_myoldbest(myoldbest): + def convert_myoldbest(self, pkg, myoldbest): """converts and colorizes a version list to a string + @param pkg: _emerge.Package.Package instance @param myoldbest: list @rtype string. """ @@ -371,11 +381,13 @@ class Display(object): myoldbest_str = "" if myoldbest: versions = [] - for pos, pkg in enumerate(myoldbest): - key = catpkgsplit(pkg.cpv)[2] + \ - "-" + catpkgsplit(pkg.cpv)[3] + for pos, old_pkg in enumerate(myoldbest): + key = catpkgsplit(old_pkg.cpv)[2] + "-" + catpkgsplit(old_pkg.cpv)[3] if key[-3:] == "-r0": key = key[:-3] + if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or + any(x.repo != self.portdb.repositories.mainRepo().name for x in myoldbest + [pkg])): + key += _repo_separator + old_pkg.repo versions.append(key) myoldbest_str = blue("["+", ".join(versions)+"]") return myoldbest_str @@ -385,7 +397,7 @@ class Display(object): """Increments counters.interactive if the pkg is to be merged and it's metadata has interactive set True - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @param ordered: boolean @param addl: already defined string to add to """ @@ -401,13 +413,17 @@ class Display(object): @param addl: already defined string to add to @param pkg_info: dictionary - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @rtype string """ + ver_str = pkg_info.ver + if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or + any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): + ver_str += _repo_separator + pkg.repo if self.conf.quiet: myprint = addl + " " + self.indent + \ self.pkgprint(pkg_info.cp, pkg_info) - myprint = myprint+darkblue(" "+pkg_info.ver)+" " + myprint = myprint+darkblue(" "+ver_str)+" " myprint = myprint+pkg_info.oldbest myprint = myprint+darkgreen("to "+pkg.root) self.verboseadd = None @@ -422,7 +438,7 @@ class Display(object): self.indent, self.pkgprint(pkg.cp, pkg_info)) if (self.newlp-nc_len(myprint)) > 0: myprint = myprint+(" "*(self.newlp-nc_len(myprint))) - myprint = myprint+"["+darkblue(pkg_info.ver)+"] " + myprint = myprint+" "+darkblue("["+ver_str+"]")+" " if (self.oldlp-nc_len(myprint)) > 0: myprint = myprint+" "*(self.oldlp-nc_len(myprint)) myprint = myprint+pkg_info.oldbest @@ -435,14 +451,18 @@ class Display(object): @param addl: already defined string to add to @param pkg_info: dictionary - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @rtype string Modifies self.verboseadd """ + ver_str = pkg_info.ver + if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or + any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): + ver_str += _repo_separator + pkg.repo if self.conf.quiet: myprint = addl + " " + self.indent + \ self.pkgprint(pkg_info.cp, pkg_info) - myprint = myprint+" "+green(pkg_info.ver)+" " + myprint = myprint+" "+green(ver_str)+" " myprint = myprint+pkg_info.oldbest self.verboseadd = None else: @@ -457,7 +477,7 @@ class Display(object): self.indent, self.pkgprint(pkg.cp, pkg_info)) if (self.newlp-nc_len(myprint)) > 0: myprint = myprint+(" "*(self.newlp-nc_len(myprint))) - myprint = myprint+green(" ["+pkg_info.ver+"] ") + myprint = myprint+" "+green("["+ver_str+"]")+" " if (self.oldlp-nc_len(myprint)) > 0: myprint = myprint+(" "*(self.oldlp-nc_len(myprint))) myprint += pkg_info.oldbest @@ -467,31 +487,35 @@ class Display(object): def _set_no_columns(self, pkg, pkg_info, addl): """prints pkg info without column indentation. - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @param pkg_info: dictionary @param addl: the current text to add for the next line to output @rtype the updated addl """ + pkg_str = pkg.cpv + if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or + any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): + pkg_str += _repo_separator + pkg.repo if not pkg_info.merge: addl = self.empty_space_in_brackets() myprint = "[%s%s] %s%s %s" % \ (self.pkgprint(pkg_info.operation.ljust(13), pkg_info), addl, - self.indent, self.pkgprint(pkg.cpv, pkg_info), + self.indent, self.pkgprint(pkg_str, pkg_info), pkg_info.oldbest) else: myprint = "[%s %s] %s%s %s" % \ (self.pkgprint(pkg.type_name, pkg_info), addl, self.indent, - self.pkgprint(pkg.cpv, pkg_info), pkg_info.oldbest) + self.pkgprint(pkg_str, pkg_info), pkg_info.oldbest) return myprint def _insert_slot(self, pkg, pkg_info, myinslotlist): """Adds slot info to the message - @returns addl: formatted slot info - @returns myoldbest: installed version list + @return addl: formatted slot info + @return myoldbest: installed version list Modifies self.counters.downgrades, self.counters.upgrades, self.counters.binary """ @@ -517,8 +541,8 @@ class Display(object): def _new_slot(self, pkg, pkg_info): """New slot, mark it new. - @returns addl: formatted slot info - @returns myoldbest: installed version list + @return addl: formatted slot info + @return myoldbest: installed version list Modifies self.counters.newslot, self.counters.binary """ addl = " " + green("NS") + pkg_info.fetch_symbol + " " @@ -574,11 +598,9 @@ class Display(object): def print_changelog(self): """Prints the changelog text to std_out """ - writemsg_stdout('\n', noiselevel=-1) - for revision, text in self.changelogs: - writemsg_stdout(bold('*'+revision) + '\n' + text, + for chunk in self.changelogs: + writemsg_stdout(chunk, noiselevel=-1) - return def get_display_list(self, mylist): @@ -613,7 +635,7 @@ class Display(object): def set_pkg_info(self, pkg, ordered): """Sets various pkg_info dictionary variables - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @param ordered: bool @rtype pkg_info dictionary Modifies self.counters.restrict_fetch, @@ -643,7 +665,6 @@ class Display(object): pkg_info.use = list(self.conf.pkg_use_enabled(pkg)) if not pkg.built and pkg.operation == 'merge' and \ 'fetch' in pkg.metadata.restrict: - pkg_info.fetch_symbol = red("F") if pkg_info.ordered: self.counters.restrict_fetch += 1 if not self.portdb.getfetchsizes(pkg.cpv, @@ -651,13 +672,17 @@ class Display(object): pkg_info.fetch_symbol = green("f") if pkg_info.ordered: self.counters.restrict_fetch_satisfied += 1 + else: + pkg_info.fetch_symbol = red("F") + if pkg_info.ebuild_path is not None: + self.restrict_fetch_list[pkg] = pkg_info return pkg_info def do_changelog(self, pkg, pkg_info): """Processes and adds the changelog text to the master text for output - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @param pkg_info: dictionay Modifies self.changelogs """ @@ -676,7 +701,7 @@ class Display(object): def check_system_world(self, pkg): """Checks for any occurances of the package in the system or world sets - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @rtype system and world booleans """ root_config = self.conf.roots[pkg.root] @@ -706,7 +731,7 @@ class Display(object): @staticmethod def get_ver_str(pkg): """Obtains the version string - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @rtype string """ ver_str = list(catpkgsplit(pkg.cpv)[2:]) @@ -723,7 +748,7 @@ class Display(object): param is used for -u, where you still *do* want to see when something is being upgraded. - @param pkg: _emerge.Package instance + @param pkg: _emerge.Package.Package instance @param pkg_info: dictionay @rtype addl, myoldbest: list, myinslotlist: list Modifies self.counters.reinst, self.counters.binary, self.counters.new @@ -734,6 +759,9 @@ class Display(object): installed_versions = self.vardb.match_pkgs(pkg.cp) if self.vardb.cpv_exists(pkg.cpv): addl = " "+yellow("R")+pkg_info.fetch_symbol+" " + installed_version = self.vardb.match_pkgs(pkg.cpv)[0] + if not self.quiet_repo_display and installed_version.repo != pkg.repo: + myoldbest = [installed_version] if pkg_info.ordered: if pkg_info.merge: self.counters.reinst += 1 @@ -784,10 +812,16 @@ class Display(object): mylist = self.get_display_list(self.conf.mylist) # files to fetch list - avoids counting a same file twice # in size display (verbose mode) - self.myfetchlist = [] - # Use this set to detect when all the "repoadd" strings are "[0]" - # and disable the entire repo display in this case. - repoadd_set = set() + self.myfetchlist = set() + + self.quiet_repo_display = "--quiet-repo-display" in depgraph._frozen_config.myopts + if self.quiet_repo_display: + # Use this set to detect when all the "repoadd" strings are "[0]" + # and disable the entire repo display in this case. + repoadd_set = set() + + self.verbose_main_repo_display = "--verbose-main-repo-display" in depgraph._frozen_config.myopts + self.restrict_fetch_list = {} for mylist_index in range(len(mylist)): pkg, depth, ordered = mylist[mylist_index] @@ -801,21 +835,25 @@ class Display(object): continue else: pkg_info = self.set_pkg_info(pkg, ordered) - addl, pkg_info.oldbest, myinslotlist = \ + addl, pkg_info.oldbest_list, myinslotlist = \ self._get_installed_best(pkg, pkg_info) self.verboseadd = "" - self.repoadd = None - self._display_use(pkg, pkg_info.oldbest, myinslotlist) + if self.quiet_repo_display: + self.repoadd = None + self._display_use(pkg, pkg_info.oldbest_list, myinslotlist) self.recheck_hidden(pkg) if self.conf.verbosity == 3: - self.verbose_size(pkg, repoadd_set, pkg_info) + if self.quiet_repo_display: + self.verbose_size(pkg, repoadd_set, pkg_info) + else: + self.verbose_size(pkg, None, pkg_info) pkg_info.cp = pkg.cp pkg_info.ver = self.get_ver_str(pkg) self.oldlp = self.conf.columnwidth - 30 self.newlp = self.oldlp - 30 - pkg_info.oldbest = self.convert_myoldbest(pkg_info.oldbest) + pkg_info.oldbest = self.convert_myoldbest(pkg, pkg_info.oldbest_list) pkg_info.system, pkg_info.world = \ self.check_system_world(pkg) addl = self.set_interactive(pkg, pkg_info.ordered, addl) @@ -823,13 +861,17 @@ class Display(object): if self.include_mask_str(): addl += self.gen_mask_str(pkg) - if pkg.root != "/": + if pkg.root_config.settings["ROOT"] != "/": if pkg_info.oldbest: pkg_info.oldbest += " " if self.conf.columns: myprint = self._set_non_root_columns( addl, pkg_info, pkg) else: + pkg_str = pkg.cpv + if self.conf.verbosity == 3 and not self.quiet_repo_display and (self.verbose_main_repo_display or + any(x.repo != self.portdb.repositories.mainRepo().name for x in pkg_info.oldbest_list + [pkg])): + pkg_str += _repo_separator + pkg.repo if not pkg_info.merge: addl = self.empty_space_in_brackets() myprint = "[%s%s] " % ( @@ -840,7 +882,7 @@ class Display(object): myprint = "[%s %s] " % ( self.pkgprint(pkg.type_name, pkg_info), addl) myprint += self.indent + \ - self.pkgprint(pkg.cpv, pkg_info) + " " + \ + self.pkgprint(pkg_str, pkg_info) + " " + \ pkg_info.oldbest + darkgreen("to " + pkg.root) else: if self.conf.columns: @@ -852,36 +894,23 @@ class Display(object): if self.conf.columns and pkg.operation == "uninstall": continue - self.print_msg.append((myprint, self.verboseadd, self.repoadd)) - - if not self.conf.tree_display \ - and not self.conf.no_restart \ - and pkg.root == self.conf.running_root.root \ - and match_from_list(PORTAGE_PACKAGE_ATOM, [pkg]) \ - and not self.conf.quiet: - - if not self.vardb.cpv_exists(pkg.cpv) or \ - '9999' in pkg.cpv or \ - 'git' in pkg.inherited or \ - 'git-2' in pkg.inherited: - if mylist_index < len(mylist) - 1: - self.print_msg.append( - colorize( - "WARN", "*** Portage will stop merging " - "at this point and reload itself," - ) - ) - self.print_msg.append( - colorize("WARN", " then resume the merge.") - ) + if self.quiet_repo_display: + self.print_msg.append((myprint, self.verboseadd, self.repoadd)) + else: + self.print_msg.append((myprint, self.verboseadd, None)) - show_repos = repoadd_set and repoadd_set != set(["0"]) + show_repos = self.quiet_repo_display and repoadd_set and repoadd_set != set(["0"]) # now finally print out the messages self.print_messages(show_repos) self.print_blockers() if self.conf.verbosity == 3: self.print_verbose(show_repos) + for pkg, pkg_info in self.restrict_fetch_list.items(): + writemsg_stdout("\nFetch instructions for %s:\n" % (pkg.cpv,), + noiselevel=-1) + spawn_nofetch(self.conf.trees[pkg.root]["porttree"].dbapi, + pkg_info.ebuild_path) if self.conf.changelog: self.print_changelog() diff --git a/portage_with_autodep/pym/_emerge/resolver/output.pyo b/portage_with_autodep/pym/_emerge/resolver/output.pyo Binary files differnew file mode 100644 index 0000000..bd2ae2f --- /dev/null +++ b/portage_with_autodep/pym/_emerge/resolver/output.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/output_helpers.py b/portage_with_autodep/pym/_emerge/resolver/output_helpers.py index b7e7376..e751dd8 100644 --- a/portage_with_autodep/pym/_emerge/resolver/output_helpers.py +++ b/portage_with_autodep/pym/_emerge/resolver/output_helpers.py @@ -14,10 +14,10 @@ import sys from portage import os from portage import _encodings, _unicode_encode from portage._sets.base import InternalPackageSet -from portage.output import blue, colorize, create_color_func, green, red, \ - teal, yellow +from portage.output import (blue, bold, colorize, create_color_func, + green, red, teal, yellow) bad = create_color_func("BAD") -from portage.util import writemsg +from portage.util import shlex_split, writemsg from portage.versions import catpkgsplit from _emerge.Blocker import Blocker @@ -39,7 +39,7 @@ class _RepoDisplay(object): repo_paths.add(portdir) overlays = root_config.settings.get("PORTDIR_OVERLAY") if overlays: - repo_paths.update(overlays.split()) + repo_paths.update(shlex_split(overlays)) repo_paths = list(repo_paths) self._repo_paths = repo_paths self._repo_paths_real = [ os.path.realpath(repo_path) \ @@ -198,7 +198,6 @@ class _DisplayConfig(object): self.print_use_string = self.verbosity != 1 or "--verbose" in frozen_config.myopts self.changelog = "--changelog" in frozen_config.myopts self.edebug = frozen_config.edebug - self.no_restart = frozen_config._opts_no_restart.intersection(frozen_config.myopts) self.unordered_display = "--unordered-display" in frozen_config.myopts mywidth = 130 @@ -212,7 +211,8 @@ class _DisplayConfig(object): del e self.columnwidth = mywidth - self.repo_display = _RepoDisplay(frozen_config.roots) + if "--quiet-repo-display" in frozen_config.myopts: + self.repo_display = _RepoDisplay(frozen_config.roots) self.trees = frozen_config.trees self.pkgsettings = frozen_config.pkgsettings self.target_root = frozen_config.target_root @@ -500,63 +500,120 @@ def _calc_changelog(ebuildpath,current,next): next = '-'.join(catpkgsplit(next)[1:]) if next.endswith('-r0'): next = next[:-3] - changelogpath = os.path.join(os.path.split(ebuildpath)[0],'ChangeLog') - try: - changelog = io.open(_unicode_encode(changelogpath, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace' - ).read() - except SystemExit: - raise # Needed else can't exit - except: + + changelogdir = os.path.dirname(ebuildpath) + changelogs = ['ChangeLog'] + # ChangeLog-YYYY (see bug #389611) + changelogs.extend(sorted((fn for fn in os.listdir(changelogdir) + if fn.startswith('ChangeLog-')), reverse=True)) + + divisions = [] + found_current = False + for fn in changelogs: + changelogpath = os.path.join(changelogdir, fn) + try: + with io.open(_unicode_encode(changelogpath, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + changelog = f.read() + except EnvironmentError: + return [] + for node in _find_changelog_tags(changelog): + if node[0] == current: + found_current = True + break + else: + divisions.append(node) + if found_current: + break + + if not found_current: return [] - divisions = _find_changelog_tags(changelog) + #print 'XX from',current,'to',next #for div,text in divisions: print 'XX',div # skip entries for all revisions above the one we are about to emerge - for i in range(len(divisions)): - if divisions[i][0]==next: - divisions = divisions[i:] - break - # find out how many entries we are going to display - for i in range(len(divisions)): - if divisions[i][0]==current: - divisions = divisions[:i] + later_rev_index = None + for i, node in enumerate(divisions): + if node[0] == next: + if later_rev_index is not None: + first_node = divisions[later_rev_index] + # Discard the later revision and the first ChangeLog entry + # that follows it. We want to display all the entries after + # that first entry, as discussed in bug #373009. + trimmed_lines = [] + iterator = iter(first_node[1]) + for l in iterator: + if not l: + # end of the first entry that's discarded + break + first_node = (None, list(iterator)) + divisions = [first_node] + divisions[later_rev_index+1:] break - else: - # couldnt find the current revision in the list. display nothing - return [] - return divisions - + if node[0] is not None: + later_rev_index = i + + output = [] + prev_blank = False + prev_rev = False + for rev, lines in divisions: + if rev is not None: + if not (prev_blank or prev_rev): + output.append("\n") + output.append(bold('*' + rev) + '\n') + prev_rev = True + prev_blank = False + if lines: + prev_rev = False + if not prev_blank: + output.append("\n") + for l in lines: + output.append(l + "\n") + output.append("\n") + prev_blank = True + return output + +def _strip_header_comments(lines): + # strip leading and trailing blank or header/comment lines + i = 0 + while i < len(lines) and (not lines[i] or lines[i][:1] == "#"): + i += 1 + if i: + lines = lines[i:] + while lines and (not lines[-1] or lines[-1][:1] == "#"): + lines.pop() + return lines def _find_changelog_tags(changelog): divs = [] + if not changelog: + return divs release = None - while 1: - match = re.search(r'^\*\ ?([-a-zA-Z0-9_.+]*)(?:\ .*)?\n',changelog,re.M) - if match is None: - if release is not None: - divs.append((release,changelog)) - return divs - if release is not None: - divs.append((release,changelog[:match.start()])) - changelog = changelog[match.end():] + release_end = 0 + for match in re.finditer(r'^\*\ ?([-a-zA-Z0-9_.+]*)(?:\ .*)?$', + changelog, re.M): + divs.append((release, _strip_header_comments( + changelog[release_end:match.start()].splitlines()))) + release_end = match.end() release = match.group(1) if release.endswith('.ebuild'): release = release[:-7] if release.endswith('-r0'): release = release[:-3] + divs.append((release, + _strip_header_comments(changelog[release_end:].splitlines()))) + return divs class PkgInfo(object): """Simple class to hold instance attributes for current information about the pkg being printed. """ - __slots__ = ("ordered", "fetch_symbol", "operation", "merge", - "built", "cp", "ebuild_path", "repo_name", "repo_path_real", - "world", "system", "use", "oldbest", "ver" - ) + __slots__ = ("built", "cp", "ebuild_path", "fetch_symbol", "merge", + "oldbest", "oldbest_list", "operation", "ordered", + "repo_name", "repo_path_real", "system", "use", "ver", "world") def __init__(self): @@ -566,6 +623,7 @@ class PkgInfo(object): self.fetch_symbol = '' self.merge = '' self.oldbest = '' + self.oldbest_list = [] self.operation = '' self.ordered = False self.repo_path_real = '' diff --git a/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo b/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo Binary files differnew file mode 100644 index 0000000..ae39dd4 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/resolver/output_helpers.pyo diff --git a/portage_with_autodep/pym/_emerge/resolver/slot_collision.py b/portage_with_autodep/pym/_emerge/resolver/slot_collision.py index 0df8f20..a1c8714 100644 --- a/portage_with_autodep/pym/_emerge/resolver/slot_collision.py +++ b/portage_with_autodep/pym/_emerge/resolver/slot_collision.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -80,6 +80,8 @@ class slot_conflict_handler(object): the needed USE changes and prepare the message for the user. """ + _check_configuration_max = 1024 + def __init__(self, depgraph): self.depgraph = depgraph self.myopts = depgraph._frozen_config.myopts @@ -244,7 +246,7 @@ class slot_conflict_handler(object): for (slot_atom, root), pkgs \ in self.slot_collision_info.items(): msg.append(str(slot_atom)) - if root != '/': + if root != self.depgraph._frozen_config._running_root.root: msg.append(" for %s" % (root,)) msg.append("\n\n") @@ -663,14 +665,24 @@ class slot_conflict_handler(object): solutions = [] sol_gen = _solution_candidate_generator(all_involved_flags) - while(True): + checked = 0 + while True: candidate = sol_gen.get_candidate() if not candidate: break solution = self._check_solution(config, candidate, all_conflict_atoms_by_slotatom) + checked += 1 if solution: solutions.append(solution) - + + if checked >= self._check_configuration_max: + # TODO: Implement early elimination for candidates that would + # change forced or masked flags, and don't count them here. + if self.debug: + writemsg("\nAborting _check_configuration due to " + "excessive number of candidates.\n", noiselevel=-1) + break + if self.debug: if not solutions: writemsg("No viable solutions. Rejecting configuration.\n", noiselevel=-1) @@ -843,7 +855,7 @@ class slot_conflict_handler(object): #Make sure the changes don't violate REQUIRED_USE for pkg in required_changes: - required_use = pkg.metadata["REQUIRED_USE"] + required_use = pkg.metadata.get("REQUIRED_USE") if not required_use: continue diff --git a/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo b/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo Binary files differnew file mode 100644 index 0000000..1fc3a13 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/resolver/slot_collision.pyo diff --git a/portage_with_autodep/pym/_emerge/search.py b/portage_with_autodep/pym/_emerge/search.py index 35f0412..5abc8a0 100644 --- a/portage_with_autodep/pym/_emerge/search.py +++ b/portage_with_autodep/pym/_emerge/search.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -6,6 +6,7 @@ from __future__ import print_function import re import portage from portage import os +from portage.dbapi.porttree import _parse_uri_map from portage.output import bold, bold as white, darkgreen, green, red from portage.util import writemsg_stdout @@ -150,7 +151,7 @@ class search(object): if not result or cpv == portage.best([cpv, result]): result = cpv else: - db_keys = Package.metadata_keys + db_keys = list(db._aux_cache_keys) # break out of this loop with highest visible # match, checked in descending order for cpv in reversed(db.match(atom)): @@ -261,11 +262,13 @@ class search(object): msg.append("[ Applications found : " + \ bold(str(self.mlen)) + " ]\n\n") vardb = self.vartree.dbapi + metadata_keys = set(Package.metadata_keys) + metadata_keys.update(["DESCRIPTION", "HOMEPAGE", "LICENSE", "SRC_URI"]) + metadata_keys = tuple(metadata_keys) for mtype in self.matches: for match,masked in self.matches[mtype]: full_package = None if mtype == "pkg": - catpack = match full_package = self._xmatch( "bestmatch-visible", match) if not full_package: @@ -285,11 +288,16 @@ class search(object): + "\n\n") if full_package: try: - desc, homepage, license = self._aux_get( - full_package, ["DESCRIPTION","HOMEPAGE","LICENSE"]) + metadata = dict(zip(metadata_keys, + self._aux_get(full_package, metadata_keys))) except KeyError: msg.append("emerge: search: aux_get() failed, skipping\n") continue + + desc = metadata["DESCRIPTION"] + homepage = metadata["HOMEPAGE"] + license = metadata["LICENSE"] + if masked: msg.append(green("*") + " " + \ white(match) + " " + red("[ Masked ]") + "\n") @@ -304,12 +312,17 @@ class search(object): mycpv = match + "-" + myversion myebuild = self._findname(mycpv) if myebuild: + pkg = Package(built=False, cpv=mycpv, + installed=False, metadata=metadata, + root_config=self.root_config, type_name="ebuild") pkgdir = os.path.dirname(myebuild) - from portage import manifest - mf = manifest.Manifest( + mf = self.settings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + mf = mf.load_manifest( pkgdir, self.settings["DISTDIR"]) try: - uri_map = self._getFetchMap(mycpv) + uri_map = _parse_uri_map(mycpv, metadata, + use=pkg.use.enabled) except portage.exception.InvalidDependString as e: file_size_str = "Unknown (%s)" % (e,) del e diff --git a/portage_with_autodep/pym/_emerge/search.pyo b/portage_with_autodep/pym/_emerge/search.pyo Binary files differnew file mode 100644 index 0000000..055a734 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/search.pyo diff --git a/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo b/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo Binary files differnew file mode 100644 index 0000000..337e135 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/show_invalid_depstring_notice.pyo diff --git a/portage_with_autodep/pym/_emerge/stdout_spinner.pyo b/portage_with_autodep/pym/_emerge/stdout_spinner.pyo Binary files differnew file mode 100644 index 0000000..b091171 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/stdout_spinner.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/__init__.pyo b/portage_with_autodep/pym/_emerge/sync/__init__.pyo Binary files differnew file mode 100644 index 0000000..7314f80 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/sync/__init__.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo b/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo Binary files differnew file mode 100644 index 0000000..8e41377 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/sync/getaddrinfo_validate.pyo diff --git a/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo b/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo Binary files differnew file mode 100644 index 0000000..5b59c5a --- /dev/null +++ b/portage_with_autodep/pym/_emerge/sync/old_tree_timestamp.pyo diff --git a/portage_with_autodep/pym/_emerge/unmerge.py b/portage_with_autodep/pym/_emerge/unmerge.py index 3db3a8b..b46b89c 100644 --- a/portage_with_autodep/pym/_emerge/unmerge.py +++ b/portage_with_autodep/pym/_emerge/unmerge.py @@ -1,9 +1,10 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function import logging +import signal import sys import textwrap import portage @@ -12,7 +13,7 @@ from portage.dbapi._expand_new_virt import expand_new_virt from portage.output import bold, colorize, darkgreen, green from portage._sets import SETPREFIX from portage._sets.base import EditablePackageSet -from portage.util import cmp_sort_key +from portage.versions import cpv_sort_key, _pkg_str from _emerge.emergelog import emergelog from _emerge.Package import Package @@ -467,20 +468,22 @@ def _unmerge_display(root_config, myopts, unmerge_action, if not quiet: writemsg_level((mytype + ": ").rjust(14), noiselevel=-1) if pkgmap[x][mytype]: - sorted_pkgs = [portage.catpkgsplit(mypkg)[1:] for mypkg in pkgmap[x][mytype]] - sorted_pkgs.sort(key=cmp_sort_key(portage.pkgcmp)) - for pn, ver, rev in sorted_pkgs: - if rev == "r0": - myversion = ver - else: - myversion = ver + "-" + rev + sorted_pkgs = [] + for mypkg in pkgmap[x][mytype]: + try: + sorted_pkgs.append(mypkg.cpv) + except AttributeError: + sorted_pkgs.append(_pkg_str(mypkg)) + sorted_pkgs.sort(key=cpv_sort_key()) + for mypkg in sorted_pkgs: if mytype == "selected": writemsg_level( - colorize("UNMERGE_WARN", myversion + " "), + colorize("UNMERGE_WARN", mypkg.version + " "), noiselevel=-1) else: writemsg_level( - colorize("GOOD", myversion + " "), noiselevel=-1) + colorize("GOOD", mypkg.version + " "), + noiselevel=-1) else: writemsg_level("none ", noiselevel=-1) if not quiet: @@ -503,7 +506,8 @@ def unmerge(root_config, myopts, unmerge_action, clean_world=1, clean_delay=1, ordered=0, raise_on_error=0, scheduler=None, writemsg_level=portage.util.writemsg_level): """ - Returns 1 if successful, otherwise 0. + Returns os.EX_OK if no errors occur, 1 if an error occurs, and + 130 if interrupted due to a 'no' answer for --ask. """ if clean_world: @@ -515,7 +519,7 @@ def unmerge(root_config, myopts, unmerge_action, writemsg_level=writemsg_level) if rval != os.EX_OK: - return 0 + return rval enter_invalid = '--ask-enter-invalid' in myopts vartree = root_config.trees["vartree"] @@ -526,7 +530,7 @@ def unmerge(root_config, myopts, unmerge_action, if "--pretend" in myopts: #we're done... return - return 0 + return os.EX_OK if "--ask" in myopts: if userquery("Would you like to unmerge these packages?", enter_invalid) == "No": @@ -535,19 +539,32 @@ def unmerge(root_config, myopts, unmerge_action, print() print("Quitting.") print() - return 0 + return 128 + signal.SIGINT #the real unmerging begins, after a short delay.... if clean_delay and not autoclean: countdown(int(settings["CLEAN_DELAY"]), ">>> Unmerging") + all_selected = set() + all_selected.update(*[x["selected"] for x in pkgmap]) + + # Set counter variables + curval = 1 + maxval = len(all_selected) + for x in range(len(pkgmap)): for y in pkgmap[x]["selected"]: - writemsg_level(">>> Unmerging "+y+"...\n", noiselevel=-1) emergelog(xterm_titles, "=== Unmerging... ("+y+")") + message = ">>> Unmerging ({0} of {1}) {2}...\n".format( + colorize("MERGE_LIST_PROGRESS", str(curval)), + colorize("MERGE_LIST_PROGRESS", str(maxval)), + y) + writemsg_level(message, noiselevel=-1) + curval += 1 + mysplit = y.split("/") #unmerge... - retval = portage.unmerge(mysplit[0], mysplit[1], settings["ROOT"], - mysettings, unmerge_action not in ["clean","prune"], + retval = portage.unmerge(mysplit[0], mysplit[1], + settings=mysettings, vartree=vartree, ldpath_mtimes=ldpath_mtimes, scheduler=scheduler) @@ -574,5 +591,5 @@ def unmerge(root_config, myopts, unmerge_action, sets["selected"].remove(SETPREFIX + s) sets["selected"].unlock() - return 1 + return os.EX_OK diff --git a/portage_with_autodep/pym/_emerge/unmerge.pyo b/portage_with_autodep/pym/_emerge/unmerge.pyo Binary files differnew file mode 100644 index 0000000..c16c9ec --- /dev/null +++ b/portage_with_autodep/pym/_emerge/unmerge.pyo diff --git a/portage_with_autodep/pym/_emerge/userquery.py b/portage_with_autodep/pym/_emerge/userquery.py index e7ed400..efae80a 100644 --- a/portage_with_autodep/pym/_emerge/userquery.py +++ b/portage_with_autodep/pym/_emerge/userquery.py @@ -1,8 +1,9 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function +import signal import sys from portage.output import bold, create_color_func @@ -51,5 +52,4 @@ def userquery(prompt, enter_invalid, responses=None, colours=None): print("Sorry, response '%s' not understood." % response, end=' ') except (EOFError, KeyboardInterrupt): print("Interrupted.") - sys.exit(1) - + sys.exit(128 + signal.SIGINT) diff --git a/portage_with_autodep/pym/_emerge/userquery.pyo b/portage_with_autodep/pym/_emerge/userquery.pyo Binary files differnew file mode 100644 index 0000000..5492f90 --- /dev/null +++ b/portage_with_autodep/pym/_emerge/userquery.pyo diff --git a/portage_with_autodep/pym/portage/__init__.py b/portage_with_autodep/pym/portage/__init__.py index 2a2eb99..431dc26 100644 --- a/portage_with_autodep/pym/portage/__init__.py +++ b/portage_with_autodep/pym/portage/__init__.py @@ -2,7 +2,7 @@ # Copyright 1998-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -VERSION="HEAD" +VERSION="2.2.0_alpha108" # =========================================================================== # START OF IMPORTS -- START OF IMPORTS -- START OF IMPORTS -- START OF IMPORT @@ -148,20 +148,35 @@ if sys.hexversion >= 0x3000000: basestring = str long = int -# Assume utf_8 fs encoding everywhere except in merge code, where the -# user's locale is respected. +# We use utf_8 encoding everywhere. Previously, we used +# sys.getfilesystemencoding() for the 'merge' encoding, but that had +# various problems: +# +# 1) If the locale is ever changed then it can cause orphan files due +# to changed character set translation. +# +# 2) Ebuilds typically install files with utf_8 encoded file names, +# and then portage would be forced to rename those files to match +# sys.getfilesystemencoding(), possibly breaking things. +# +# 3) Automatic translation between encodings can lead to nonsensical +# file names when the source encoding is unknown by portage. +# +# 4) It's inconvenient for ebuilds to convert the encodings of file +# names to match the current locale, and upstreams typically encode +# file names with utf_8 encoding. +# +# So, instead of relying on sys.getfilesystemencoding(), we avoid the above +# problems by using a constant utf_8 'merge' encoding for all locales, as +# discussed in bug #382199 and bug #381509. _encodings = { 'content' : 'utf_8', 'fs' : 'utf_8', - 'merge' : sys.getfilesystemencoding(), + 'merge' : 'utf_8', 'repo.content' : 'utf_8', 'stdio' : 'utf_8', } -# This can happen if python is built with USE=build (stage 1). -if _encodings['merge'] is None: - _encodings['merge'] = 'ascii' - if sys.hexversion >= 0x3000000: def _unicode_encode(s, encoding=_encodings['content'], errors='backslashreplace'): if isinstance(s, str): @@ -215,7 +230,7 @@ class _unicode_func_wrapper(object): rval = self._func(*wrapped_args, **wrapped_kwargs) # Don't use isinstance() since we don't want to convert subclasses - # of tuple such as posix.stat_result in python-3.2. + # of tuple such as posix.stat_result in Python >=3.2. if rval.__class__ in (list, tuple): decoded_rval = [] for x in rval: @@ -320,6 +335,16 @@ _python_interpreter = os.path.realpath(sys.executable) _bin_path = PORTAGE_BIN_PATH _pym_path = PORTAGE_PYM_PATH +if sys.hexversion >= 0x3030000: + # Workaround for http://bugs.python.org/issue14007 + def _test_xml_etree_ElementTree_TreeBuilder_type(): + import subprocess + p = subprocess.Popen([_python_interpreter, "-c", + "import sys, xml.etree.ElementTree; sys.exit(not isinstance(xml.etree.ElementTree.TreeBuilder, type))"]) + if p.wait() != 0: + sys.modules["_elementtree"] = None + _test_xml_etree_ElementTree_TreeBuilder_type() + def _shell_quote(s): """ Quote a string in double-quotes and use backslashes to @@ -380,9 +405,12 @@ def getcwd(): return "/" getcwd() -def abssymlink(symlink): +def abssymlink(symlink, target=None): "This reads symlinks, resolving the relative symlinks, and returning the absolute." - mylink=os.readlink(symlink) + if target is not None: + mylink = target + else: + mylink = os.readlink(symlink) if mylink[0] != '/': mydir=os.path.dirname(symlink) mylink=mydir+"/"+mylink @@ -417,29 +445,25 @@ def eapi_is_supported(eapi): return False return eapi <= portage.const.EAPI -# Generally, it's best not to assume that cache entries for unsupported EAPIs -# can be validated. However, the current package manager specification does not -# guarantee that the EAPI can be parsed without sourcing the ebuild, so -# it's too costly to discard existing cache entries for unsupported EAPIs. -# Therefore, by default, assume that cache entries for unsupported EAPIs can be -# validated. If FEATURES=parse-eapi-* is enabled, this assumption is discarded -# since the EAPI can be determined without the incurring the cost of sourcing -# the ebuild. -_validate_cache_for_unsupported_eapis = True - -_parse_eapi_ebuild_head_re = re.compile(r'^EAPI=[\'"]?([^\'"#]*)') -_parse_eapi_ebuild_head_max_lines = 30 +# This pattern is specified by PMS section 7.3.1. +_pms_eapi_re = re.compile(r"^[ \t]*EAPI=(['\"]?)([A-Za-z0-9+_.-]*)\1[ \t]*([ \t]#.*)?$") +_comment_or_blank_line = re.compile(r"^\s*(#.*)?$") def _parse_eapi_ebuild_head(f): - count = 0 + eapi = None + eapi_lineno = None + lineno = 0 for line in f: - m = _parse_eapi_ebuild_head_re.match(line) - if m is not None: - return m.group(1).strip() - count += 1 - if count >= _parse_eapi_ebuild_head_max_lines: + lineno += 1 + m = _comment_or_blank_line.match(line) + if m is None: + eapi_lineno = lineno + m = _pms_eapi_re.match(line) + if m is not None: + eapi = m.group(2) break - return '0' + + return (eapi, eapi_lineno) def _movefile(src, dest, **kwargs): """Calls movefile and raises a PortageException if an error occurs.""" @@ -461,10 +485,16 @@ def portageexit(): if data.secpass > 1 and os.environ.get("SANDBOX_ON") != "1": close_portdbapi_caches() -def create_trees(config_root=None, target_root=None, trees=None): - if trees is None: - trees = {} - else: +class _trees_dict(dict): + __slots__ = ('_running_eroot', '_target_eroot',) + def __init__(self, *pargs, **kargs): + dict.__init__(self, *pargs, **kargs) + self._running_eroot = None + self._target_eroot = None + +def create_trees(config_root=None, target_root=None, trees=None, env=None, + eprefix=None): + if trees is not None: # clean up any existing portdbapi instances for myroot in trees: portdb = trees[myroot]["porttree"].dbapi @@ -472,12 +502,25 @@ def create_trees(config_root=None, target_root=None, trees=None): portdbapi.portdbapi_instances.remove(portdb) del trees[myroot]["porttree"], myroot, portdb + if trees is None: + trees = _trees_dict() + elif not isinstance(trees, _trees_dict): + # caller passed a normal dict or something, + # but we need a _trees_dict instance + trees = _trees_dict(trees) + + if env is None: + env = os.environ + settings = config(config_root=config_root, target_root=target_root, - config_incrementals=portage.const.INCREMENTALS) + env=env, eprefix=eprefix) settings.lock() - myroots = [(settings["ROOT"], settings)] - if settings["ROOT"] != "/": + trees._target_eroot = settings['EROOT'] + myroots = [(settings['EROOT'], settings)] + if settings["ROOT"] == "/": + trees._running_eroot = trees._target_eroot + else: # When ROOT != "/" we only want overrides from the calling # environment to apply to the config that's associated @@ -485,24 +528,27 @@ def create_trees(config_root=None, target_root=None, trees=None): clean_env = {} for k in ('PATH', 'PORTAGE_GRPNAME', 'PORTAGE_USERNAME', 'SSH_AGENT_PID', 'SSH_AUTH_SOCK', 'TERM', - 'ftp_proxy', 'http_proxy', 'no_proxy'): + 'ftp_proxy', 'http_proxy', 'no_proxy', + '__PORTAGE_TEST_HARDLINK_LOCKS'): v = settings.get(k) if v is not None: clean_env[k] = v - settings = config(config_root=None, target_root="/", env=clean_env) + settings = config(config_root=None, target_root="/", + env=clean_env, eprefix=eprefix) settings.lock() - myroots.append((settings["ROOT"], settings)) + trees._running_eroot = settings['EROOT'] + myroots.append((settings['EROOT'], settings)) for myroot, mysettings in myroots: trees[myroot] = portage.util.LazyItemsDict(trees.get(myroot, {})) trees[myroot].addLazySingleton("virtuals", mysettings.getvirtuals) trees[myroot].addLazySingleton( - "vartree", vartree, myroot, categories=mysettings.categories, + "vartree", vartree, categories=mysettings.categories, settings=mysettings) trees[myroot].addLazySingleton("porttree", - portagetree, myroot, settings=mysettings) + portagetree, settings=mysettings) trees[myroot].addLazySingleton("bintree", - binarytree, myroot, mysettings["PKGDIR"], settings=mysettings) + binarytree, pkgdir=mysettings["PKGDIR"], settings=mysettings) return trees if VERSION == 'HEAD': diff --git a/portage_with_autodep/pym/portage/__init__.pyo b/portage_with_autodep/pym/portage/__init__.pyo Binary files differnew file mode 100644 index 0000000..9fc449e --- /dev/null +++ b/portage_with_autodep/pym/portage/__init__.pyo diff --git a/portage_with_autodep/pym/portage/_emirrordist/Config.py b/portage_with_autodep/pym/portage/_emirrordist/Config.py new file mode 100644 index 0000000..db4bfeb --- /dev/null +++ b/portage_with_autodep/pym/portage/_emirrordist/Config.py @@ -0,0 +1,132 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import copy +import io +import logging +import shelve +import sys +import time + +import portage +from portage import os +from portage.util import grabdict, grablines +from portage.util._ShelveUnicodeWrapper import ShelveUnicodeWrapper + +class Config(object): + def __init__(self, options, portdb, event_loop): + self.options = options + self.portdb = portdb + self.event_loop = event_loop + self.added_byte_count = 0 + self.added_file_count = 0 + self.scheduled_deletion_count = 0 + self.delete_count = 0 + self.file_owners = {} + self.file_failures = {} + self.start_time = time.time() + self._open_files = [] + + self.log_success = self._open_log('success', options.success_log, 'a') + self.log_failure = self._open_log('failure', options.failure_log, 'a') + + self.distfiles = None + if options.distfiles is not None: + self.distfiles = options.distfiles + + self.mirrors = copy.copy(portdb.settings.thirdpartymirrors()) + + if options.mirror_overrides is not None: + self.mirrors.update(grabdict(options.mirror_overrides)) + + if options.mirror_skip is not None: + for x in options.mirror_skip.split(","): + self.mirrors[x] = [] + + self.whitelist = None + if options.whitelist_from is not None: + self.whitelist = set() + for filename in options.whitelist_from: + for line in grablines(filename): + line = line.strip() + if line and not line.startswith("#"): + self.whitelist.add(line) + + self.restrict_mirror_exemptions = None + if options.restrict_mirror_exemptions is not None: + self.restrict_mirror_exemptions = frozenset( + options.restrict_mirror_exemptions.split(",")) + + self.recycle_db = None + if options.recycle_db is not None: + self.recycle_db = self._open_shelve( + options.recycle_db, 'recycle') + + self.distfiles_db = None + if options.distfiles_db is not None: + self.distfiles_db = self._open_shelve( + options.distfiles_db, 'distfiles') + + self.deletion_db = None + if options.deletion_db is not None: + self.deletion_db = self._open_shelve( + options.deletion_db, 'deletion') + + def _open_log(self, log_desc, log_path, mode): + + if log_path is None or self.options.dry_run: + log_func = logging.info + line_format = "%s: %%s" % log_desc + add_newline = False + if log_path is not None: + logging.warn(("dry-run: %s log " + "redirected to logging.info") % log_desc) + else: + self._open_files.append(io.open(log_path, mode=mode, + encoding='utf_8')) + line_format = "%s\n" + log_func = self._open_files[-1].write + + return self._LogFormatter(line_format, log_func) + + class _LogFormatter(object): + + __slots__ = ('_line_format', '_log_func') + + def __init__(self, line_format, log_func): + self._line_format = line_format + self._log_func = log_func + + def __call__(self, msg): + self._log_func(self._line_format % (msg,)) + + def _open_shelve(self, db_file, db_desc): + if self.options.dry_run: + open_flag = "r" + else: + open_flag = "c" + + if self.options.dry_run and not os.path.exists(db_file): + db = {} + else: + db = shelve.open(db_file, flag=open_flag) + if sys.hexversion < 0x3000000: + db = ShelveUnicodeWrapper(db) + + if self.options.dry_run: + logging.warn("dry-run: %s db opened in readonly mode" % db_desc) + if not isinstance(db, dict): + volatile_db = dict((k, db[k]) for k in db) + db.close() + db = volatile_db + else: + self._open_files.append(db) + + return db + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + while self._open_files: + self._open_files.pop().close() diff --git a/portage_with_autodep/pym/portage/_emirrordist/DeletionIterator.py b/portage_with_autodep/pym/portage/_emirrordist/DeletionIterator.py new file mode 100644 index 0000000..dff52c0 --- /dev/null +++ b/portage_with_autodep/pym/portage/_emirrordist/DeletionIterator.py @@ -0,0 +1,83 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import logging +import stat + +from portage import os +from .DeletionTask import DeletionTask + +class DeletionIterator(object): + + def __init__(self, config): + self._config = config + + def __iter__(self): + distdir = self._config.options.distfiles + file_owners = self._config.file_owners + whitelist = self._config.whitelist + distfiles_local = self._config.options.distfiles_local + deletion_db = self._config.deletion_db + deletion_delay = self._config.options.deletion_delay + start_time = self._config.start_time + distfiles_set = set(os.listdir(self._config.options.distfiles)) + for filename in distfiles_set: + try: + st = os.stat(os.path.join(distdir, filename)) + except OSError as e: + logging.error("stat failed on '%s' in distfiles: %s\n" % + (filename, e)) + continue + if not stat.S_ISREG(st.st_mode): + continue + elif filename in file_owners: + if deletion_db is not None: + try: + del deletion_db[filename] + except KeyError: + pass + elif whitelist is not None and filename in whitelist: + if deletion_db is not None: + try: + del deletion_db[filename] + except KeyError: + pass + elif distfiles_local is not None and \ + os.path.exists(os.path.join(distfiles_local, filename)): + if deletion_db is not None: + try: + del deletion_db[filename] + except KeyError: + pass + else: + self._config.scheduled_deletion_count += 1 + + if deletion_db is None or deletion_delay is None: + + yield DeletionTask(background=True, + distfile=filename, + config=self._config) + + else: + deletion_entry = deletion_db.get(filename) + + if deletion_entry is None: + logging.debug("add '%s' to deletion db" % filename) + deletion_db[filename] = start_time + + elif deletion_entry + deletion_delay <= start_time: + + yield DeletionTask(background=True, + distfile=filename, + config=self._config) + + if deletion_db is not None: + for filename in list(deletion_db): + if filename not in distfiles_set: + try: + del deletion_db[filename] + except KeyError: + pass + else: + logging.debug("drop '%s' from deletion db" % + filename) diff --git a/portage_with_autodep/pym/portage/_emirrordist/DeletionTask.py b/portage_with_autodep/pym/portage/_emirrordist/DeletionTask.py new file mode 100644 index 0000000..7d10957 --- /dev/null +++ b/portage_with_autodep/pym/portage/_emirrordist/DeletionTask.py @@ -0,0 +1,129 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import logging + +from portage import os +from portage.util._async.FileCopier import FileCopier +from _emerge.CompositeTask import CompositeTask + +class DeletionTask(CompositeTask): + + __slots__ = ('distfile', 'config') + + def _start(self): + + distfile_path = os.path.join( + self.config.options.distfiles, self.distfile) + + if self.config.options.recycle_dir is not None: + distfile_path = os.path.join(self.config.options.distfiles, self.distfile) + recycle_path = os.path.join( + self.config.options.recycle_dir, self.distfile) + if self.config.options.dry_run: + logging.info(("dry-run: move '%s' from " + "distfiles to recycle") % self.distfile) + else: + logging.debug(("move '%s' from " + "distfiles to recycle") % self.distfile) + try: + os.rename(distfile_path, recycle_path) + except OSError as e: + if e.errno != errno.EXDEV: + logging.error(("rename %s from distfiles to " + "recycle failed: %s") % (self.distfile, e)) + else: + self.returncode = os.EX_OK + self._async_wait() + return + + self._start_task( + FileCopier(src_path=distfile_path, + dest_path=recycle_path, + background=False), + self._recycle_copier_exit) + return + + success = True + + if self.config.options.dry_run: + logging.info(("dry-run: delete '%s' from " + "distfiles") % self.distfile) + else: + logging.debug(("delete '%s' from " + "distfiles") % self.distfile) + try: + os.unlink(distfile_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + logging.error("%s unlink failed in distfiles: %s" % + (self.distfile, e)) + success = False + + if success: + self._success() + self.returncode = os.EX_OK + else: + self.returncode = 1 + + self._async_wait() + + def _recycle_copier_exit(self, copier): + + self._assert_current(copier) + if self._was_cancelled(): + self.wait() + return + + success = True + if copier.returncode == os.EX_OK: + + try: + os.unlink(copier.src_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + logging.error("%s unlink failed in distfiles: %s" % + (self.distfile, e)) + success = False + + else: + logging.error(("%s copy from distfiles " + "to recycle failed: %s") % (self.distfile, e)) + success = False + + if success: + self._success() + self.returncode = os.EX_OK + else: + self.returncode = 1 + + self._current_task = None + self.wait() + + def _success(self): + + cpv = "unknown" + if self.config.distfiles_db is not None: + cpv = self.config.distfiles_db.get(self.distfile, cpv) + + self.config.delete_count += 1 + self.config.log_success("%s\t%s\tremoved" % (cpv, self.distfile)) + + if self.config.distfiles_db is not None: + try: + del self.config.distfiles_db[self.distfile] + except KeyError: + pass + else: + logging.debug(("drop '%s' from " + "distfiles db") % self.distfile) + + if self.config.deletion_db is not None: + try: + del self.config.deletion_db[self.distfile] + except KeyError: + pass + else: + logging.debug(("drop '%s' from " + "deletion db") % self.distfile) diff --git a/portage_with_autodep/pym/portage/_emirrordist/FetchIterator.py b/portage_with_autodep/pym/portage/_emirrordist/FetchIterator.py new file mode 100644 index 0000000..16a0b04 --- /dev/null +++ b/portage_with_autodep/pym/portage/_emirrordist/FetchIterator.py @@ -0,0 +1,147 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from portage.checksum import (_apply_hash_filter, + _filter_unaccelarated_hashes, _hash_filter) +from portage.dep import use_reduce +from portage.exception import PortageException +from .FetchTask import FetchTask + +class FetchIterator(object): + + def __init__(self, config): + self._config = config + self._log_failure = config.log_failure + + def _iter_every_cp(self): + # List categories individually, in order to start yielding quicker, + # and in order to reduce latency in case of a signal interrupt. + cp_all = self._config.portdb.cp_all + for category in sorted(self._config.portdb.categories): + for cp in cp_all(categories=(category,)): + yield cp + + def __iter__(self): + + portdb = self._config.portdb + get_repo_for_location = portdb.repositories.get_repo_for_location + file_owners = self._config.file_owners + file_failures = self._config.file_failures + restrict_mirror_exemptions = self._config.restrict_mirror_exemptions + + hash_filter = _hash_filter( + portdb.settings.get("PORTAGE_CHECKSUM_FILTER", "")) + if hash_filter.transparent: + hash_filter = None + + for cp in self._iter_every_cp(): + + for tree in portdb.porttrees: + + # Reset state so the Manifest is pulled once + # for this cp / tree combination. + digests = None + repo_config = get_repo_for_location(tree) + + for cpv in portdb.cp_list(cp, mytree=tree): + + try: + restrict, = portdb.aux_get(cpv, ("RESTRICT",), + mytree=tree) + except (KeyError, PortageException) as e: + self._log_failure("%s\t\taux_get exception %s" % + (cpv, e)) + continue + + # Here we use matchnone=True to ignore conditional parts + # of RESTRICT since they don't apply unconditionally. + # Assume such conditionals only apply on the client side. + try: + restrict = frozenset(use_reduce(restrict, + flat=True, matchnone=True)) + except PortageException as e: + self._log_failure("%s\t\tuse_reduce exception %s" % + (cpv, e)) + continue + + if "fetch" in restrict: + continue + + try: + uri_map = portdb.getFetchMap(cpv) + except PortageException as e: + self._log_failure("%s\t\tgetFetchMap exception %s" % + (cpv, e)) + continue + + if not uri_map: + continue + + if "mirror" in restrict: + skip = False + if restrict_mirror_exemptions is not None: + new_uri_map = {} + for filename, uri_tuple in uri_map.items(): + for uri in uri_tuple: + if uri[:9] == "mirror://": + i = uri.find("/", 9) + if i != -1 and uri[9:i].strip("/") in \ + restrict_mirror_exemptions: + new_uri_map[filename] = uri_tuple + break + if new_uri_map: + uri_map = new_uri_map + else: + skip = True + else: + skip = True + + if skip: + continue + + # Parse Manifest for this cp if we haven't yet. + if digests is None: + try: + digests = repo_config.load_manifest( + os.path.join(repo_config.location, cp) + ).getTypeDigests("DIST") + except (EnvironmentError, PortageException) as e: + for filename in uri_map: + self._log_failure( + "%s\t%s\tManifest exception %s" % + (cpv, filename, e)) + file_failures[filename] = cpv + continue + + if not digests: + for filename in uri_map: + self._log_failure("%s\t%s\tdigest entry missing" % + (cpv, filename)) + file_failures[filename] = cpv + continue + + for filename, uri_tuple in uri_map.items(): + file_digests = digests.get(filename) + if file_digests is None: + self._log_failure("%s\t%s\tdigest entry missing" % + (cpv, filename)) + file_failures[filename] = cpv + continue + if filename in file_owners: + continue + file_owners[filename] = cpv + + file_digests = \ + _filter_unaccelarated_hashes(file_digests) + if hash_filter is not None: + file_digests = _apply_hash_filter( + file_digests, hash_filter) + + yield FetchTask(cpv=cpv, + background=True, + digests=file_digests, + distfile=filename, + restrict=restrict, + uri_tuple=uri_tuple, + config=self._config) diff --git a/portage_with_autodep/pym/portage/_emirrordist/FetchTask.py b/portage_with_autodep/pym/portage/_emirrordist/FetchTask.py new file mode 100644 index 0000000..66c41c1 --- /dev/null +++ b/portage_with_autodep/pym/portage/_emirrordist/FetchTask.py @@ -0,0 +1,629 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import collections +import errno +import logging +import random +import stat +import subprocess +import sys + +import portage +from portage import _encodings, _unicode_encode +from portage import os +from portage.util._async.FileCopier import FileCopier +from portage.util._async.FileDigester import FileDigester +from portage.util._async.PipeLogger import PipeLogger +from portage.util._async.PopenProcess import PopenProcess +from _emerge.CompositeTask import CompositeTask + +default_hash_name = portage.const.MANIFEST2_REQUIRED_HASH + +# Use --no-check-certificate since Manifest digests should provide +# enough security, and certificates can be self-signed or whatnot. +default_fetchcommand = "wget -c -v -t 1 --passive-ftp --no-check-certificate --timeout=60 -O \"${DISTDIR}/${FILE}\" \"${URI}\"" + +class FetchTask(CompositeTask): + + __slots__ = ('distfile', 'digests', 'config', 'cpv', + 'restrict', 'uri_tuple', '_current_mirror', + '_current_stat', '_fetch_tmp_dir_info', '_fetch_tmp_file', + '_fs_mirror_stack', '_mirror_stack', + '_previously_added', + '_primaryuri_stack', '_log_path', '_tried_uris') + + def _start(self): + + if self.config.options.fetch_log_dir is not None and \ + not self.config.options.dry_run: + self._log_path = os.path.join( + self.config.options.fetch_log_dir, + self.distfile + '.log') + + self._previously_added = True + if self.config.distfiles_db is not None and \ + self.distfile not in self.config.distfiles_db: + self._previously_added = False + self.config.distfiles_db[self.distfile] = self.cpv + + if not self._have_needed_digests(): + msg = "incomplete digests: %s" % " ".join(self.digests) + self.scheduler.output(msg, background=self.background, + log_path=self._log_path) + self.config.log_failure("%s\t%s\t%s" % + (self.cpv, self.distfile, msg)) + self.config.file_failures[self.distfile] = self.cpv + self.returncode = os.EX_OK + self._async_wait() + return + + distfile_path = os.path.join( + self.config.options.distfiles, self.distfile) + + st = None + size_ok = False + try: + st = os.stat(distfile_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + msg = "%s stat failed in %s: %s" % \ + (self.distfile, "distfiles", e) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + else: + size_ok = st.st_size == self.digests["size"] + + if not size_ok: + if self.config.options.dry_run: + if st is not None: + logging.info(("dry-run: delete '%s' with " + "wrong size from distfiles") % (self.distfile,)) + else: + # Do the unlink in order to ensure that the path is clear, + # even if stat raised ENOENT, since a broken symlink can + # trigger ENOENT. + if self._unlink_file(distfile_path, "distfiles"): + if st is not None: + logging.debug(("delete '%s' with " + "wrong size from distfiles") % (self.distfile,)) + else: + self.config.log_failure("%s\t%s\t%s" % + (self.cpv, self.distfile, "unlink failed in distfiles")) + self.returncode = os.EX_OK + self._async_wait() + return + + if size_ok: + if self.config.options.verify_existing_digest: + self._start_task( + FileDigester(file_path=distfile_path, + hash_names=(self._select_hash(),), + background=self.background, + logfile=self._log_path), self._distfiles_digester_exit) + return + + self._success() + self.returncode = os.EX_OK + self._async_wait() + return + + self._start_fetch() + + def _success(self): + if not self._previously_added: + size = self.digests["size"] + self.config.added_byte_count += size + self.config.added_file_count += 1 + self.config.log_success("%s\t%s\tadded %i bytes" % + (self.cpv, self.distfile, size)) + + if self._log_path is not None: + if not self.config.options.dry_run: + try: + os.unlink(self._log_path) + except OSError: + pass + + if self.config.options.recycle_dir is not None: + + recycle_file = os.path.join( + self.config.options.recycle_dir, self.distfile) + + if self.config.options.dry_run: + if os.path.exists(recycle_file): + logging.info("dry-run: delete '%s' from recycle" % + (self.distfile,)) + else: + try: + os.unlink(recycle_file) + except OSError: + pass + else: + logging.debug("delete '%s' from recycle" % + (self.distfile,)) + + def _distfiles_digester_exit(self, digester): + + self._assert_current(digester) + if self._was_cancelled(): + self.wait() + return + + if self._default_exit(digester) != os.EX_OK: + # IOError reading file in our main distfiles directory? This + # is a bad situation which normally does not occur, so + # skip this file and report it, in order to draw attention + # from the administrator. + msg = "%s distfiles digester failed unexpectedly" % \ + (self.distfile,) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + self.config.log_failure("%s\t%s\t%s" % + (self.cpv, self.distfile, msg)) + self.config.file_failures[self.distfile] = self.cpv + self.wait() + return + + wrong_digest = self._find_bad_digest(digester.digests) + if wrong_digest is None: + self._success() + self.returncode = os.EX_OK + self.wait() + return + + self._start_fetch() + + _mirror_info = collections.namedtuple('_mirror_info', + 'name location') + + def _start_fetch(self): + + self._previously_added = False + self._fs_mirror_stack = [] + if self.config.options.distfiles_local is not None: + self._fs_mirror_stack.append(self._mirror_info( + 'distfiles-local', self.config.options.distfiles_local)) + if self.config.options.recycle_dir is not None: + self._fs_mirror_stack.append(self._mirror_info( + 'recycle', self.config.options.recycle_dir)) + + self._primaryuri_stack = [] + self._mirror_stack = [] + for uri in reversed(self.uri_tuple): + if uri.startswith('mirror://'): + self._mirror_stack.append( + self._mirror_iterator(uri, self.config.mirrors)) + else: + self._primaryuri_stack.append(uri) + + self._tried_uris = set() + self._try_next_mirror() + + @staticmethod + def _mirror_iterator(uri, mirrors_dict): + + slash_index = uri.find("/", 9) + if slash_index != -1: + mirror_name = uri[9:slash_index].strip("/") + mirrors = mirrors_dict.get(mirror_name) + if not mirrors: + return + mirrors = list(mirrors) + while mirrors: + mirror = mirrors.pop(random.randint(0, len(mirrors) - 1)) + yield mirror.rstrip("/") + "/" + uri[slash_index+1:] + + def _try_next_mirror(self): + if self._fs_mirror_stack: + self._fetch_fs(self._fs_mirror_stack.pop()) + return + else: + uri = self._next_uri() + if uri is not None: + self._tried_uris.add(uri) + self._fetch_uri(uri) + return + + if self._tried_uris: + msg = "all uris failed" + else: + msg = "no fetchable uris" + + self.config.log_failure("%s\t%s\t%s" % + (self.cpv, self.distfile, msg)) + self.config.file_failures[self.distfile] = self.cpv + self.returncode = os.EX_OK + self.wait() + + def _next_uri(self): + remaining_tries = self.config.options.tries - len(self._tried_uris) + if remaining_tries > 0: + + if remaining_tries <= self.config.options.tries / 2: + while self._primaryuri_stack: + uri = self._primaryuri_stack.pop() + if uri not in self._tried_uris: + return uri + + while self._mirror_stack: + uri = next(self._mirror_stack[-1], None) + if uri is None: + self._mirror_stack.pop() + else: + if uri not in self._tried_uris: + return uri + + while self._primaryuri_stack: + uri = self._primaryuri_stack.pop() + if uri not in self._tried_uris: + return uri + + return None + + def _fetch_fs(self, mirror_info): + file_path = os.path.join(mirror_info.location, self.distfile) + + st = None + size_ok = False + try: + st = os.stat(file_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + msg = "%s stat failed in %s: %s" % \ + (self.distfile, mirror_info.name, e) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + else: + size_ok = st.st_size == self.digests["size"] + self._current_stat = st + + if size_ok: + self._current_mirror = mirror_info + self._start_task( + FileDigester(file_path=file_path, + hash_names=(self._select_hash(),), + background=self.background, + logfile=self._log_path), + self._fs_mirror_digester_exit) + else: + self._try_next_mirror() + + def _fs_mirror_digester_exit(self, digester): + + self._assert_current(digester) + if self._was_cancelled(): + self.wait() + return + + current_mirror = self._current_mirror + if digester.returncode != os.EX_OK: + msg = "%s %s digester failed unexpectedly" % \ + (self.distfile, current_mirror.name) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + else: + bad_digest = self._find_bad_digest(digester.digests) + if bad_digest is not None: + msg = "%s %s has bad %s digest: expected %s, got %s" % \ + (self.distfile, current_mirror.name, bad_digest, + self.digests[bad_digest], digester.digests[bad_digest]) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + elif self.config.options.dry_run: + # Report success without actually touching any files + if self._same_device(current_mirror.location, + self.config.options.distfiles): + logging.info(("dry-run: hardlink '%s' from %s " + "to distfiles") % (self.distfile, current_mirror.name)) + else: + logging.info("dry-run: copy '%s' from %s to distfiles" % + (self.distfile, current_mirror.name)) + self._success() + self.returncode = os.EX_OK + self.wait() + return + else: + src = os.path.join(current_mirror.location, self.distfile) + dest = os.path.join(self.config.options.distfiles, self.distfile) + if self._hardlink_atomic(src, dest, + "%s to %s" % (current_mirror.name, "distfiles")): + logging.debug("hardlink '%s' from %s to distfiles" % + (self.distfile, current_mirror.name)) + self._success() + self.returncode = os.EX_OK + self.wait() + return + else: + self._start_task( + FileCopier(src_path=src, dest_path=dest, + background=(self.background and + self._log_path is not None), + logfile=self._log_path), + self._fs_mirror_copier_exit) + return + + self._try_next_mirror() + + def _fs_mirror_copier_exit(self, copier): + + self._assert_current(copier) + if self._was_cancelled(): + self.wait() + return + + current_mirror = self._current_mirror + if copier.returncode != os.EX_OK: + msg = "%s %s copy failed unexpectedly" % \ + (self.distfile, current_mirror.name) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + else: + + logging.debug("copy '%s' from %s to distfiles" % + (self.distfile, current_mirror.name)) + + # Apply the timestamp from the source file, but + # just rely on umask for permissions. + try: + if sys.hexversion >= 0x3030000: + os.utime(copier.dest_path, + ns=(self._current_stat.st_mtime_ns, + self._current_stat.st_mtime_ns)) + else: + os.utime(copier.dest_path, + (self._current_stat[stat.ST_MTIME], + self._current_stat[stat.ST_MTIME])) + except OSError as e: + msg = "%s %s utime failed unexpectedly: %s" % \ + (self.distfile, current_mirror.name, e) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + + self._success() + self.returncode = os.EX_OK + self.wait() + return + + self._try_next_mirror() + + def _fetch_uri(self, uri): + + if self.config.options.dry_run: + # Simply report success. + logging.info("dry-run: fetch '%s' from '%s'" % + (self.distfile, uri)) + self._success() + self.returncode = os.EX_OK + self.wait() + return + + if self.config.options.temp_dir: + self._fetch_tmp_dir_info = 'temp-dir' + distdir = self.config.options.temp_dir + else: + self._fetch_tmp_dir_info = 'distfiles' + distdir = self.config.options.distfiles + + tmp_basename = self.distfile + '._emirrordist_fetch_.%s' % os.getpid() + + variables = { + "DISTDIR": distdir, + "URI": uri, + "FILE": tmp_basename + } + + self._fetch_tmp_file = os.path.join(distdir, tmp_basename) + + try: + os.unlink(self._fetch_tmp_file) + except OSError: + pass + + args = portage.util.shlex_split(default_fetchcommand) + args = [portage.util.varexpand(x, mydict=variables) + for x in args] + + if sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000 and \ + not os.path.isabs(args[0]): + # Python 3.1 _execvp throws TypeError for non-absolute executable + # path passed as bytes (see http://bugs.python.org/issue8513). + fullname = portage.process.find_binary(args[0]) + if fullname is None: + raise portage.exception.CommandNotFound(args[0]) + args[0] = fullname + + args = [_unicode_encode(x, + encoding=_encodings['fs'], errors='strict') for x in args] + + null_fd = os.open(os.devnull, os.O_RDONLY) + fetcher = PopenProcess(background=self.background, + proc=subprocess.Popen(args, stdin=null_fd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT), + scheduler=self.scheduler) + os.close(null_fd) + + fetcher.pipe_reader = PipeLogger(background=self.background, + input_fd=fetcher.proc.stdout, log_file_path=self._log_path, + scheduler=self.scheduler) + + self._start_task(fetcher, self._fetcher_exit) + + def _fetcher_exit(self, fetcher): + + self._assert_current(fetcher) + if self._was_cancelled(): + self.wait() + return + + if os.path.exists(self._fetch_tmp_file): + self._start_task( + FileDigester(file_path=self._fetch_tmp_file, + hash_names=(self._select_hash(),), + background=self.background, + logfile=self._log_path), + self._fetch_digester_exit) + else: + self._try_next_mirror() + + def _fetch_digester_exit(self, digester): + + self._assert_current(digester) + if self._was_cancelled(): + self.wait() + return + + if digester.returncode != os.EX_OK: + msg = "%s %s digester failed unexpectedly" % \ + (self.distfile, self._fetch_tmp_dir_info) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + else: + bad_digest = self._find_bad_digest(digester.digests) + if bad_digest is not None: + msg = "%s has bad %s digest: expected %s, got %s" % \ + (self.distfile, bad_digest, + self.digests[bad_digest], digester.digests[bad_digest]) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + try: + os.unlink(self._fetch_tmp_file) + except OSError: + pass + else: + dest = os.path.join(self.config.options.distfiles, self.distfile) + try: + os.rename(self._fetch_tmp_file, dest) + except OSError: + self._start_task( + FileCopier(src_path=self._fetch_tmp_file, + dest_path=dest, + background=(self.background and + self._log_path is not None), + logfile=self._log_path), + self._fetch_copier_exit) + return + else: + self._success() + self.returncode = os.EX_OK + self.wait() + return + + self._try_next_mirror() + + def _fetch_copier_exit(self, copier): + + self._assert_current(copier) + + try: + os.unlink(self._fetch_tmp_file) + except OSError: + pass + + if self._was_cancelled(): + self.wait() + return + + if copier.returncode == os.EX_OK: + self._success() + self.returncode = os.EX_OK + self.wait() + else: + # out of space? + msg = "%s %s copy failed unexpectedly" % \ + (self.distfile, self._fetch_tmp_dir_info) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + self.config.log_failure("%s\t%s\t%s" % + (self.cpv, self.distfile, msg)) + self.config.file_failures[self.distfile] = self.cpv + self.returncode = 1 + self.wait() + + def _unlink_file(self, file_path, dir_info): + try: + os.unlink(file_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + msg = "unlink '%s' failed in %s: %s" % \ + (self.distfile, dir_info, e) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + return False + return True + + def _have_needed_digests(self): + return "size" in self.digests and \ + self._select_hash() is not None + + def _select_hash(self): + if default_hash_name in self.digests: + return default_hash_name + else: + for hash_name in self.digests: + if hash_name != "size" and \ + hash_name in portage.checksum.hashfunc_map: + return hash_name + + return None + + def _find_bad_digest(self, digests): + for hash_name, hash_value in digests.items(): + if self.digests[hash_name] != hash_value: + return hash_name + return None + + @staticmethod + def _same_device(path1, path2): + try: + st1 = os.stat(path1) + st2 = os.stat(path2) + except OSError: + return False + else: + return st1.st_dev == st2.st_dev + + def _hardlink_atomic(self, src, dest, dir_info): + + head, tail = os.path.split(dest) + hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \ + (tail, os.getpid())) + + try: + try: + os.link(src, hardlink_tmp) + except OSError as e: + if e.errno != errno.EXDEV: + msg = "hardlink %s from %s failed: %s" % \ + (self.distfile, dir_info, e) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + return False + + try: + os.rename(hardlink_tmp, dest) + except OSError as e: + msg = "hardlink rename '%s' from %s failed: %s" % \ + (self.distfile, dir_info, e) + self.scheduler.output(msg + '\n', background=True, + log_path=self._log_path) + logging.error(msg) + return False + finally: + try: + os.unlink(hardlink_tmp) + except OSError: + pass + + return True diff --git a/portage_with_autodep/pym/portage/_emirrordist/MirrorDistTask.py b/portage_with_autodep/pym/portage/_emirrordist/MirrorDistTask.py new file mode 100644 index 0000000..b6f875d --- /dev/null +++ b/portage_with_autodep/pym/portage/_emirrordist/MirrorDistTask.py @@ -0,0 +1,218 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import logging +import sys +import time + +try: + import threading +except ImportError: + import dummy_threading as threading + +import portage +from portage import os +from portage.util._async.TaskScheduler import TaskScheduler +from _emerge.CompositeTask import CompositeTask +from .FetchIterator import FetchIterator +from .DeletionIterator import DeletionIterator + +if sys.hexversion >= 0x3000000: + long = int + +class MirrorDistTask(CompositeTask): + + __slots__ = ('_config', '_terminated', '_term_check_id') + + def __init__(self, config): + CompositeTask.__init__(self, scheduler=config.event_loop) + self._config = config + self._terminated = threading.Event() + + def _start(self): + self._term_check_id = self.scheduler.idle_add(self._termination_check) + fetch = TaskScheduler(iter(FetchIterator(self._config)), + max_jobs=self._config.options.jobs, + max_load=self._config.options.load_average, + event_loop=self._config.event_loop) + self._start_task(fetch, self._fetch_exit) + + def _fetch_exit(self, fetch): + + self._assert_current(fetch) + if self._was_cancelled(): + self.wait() + return + + if self._config.options.delete: + deletion = TaskScheduler(iter(DeletionIterator(self._config)), + max_jobs=self._config.options.jobs, + max_load=self._config.options.load_average, + event_loop=self._config.event_loop) + self._start_task(deletion, self._deletion_exit) + return + + self._post_deletion() + + def _deletion_exit(self, deletion): + + self._assert_current(deletion) + if self._was_cancelled(): + self.wait() + return + + self._post_deletion() + + def _post_deletion(self): + + if self._config.options.recycle_db is not None: + self._update_recycle_db() + + if self._config.options.scheduled_deletion_log is not None: + self._scheduled_deletion_log() + + self._summary() + + self.returncode = os.EX_OK + self._current_task = None + self.wait() + + def _update_recycle_db(self): + + start_time = self._config.start_time + recycle_dir = self._config.options.recycle_dir + recycle_db = self._config.recycle_db + r_deletion_delay = self._config.options.recycle_deletion_delay + + # Use a dict optimize access. + recycle_db_cache = dict(recycle_db.items()) + + for filename in os.listdir(recycle_dir): + + recycle_file = os.path.join(recycle_dir, filename) + + try: + st = os.stat(recycle_file) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + logging.error(("stat failed for '%s' in " + "recycle: %s") % (filename, e)) + continue + + value = recycle_db_cache.pop(filename, None) + if value is None: + logging.debug(("add '%s' to " + "recycle db") % filename) + recycle_db[filename] = (st.st_size, start_time) + else: + r_size, r_time = value + if long(r_size) != st.st_size: + recycle_db[filename] = (st.st_size, start_time) + elif r_time + r_deletion_delay < start_time: + if self._config.options.dry_run: + logging.info(("dry-run: delete '%s' from " + "recycle") % filename) + logging.info(("drop '%s' from " + "recycle db") % filename) + else: + try: + os.unlink(recycle_file) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + logging.error(("delete '%s' from " + "recycle failed: %s") % (filename, e)) + else: + logging.debug(("delete '%s' from " + "recycle") % filename) + try: + del recycle_db[filename] + except KeyError: + pass + else: + logging.debug(("drop '%s' from " + "recycle db") % filename) + + # Existing files were popped from recycle_db_cache, + # so any remaining entries are for files that no + # longer exist. + for filename in recycle_db_cache: + try: + del recycle_db[filename] + except KeyError: + pass + else: + logging.debug(("drop non-existent '%s' from " + "recycle db") % filename) + + def _scheduled_deletion_log(self): + + start_time = self._config.start_time + dry_run = self._config.options.dry_run + deletion_delay = self._config.options.deletion_delay + distfiles_db = self._config.distfiles_db + + date_map = {} + for filename, timestamp in self._config.deletion_db.items(): + date = timestamp + deletion_delay + if date < start_time: + date = start_time + date = time.strftime("%Y-%m-%d", time.gmtime(date)) + date_files = date_map.get(date) + if date_files is None: + date_files = [] + date_map[date] = date_files + date_files.append(filename) + + if dry_run: + logging.warn(("dry-run: scheduled-deletions log " + "will be summarized via logging.info")) + + lines = [] + for date in sorted(date_map): + date_files = date_map[date] + if dry_run: + logging.info(("dry-run: scheduled deletions for %s: %s files") % + (date, len(date_files))) + lines.append("%s\n" % date) + for filename in date_files: + cpv = "unknown" + if distfiles_db is not None: + cpv = distfiles_db.get(filename, cpv) + lines.append("\t%s\t%s\n" % (filename, cpv)) + + if not dry_run: + portage.util.write_atomic( + self._config.options.scheduled_deletion_log, + "".join(lines)) + + def _summary(self): + elapsed_time = time.time() - self._config.start_time + fail_count = len(self._config.file_failures) + delete_count = self._config.delete_count + scheduled_deletion_count = self._config.scheduled_deletion_count - delete_count + added_file_count = self._config.added_file_count + added_byte_count = self._config.added_byte_count + + logging.info("finished in %i seconds" % elapsed_time) + logging.info("failed to fetch %i files" % fail_count) + logging.info("deleted %i files" % delete_count) + logging.info("deletion of %i files scheduled" % + scheduled_deletion_count) + logging.info("added %i files" % added_file_count) + logging.info("added %i bytes total" % added_byte_count) + + def terminate(self): + self._terminated.set() + + def _termination_check(self): + if self._terminated.is_set(): + self.cancel() + self.wait() + return True + + def _wait(self): + CompositeTask._wait(self) + if self._term_check_id is not None: + self.scheduler.source_remove(self._term_check_id) + self._term_check_id = None diff --git a/portage_with_autodep/pym/portage/tests/locks/__init__.py b/portage_with_autodep/pym/portage/_emirrordist/__init__.py index 21a391a..6cde932 100644 --- a/portage_with_autodep/pym/portage/tests/locks/__init__.py +++ b/portage_with_autodep/pym/portage/_emirrordist/__init__.py @@ -1,2 +1,2 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2013 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/_emirrordist/main.py b/portage_with_autodep/pym/portage/_emirrordist/main.py new file mode 100644 index 0000000..f28aad7 --- /dev/null +++ b/portage_with_autodep/pym/portage/_emirrordist/main.py @@ -0,0 +1,455 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import logging +import sys + +import portage +from portage import os +from portage.util import normalize_path, writemsg_level +from portage.util._argparse import ArgumentParser +from portage.util._async.run_main_scheduler import run_main_scheduler +from portage.util._async.SchedulerInterface import SchedulerInterface +from portage.util._eventloop.global_event_loop import global_event_loop +from .Config import Config +from .MirrorDistTask import MirrorDistTask + +if sys.hexversion >= 0x3000000: + long = int + +seconds_per_day = 24 * 60 * 60 + +common_options = ( + { + "longopt" : "--dry-run", + "help" : "perform a trial run with no changes made (usually combined " + "with --verbose)", + "action" : "store_true" + }, + { + "longopt" : "--verbose", + "shortopt" : "-v", + "help" : "display extra information on stderr " + "(multiple occurences increase verbosity)", + "action" : "count", + "default" : 0, + }, + { + "longopt" : "--ignore-default-opts", + "help" : "do not use the EMIRRORDIST_DEFAULT_OPTS environment variable", + "action" : "store_true" + }, + { + "longopt" : "--distfiles", + "help" : "distfiles directory to use (required)", + "metavar" : "DIR" + }, + { + "longopt" : "--jobs", + "shortopt" : "-j", + "help" : "number of concurrent jobs to run", + "type" : int + }, + { + "longopt" : "--load-average", + "shortopt" : "-l", + "help" : "load average limit for spawning of new concurrent jobs", + "metavar" : "LOAD", + "type" : float + }, + { + "longopt" : "--tries", + "help" : "maximum number of tries per file, 0 means unlimited (default is 10)", + "default" : 10, + "type" : int + }, + { + "longopt" : "--repo", + "help" : "name of repo to operate on" + }, + { + "longopt" : "--config-root", + "help" : "location of portage config files", + "metavar" : "DIR" + }, + { + "longopt" : "--portdir", + "help" : "override the PORTDIR variable (deprecated in favor of --repositories-configuration)", + "metavar" : "DIR" + }, + { + "longopt" : "--portdir-overlay", + "help" : "override the PORTDIR_OVERLAY variable (deprecated in favor of --repositories-configuration)" + }, + { + "longopt" : "--repositories-configuration", + "help" : "override configuration of repositories (in format of repos.conf)" + }, + { + "longopt" : "--strict-manifests", + "help" : "manually override \"strict\" FEATURES setting", + "choices" : ("y", "n"), + "metavar" : "<y|n>", + }, + { + "longopt" : "--failure-log", + "help" : "log file for fetch failures, with tab-delimited " + "output, for reporting purposes", + "metavar" : "FILE" + }, + { + "longopt" : "--success-log", + "help" : "log file for fetch successes, with tab-delimited " + "output, for reporting purposes", + "metavar" : "FILE" + }, + { + "longopt" : "--scheduled-deletion-log", + "help" : "log file for scheduled deletions, with tab-delimited " + "output, for reporting purposes", + "metavar" : "FILE" + }, + { + "longopt" : "--delete", + "help" : "enable deletion of unused distfiles", + "action" : "store_true" + }, + { + "longopt" : "--deletion-db", + "help" : "database file used to track lifetime of files " + "scheduled for delayed deletion", + "metavar" : "FILE" + }, + { + "longopt" : "--deletion-delay", + "help" : "delay time for deletion, measured in seconds", + "metavar" : "SECONDS" + }, + { + "longopt" : "--temp-dir", + "help" : "temporary directory for downloads", + "metavar" : "DIR" + }, + { + "longopt" : "--mirror-overrides", + "help" : "file holding a list of mirror overrides", + "metavar" : "FILE" + }, + { + "longopt" : "--mirror-skip", + "help" : "comma delimited list of mirror targets to skip " + "when fetching" + }, + { + "longopt" : "--restrict-mirror-exemptions", + "help" : "comma delimited list of mirror targets for which to " + "ignore RESTRICT=\"mirror\"" + }, + { + "longopt" : "--verify-existing-digest", + "help" : "use digest as a verification of whether existing " + "distfiles are valid", + "action" : "store_true" + }, + { + "longopt" : "--distfiles-local", + "help" : "distfiles-local directory to use", + "metavar" : "DIR" + }, + { + "longopt" : "--distfiles-db", + "help" : "database file used to track which ebuilds a " + "distfile belongs to", + "metavar" : "FILE" + }, + { + "longopt" : "--recycle-dir", + "help" : "directory for extended retention of files that " + "are removed from distdir with the --delete option", + "metavar" : "DIR" + }, + { + "longopt" : "--recycle-db", + "help" : "database file used to track lifetime of files " + "in recycle dir", + "metavar" : "FILE" + }, + { + "longopt" : "--recycle-deletion-delay", + "help" : "delay time for deletion of unused files from " + "recycle dir, measured in seconds (defaults to " + "the equivalent of 60 days)", + "default" : 60 * seconds_per_day, + "metavar" : "SECONDS", + "type" : int + }, + { + "longopt" : "--fetch-log-dir", + "help" : "directory for individual fetch logs", + "metavar" : "DIR" + }, + { + "longopt" : "--whitelist-from", + "help" : "specifies a file containing a list of files to " + "whitelist, one per line, # prefixed lines ignored", + "action" : "append", + "metavar" : "FILE" + }, +) + +def parse_args(args): + description = "emirrordist - a fetch tool for mirroring " \ + "of package distfiles" + usage = "emirrordist [options] <action>" + parser = ArgumentParser(description=description, usage=usage) + + actions = parser.add_argument_group('Actions') + actions.add_argument("--version", + action="store_true", + help="display portage version and exit") + actions.add_argument("--mirror", + action="store_true", + help="mirror distfiles for the selected repository") + + common = parser.add_argument_group('Common options') + for opt_info in common_options: + opt_pargs = [opt_info["longopt"]] + if opt_info.get("shortopt"): + opt_pargs.append(opt_info["shortopt"]) + opt_kwargs = {"help" : opt_info["help"]} + for k in ("action", "choices", "default", "metavar", "type"): + if k in opt_info: + opt_kwargs[k] = opt_info[k] + common.add_argument(*opt_pargs, **opt_kwargs) + + options, args = parser.parse_known_args(args) + + return (parser, options, args) + +def emirrordist_main(args): + + # The calling environment is ignored, so the program is + # completely controlled by commandline arguments. + env = {} + + if not sys.stdout.isatty(): + portage.output.nocolor() + env['NOCOLOR'] = 'true' + + parser, options, args = parse_args(args) + + if options.version: + sys.stdout.write("Portage %s\n" % portage.VERSION) + return os.EX_OK + + config_root = options.config_root + + if options.portdir is not None: + writemsg_level("emirrordist: warning: --portdir option is deprecated in favor of --repositories-configuration option\n", + level=logging.WARNING, noiselevel=-1) + if options.portdir_overlay is not None: + writemsg_level("emirrordist: warning: --portdir-overlay option is deprecated in favor of --repositories-configuration option\n", + level=logging.WARNING, noiselevel=-1) + + if options.repositories_configuration is not None: + env['PORTAGE_REPOSITORIES'] = options.repositories_configuration + elif options.portdir_overlay is not None: + env['PORTDIR_OVERLAY'] = options.portdir_overlay + + if options.portdir is not None: + env['PORTDIR'] = options.portdir + + settings = portage.config(config_root=config_root, + local_config=False, env=env) + + default_opts = None + if not options.ignore_default_opts: + default_opts = settings.get('EMIRRORDIST_DEFAULT_OPTS', '').split() + + if default_opts: + parser, options, args = parse_args(default_opts + args) + + settings = portage.config(config_root=config_root, + local_config=False, env=env) + + if options.repo is None: + if len(settings.repositories.prepos) == 2: + for repo in settings.repositories: + if repo.name != "DEFAULT": + options.repo = repo.name + break + + if options.repo is None: + parser.error("--repo option is required") + + repo_path = settings.repositories.treemap.get(options.repo) + if repo_path is None: + parser.error("Unable to locate repository named '%s'" % (options.repo,)) + + if options.jobs is not None: + options.jobs = int(options.jobs) + + if options.load_average is not None: + options.load_average = float(options.load_average) + + if options.failure_log is not None: + options.failure_log = normalize_path( + os.path.abspath(options.failure_log)) + + parent_dir = os.path.dirname(options.failure_log) + if not (os.path.isdir(parent_dir) and + os.access(parent_dir, os.W_OK|os.X_OK)): + parser.error(("--failure-log '%s' parent is not a " + "writable directory") % options.failure_log) + + if options.success_log is not None: + options.success_log = normalize_path( + os.path.abspath(options.success_log)) + + parent_dir = os.path.dirname(options.success_log) + if not (os.path.isdir(parent_dir) and + os.access(parent_dir, os.W_OK|os.X_OK)): + parser.error(("--success-log '%s' parent is not a " + "writable directory") % options.success_log) + + if options.scheduled_deletion_log is not None: + options.scheduled_deletion_log = normalize_path( + os.path.abspath(options.scheduled_deletion_log)) + + parent_dir = os.path.dirname(options.scheduled_deletion_log) + if not (os.path.isdir(parent_dir) and + os.access(parent_dir, os.W_OK|os.X_OK)): + parser.error(("--scheduled-deletion-log '%s' parent is not a " + "writable directory") % options.scheduled_deletion_log) + + if options.deletion_db is None: + parser.error("--scheduled-deletion-log requires --deletion-db") + + if options.deletion_delay is not None: + options.deletion_delay = long(options.deletion_delay) + if options.deletion_db is None: + parser.error("--deletion-delay requires --deletion-db") + + if options.deletion_db is not None: + if options.deletion_delay is None: + parser.error("--deletion-db requires --deletion-delay") + options.deletion_db = normalize_path( + os.path.abspath(options.deletion_db)) + + if options.temp_dir is not None: + options.temp_dir = normalize_path( + os.path.abspath(options.temp_dir)) + + if not (os.path.isdir(options.temp_dir) and + os.access(options.temp_dir, os.W_OK|os.X_OK)): + parser.error(("--temp-dir '%s' is not a " + "writable directory") % options.temp_dir) + + if options.distfiles is not None: + options.distfiles = normalize_path( + os.path.abspath(options.distfiles)) + + if not (os.path.isdir(options.distfiles) and + os.access(options.distfiles, os.W_OK|os.X_OK)): + parser.error(("--distfiles '%s' is not a " + "writable directory") % options.distfiles) + else: + parser.error("missing required --distfiles parameter") + + if options.mirror_overrides is not None: + options.mirror_overrides = normalize_path( + os.path.abspath(options.mirror_overrides)) + + if not (os.access(options.mirror_overrides, os.R_OK) and + os.path.isfile(options.mirror_overrides)): + parser.error( + "--mirror-overrides-file '%s' is not a readable file" % + options.mirror_overrides) + + if options.distfiles_local is not None: + options.distfiles_local = normalize_path( + os.path.abspath(options.distfiles_local)) + + if not (os.path.isdir(options.distfiles_local) and + os.access(options.distfiles_local, os.W_OK|os.X_OK)): + parser.error(("--distfiles-local '%s' is not a " + "writable directory") % options.distfiles_local) + + if options.distfiles_db is not None: + options.distfiles_db = normalize_path( + os.path.abspath(options.distfiles_db)) + + if options.tries is not None: + options.tries = int(options.tries) + + if options.recycle_dir is not None: + options.recycle_dir = normalize_path( + os.path.abspath(options.recycle_dir)) + if not (os.path.isdir(options.recycle_dir) and + os.access(options.recycle_dir, os.W_OK|os.X_OK)): + parser.error(("--recycle-dir '%s' is not a " + "writable directory") % options.recycle_dir) + + if options.recycle_db is not None: + if options.recycle_dir is None: + parser.error("--recycle-db requires " + "--recycle-dir to be specified") + options.recycle_db = normalize_path( + os.path.abspath(options.recycle_db)) + + if options.recycle_deletion_delay is not None: + options.recycle_deletion_delay = \ + long(options.recycle_deletion_delay) + + if options.fetch_log_dir is not None: + options.fetch_log_dir = normalize_path( + os.path.abspath(options.fetch_log_dir)) + + if not (os.path.isdir(options.fetch_log_dir) and + os.access(options.fetch_log_dir, os.W_OK|os.X_OK)): + parser.error(("--fetch-log-dir '%s' is not a " + "writable directory") % options.fetch_log_dir) + + if options.whitelist_from: + normalized_paths = [] + for x in options.whitelist_from: + path = normalize_path(os.path.abspath(x)) + normalized_paths.append(path) + if not (os.access(path, os.R_OK) and os.path.isfile(path)): + parser.error( + "--whitelist-from '%s' is not a readable file" % x) + options.whitelist_from = normalized_paths + + if options.strict_manifests is not None: + if options.strict_manifests == "y": + settings.features.add("strict") + else: + settings.features.discard("strict") + + settings.lock() + + portdb = portage.portdbapi(mysettings=settings) + + # Limit ebuilds to the specified repo. + portdb.porttrees = [repo_path] + + portage.util.initialize_logger() + + if options.verbose > 0: + l = logging.getLogger() + l.setLevel(l.getEffectiveLevel() - 10 * options.verbose) + + with Config(options, portdb, + SchedulerInterface(global_event_loop())) as config: + + if not options.mirror: + parser.error('No action specified') + + returncode = os.EX_OK + + if options.mirror: + signum = run_main_scheduler(MirrorDistTask(config)) + if signum is not None: + sys.exit(128 + signum) + + return returncode diff --git a/portage_with_autodep/pym/portage/_global_updates.py b/portage_with_autodep/pym/portage/_global_updates.py index 868d1ee..5175043 100644 --- a/portage_with_autodep/pym/portage/_global_updates.py +++ b/portage_with_autodep/pym/portage/_global_updates.py @@ -15,7 +15,7 @@ from portage.update import grab_updates, parse_updates, update_config_files, upd from portage.util import grabfile, shlex_split, \ writemsg, writemsg_stdout, write_atomic -def _global_updates(trees, prev_mtimes, quiet=False): +def _global_updates(trees, prev_mtimes, quiet=False, if_mtime_changed=True): """ Perform new global updates if they exist in 'profiles/updates/' subdirectories of all active repositories (PORTDIR + PORTDIR_OVERLAY). @@ -37,7 +37,7 @@ def _global_updates(trees, prev_mtimes, quiet=False): "SANDBOX_ACTIVE" in os.environ or \ len(trees) != 1: return retupd - root = "/" + root = trees._running_eroot mysettings = trees[root]["vartree"].settings portdb = trees[root]["porttree"].dbapi vardb = trees[root]["vartree"].dbapi @@ -73,10 +73,10 @@ def _global_updates(trees, prev_mtimes, quiet=False): continue try: - if mysettings.get("PORTAGE_CALLER") == "fixpackages": - update_data = grab_updates(updpath) + if if_mtime_changed: + update_data = grab_updates(updpath, prev_mtimes=prev_mtimes) else: - update_data = grab_updates(updpath, prev_mtimes) + update_data = grab_updates(updpath) except DirectoryNotFound: continue myupd = [] @@ -217,8 +217,7 @@ def _global_updates(trees, prev_mtimes, quiet=False): do_upgrade_packagesmessage = False # We gotta do the brute force updates for these now. - if mysettings.get("PORTAGE_CALLER") == "fixpackages" or \ - "fixpackages" in mysettings.features: + if True: def onUpdate(maxval, curval): if curval > 0: writemsg_stdout("#") diff --git a/portage_with_autodep/pym/portage/_global_updates.pyo b/portage_with_autodep/pym/portage/_global_updates.pyo Binary files differnew file mode 100644 index 0000000..3e2e8de --- /dev/null +++ b/portage_with_autodep/pym/portage/_global_updates.pyo diff --git a/portage_with_autodep/pym/portage/_legacy_globals.py b/portage_with_autodep/pym/portage/_legacy_globals.py index 615591a..abffa0e 100644 --- a/portage_with_autodep/pym/portage/_legacy_globals.py +++ b/portage_with_autodep/pym/portage/_legacy_globals.py @@ -35,19 +35,14 @@ def _get_legacy_global(name): constructed.add('db') del portage._initializing_globals - settings = portage.db["/"]["vartree"].settings - - for root in portage.db: - if root != "/": - settings = portage.db[root]["vartree"].settings - break - - portage.output._init(config_root=settings['PORTAGE_CONFIGROOT']) + settings = portage.db[portage.db._target_eroot]["vartree"].settings portage.settings = settings constructed.add('settings') - portage.root = root + # Since portage.db now uses EROOT for keys instead of ROOT, we make + # portage.root refer to EROOT such that it continues to work as a key. + portage.root = portage.db._target_eroot constructed.add('root') # COMPATIBILITY diff --git a/portage_with_autodep/pym/portage/_legacy_globals.pyo b/portage_with_autodep/pym/portage/_legacy_globals.pyo Binary files differnew file mode 100644 index 0000000..2e50cbe --- /dev/null +++ b/portage_with_autodep/pym/portage/_legacy_globals.pyo diff --git a/portage_with_autodep/pym/portage/_selinux.pyo b/portage_with_autodep/pym/portage/_selinux.pyo Binary files differnew file mode 100644 index 0000000..7a413e0 --- /dev/null +++ b/portage_with_autodep/pym/portage/_selinux.pyo diff --git a/portage_with_autodep/pym/portage/_sets/__init__.py b/portage_with_autodep/pym/portage/_sets/__init__.py index 1b3484e..88a4b3b 100644 --- a/portage_with_autodep/pym/portage/_sets/__init__.py +++ b/portage_with_autodep/pym/portage/_sets/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2007 Gentoo Foundation +# Copyright 2007-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -6,16 +6,27 @@ from __future__ import print_function __all__ = ["SETPREFIX", "get_boolean", "SetConfigError", "SetConfig", "load_default_config"] +import io +import logging +import sys try: - from configparser import SafeConfigParser, NoOptionError + from configparser import NoOptionError, ParsingError + if sys.hexversion >= 0x3020000: + from configparser import ConfigParser as SafeConfigParser + else: + from configparser import SafeConfigParser except ImportError: - from ConfigParser import SafeConfigParser, NoOptionError + from ConfigParser import SafeConfigParser, NoOptionError, ParsingError from portage import os from portage import load_mod +from portage import _unicode_decode +from portage import _unicode_encode +from portage import _encodings from portage.const import USER_CONFIG_PATH, GLOBAL_CONFIG_PATH from portage.const import _ENABLE_SET_CONFIG from portage.exception import PackageSetNotFound from portage.localization import _ +from portage.util import writemsg_level SETPREFIX = "@" @@ -43,7 +54,32 @@ class SetConfig(object): }) if _ENABLE_SET_CONFIG: - self._parser.read(paths) + # use read_file/readfp in order to control decoding of unicode + try: + # Python >=3.2 + read_file = self._parser.read_file + except AttributeError: + read_file = self._parser.readfp + + for p in paths: + f = None + try: + f = io.open(_unicode_encode(p, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + except EnvironmentError: + pass + else: + try: + read_file(f) + except ParsingError as e: + writemsg_level(_unicode_decode( + _("!!! Error while reading sets config file: %s\n") + ) % e, level=logging.ERROR, noiselevel=-1) + finally: + if f is not None: + f.close() else: self._create_default_config() @@ -201,7 +237,6 @@ class SetConfig(object): except KeyError: raise PackageSetNotFound(setname) myatoms = myset.getAtoms() - parser = self._parser if ignorelist is None: ignorelist = set() diff --git a/portage_with_autodep/pym/portage/_sets/__init__.pyo b/portage_with_autodep/pym/portage/_sets/__init__.pyo Binary files differnew file mode 100644 index 0000000..5318dbe --- /dev/null +++ b/portage_with_autodep/pym/portage/_sets/__init__.pyo diff --git a/portage_with_autodep/pym/portage/_sets/base.pyo b/portage_with_autodep/pym/portage/_sets/base.pyo Binary files differnew file mode 100644 index 0000000..89e53be --- /dev/null +++ b/portage_with_autodep/pym/portage/_sets/base.pyo diff --git a/portage_with_autodep/pym/portage/_sets/dbapi.py b/portage_with_autodep/pym/portage/_sets/dbapi.py index 0f238f0..4982a92 100644 --- a/portage_with_autodep/pym/portage/_sets/dbapi.py +++ b/portage_with_autodep/pym/portage/_sets/dbapi.py @@ -1,10 +1,10 @@ -# Copyright 2007-2010 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import time from portage import os -from portage.versions import catpkgsplit, catsplit, pkgcmp, best +from portage.versions import best, catsplit, vercmp from portage.dep import Atom from portage.localization import _ from portage._sets.base import PackageSet @@ -72,18 +72,16 @@ class OwnerSet(PackageSet): aux_keys = ["SLOT"] if exclude_paths is None: for link, p in vardb._owners.iter_owners(paths): - cat, pn = catpkgsplit(link.mycpv)[:2] slot, = aux_get(link.mycpv, aux_keys) - rValue.add("%s/%s:%s" % (cat, pn, slot)) + rValue.add("%s:%s" % (link.mycpv.cp, slot)) else: all_paths = set() all_paths.update(paths) all_paths.update(exclude_paths) exclude_atoms = set() for link, p in vardb._owners.iter_owners(all_paths): - cat, pn = catpkgsplit(link.mycpv)[:2] slot, = aux_get(link.mycpv, aux_keys) - atom = "%s/%s:%s" % (cat, pn, slot) + atom = "%s:%s" % (link.mycpv.cp, slot) rValue.add(atom) if p in exclude_paths: exclude_atoms.add(atom) @@ -184,9 +182,7 @@ class DowngradeSet(PackageSet): ebuild = xmatch(xmatch_level, slot_atom) if not ebuild: continue - ebuild_split = catpkgsplit(ebuild)[1:] - installed_split = catpkgsplit(cpv)[1:] - if pkgcmp(installed_split, ebuild_split) > 0: + if vercmp(cpv.version, ebuild.version) > 0: atoms.append(slot_atom) self._setAtoms(atoms) diff --git a/portage_with_autodep/pym/portage/_sets/dbapi.pyo b/portage_with_autodep/pym/portage/_sets/dbapi.pyo Binary files differnew file mode 100644 index 0000000..20bf848 --- /dev/null +++ b/portage_with_autodep/pym/portage/_sets/dbapi.pyo diff --git a/portage_with_autodep/pym/portage/_sets/files.pyo b/portage_with_autodep/pym/portage/_sets/files.pyo Binary files differnew file mode 100644 index 0000000..eb03c00 --- /dev/null +++ b/portage_with_autodep/pym/portage/_sets/files.pyo diff --git a/portage_with_autodep/pym/portage/_sets/libs.pyo b/portage_with_autodep/pym/portage/_sets/libs.pyo Binary files differnew file mode 100644 index 0000000..72fc1bb --- /dev/null +++ b/portage_with_autodep/pym/portage/_sets/libs.pyo diff --git a/portage_with_autodep/pym/portage/_sets/profiles.pyo b/portage_with_autodep/pym/portage/_sets/profiles.pyo Binary files differnew file mode 100644 index 0000000..9502044 --- /dev/null +++ b/portage_with_autodep/pym/portage/_sets/profiles.pyo diff --git a/portage_with_autodep/pym/portage/_sets/security.py b/portage_with_autodep/pym/portage/_sets/security.py index 2d8fcf6..7e856bc 100644 --- a/portage_with_autodep/pym/portage/_sets/security.py +++ b/portage_with_autodep/pym/portage/_sets/security.py @@ -1,9 +1,9 @@ -# Copyright 2007 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage.glsa as glsa from portage._sets.base import PackageSet -from portage.versions import catpkgsplit, pkgcmp +from portage.versions import vercmp from portage._sets import get_boolean __all__ = ["SecuritySet", "NewGlsaSet", "NewAffectedSet", "AffectedSet"] @@ -45,12 +45,12 @@ class SecuritySet(PackageSet): for atom in atomlist[:]: cpv = self._portdbapi.xmatch("match-all", atom)[0] slot = self._portdbapi.aux_get(cpv, ["SLOT"])[0] - cps = "/".join(catpkgsplit(cpv)[0:2]) + ":" + slot + cps = "%s:%s" % (cpv.cp, slot) if not cps in mydict: mydict[cps] = (atom, cpv) else: other_cpv = mydict[cps][1] - if pkgcmp(catpkgsplit(cpv)[1:], catpkgsplit(other_cpv)[1:]) > 0: + if vercmp(cpv.version, other_cpv.version) > 0: atomlist.remove(mydict[cps][0]) mydict[cps] = (atom, cpv) return atomlist diff --git a/portage_with_autodep/pym/portage/_sets/security.pyo b/portage_with_autodep/pym/portage/_sets/security.pyo Binary files differnew file mode 100644 index 0000000..ea67514 --- /dev/null +++ b/portage_with_autodep/pym/portage/_sets/security.pyo diff --git a/portage_with_autodep/pym/portage/_sets/shell.pyo b/portage_with_autodep/pym/portage/_sets/shell.pyo Binary files differnew file mode 100644 index 0000000..e5e4561 --- /dev/null +++ b/portage_with_autodep/pym/portage/_sets/shell.pyo diff --git a/portage_with_autodep/pym/portage/cache/__init__.pyo b/portage_with_autodep/pym/portage/cache/__init__.pyo Binary files differnew file mode 100644 index 0000000..eb5a90e --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/__init__.pyo diff --git a/portage_with_autodep/pym/portage/cache/anydbm.pyo b/portage_with_autodep/pym/portage/cache/anydbm.pyo Binary files differnew file mode 100644 index 0000000..5946da9 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/anydbm.pyo diff --git a/portage_with_autodep/pym/portage/cache/cache_errors.pyo b/portage_with_autodep/pym/portage/cache/cache_errors.pyo Binary files differnew file mode 100644 index 0000000..866088e --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/cache_errors.pyo diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py index 6b388fa..0086e40 100644 --- a/portage_with_autodep/pym/portage/cache/ebuild_xattr.py +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.py @@ -1,5 +1,6 @@ +# -*- coding: UTF8 -*- # Copyright: 2009-2011 Gentoo Foundation -# Author(s): Petteri Räty (betelgeuse@gentoo.org) +# Author(s): Petteri Räty (betelgeuse@gentoo.org) # License: GPL2 __all__ = ['database'] diff --git a/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo Binary files differnew file mode 100644 index 0000000..fe32dcc --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/ebuild_xattr.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.py b/portage_with_autodep/pym/portage/cache/flat_hash.py index b6bc074..2eae9f6 100644 --- a/portage_with_autodep/pym/portage/cache/flat_hash.py +++ b/portage_with_autodep/pym/portage/cache/flat_hash.py @@ -31,7 +31,7 @@ class database(fs_template.FsBased): self.label.lstrip(os.path.sep).rstrip(os.path.sep)) write_keys = set(self._known_keys) write_keys.add("_eclasses_") - write_keys.add("_mtime_") + write_keys.add("_%s_" % (self.validation_chf,)) self._write_keys = sorted(write_keys) if not self.readonly and not os.path.exists(self.location): self._ensure_dirs() @@ -69,7 +69,6 @@ class database(fs_template.FsBased): raise cache_errors.CacheCorruption(cpv, e) def _setitem(self, cpv, values): -# import pdb;pdb.set_trace() s = cpv.rfind("/") fp = os.path.join(self.location,cpv[:s],".update.%i.%s" % (os.getpid(), cpv[s+1:])) try: @@ -153,3 +152,9 @@ class database(fs_template.FsBased): dirs.append((depth+1, p)) continue yield p[len_base+1:] + + +class md5_database(database): + + validation_chf = 'md5' + store_eclass_paths = False diff --git a/portage_with_autodep/pym/portage/cache/flat_hash.pyo b/portage_with_autodep/pym/portage/cache/flat_hash.pyo Binary files differnew file mode 100644 index 0000000..4f568a8 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/flat_hash.pyo diff --git a/portage_with_autodep/pym/portage/cache/flat_list.pyo b/portage_with_autodep/pym/portage/cache/flat_list.pyo Binary files differnew file mode 100644 index 0000000..ab7dc82 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/flat_list.pyo diff --git a/portage_with_autodep/pym/portage/cache/fs_template.pyo b/portage_with_autodep/pym/portage/cache/fs_template.pyo Binary files differnew file mode 100644 index 0000000..6cbbc2f --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/fs_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/mappings.py b/portage_with_autodep/pym/portage/cache/mappings.py index 60a918e..bc8ce9a 100644 --- a/portage_with_autodep/pym/portage/cache/mappings.py +++ b/portage_with_autodep/pym/portage/cache/mappings.py @@ -316,7 +316,7 @@ def slot_dict_class(keys, prefix="_val_"): attribute names from keys @type prefix: String @rtype: SlotDict - @returns: A class that constructs SlotDict instances + @return: A class that constructs SlotDict instances having the specified keys. """ if isinstance(keys, frozenset): diff --git a/portage_with_autodep/pym/portage/cache/mappings.pyo b/portage_with_autodep/pym/portage/cache/mappings.pyo Binary files differnew file mode 100644 index 0000000..1eb3f4f --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/mappings.pyo diff --git a/portage_with_autodep/pym/portage/cache/metadata.py b/portage_with_autodep/pym/portage/cache/metadata.py index 4c735d7..9d2c3a5 100644 --- a/portage_with_autodep/pym/portage/cache/metadata.py +++ b/portage_with_autodep/pym/portage/cache/metadata.py @@ -6,6 +6,7 @@ import errno import re import stat import sys +from operator import attrgetter from portage import os from portage import _encodings from portage import _unicode_encode @@ -63,13 +64,14 @@ class database(flat_hash.database): if "INHERITED" in d: if self.ec is None: self.ec = portage.eclass_cache.cache(self.location[:-15]) + getter = attrgetter(self.validation_chf) try: - d["_eclasses_"] = self.ec.get_eclass_data( - d["INHERITED"].split()) + ec_data = self.ec.get_eclass_data(d["INHERITED"].split()) + d["_eclasses_"] = dict((k, (v.eclass_dir, getter(v))) + for k,v in ec_data.items()) except KeyError as e: # INHERITED contains a non-existent eclass. raise cache_errors.CacheCorruption(cpv, e) - del d["INHERITED"] else: d["_eclasses_"] = {} elif isinstance(d["_eclasses_"], basestring): diff --git a/portage_with_autodep/pym/portage/cache/metadata.pyo b/portage_with_autodep/pym/portage/cache/metadata.pyo Binary files differnew file mode 100644 index 0000000..c98445b --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/metadata.pyo diff --git a/portage_with_autodep/pym/portage/cache/metadata_overlay.py b/portage_with_autodep/pym/portage/cache/metadata_overlay.py deleted file mode 100644 index cfa0051..0000000 --- a/portage_with_autodep/pym/portage/cache/metadata_overlay.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 1999-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.cache import template -from portage.cache.cache_errors import CacheCorruption -from portage.cache.flat_hash import database as db_rw -from portage.cache.metadata import database as db_ro - -class database(template.database): - - serialize_eclasses = False - - def __init__(self, location, label, auxdbkeys, db_rw=db_rw, db_ro=db_ro, - *args, **config): - super_config = config.copy() - super_config.pop("gid", None) - super_config.pop("perms", None) - super(database, self).__init__(location, label, auxdbkeys, - *args, **super_config) - self.db_rw = db_rw(location, label, auxdbkeys, **config) - self.commit = self.db_rw.commit - self.autocommits = self.db_rw.autocommits - if isinstance(db_ro, type): - ro_config = config.copy() - ro_config["readonly"] = True - self.db_ro = db_ro(label, "metadata/cache", auxdbkeys, **ro_config) - else: - self.db_ro = db_ro - - def __getitem__(self, cpv): - """funnel whiteout validation through here, since value needs to be fetched""" - try: - value = self.db_rw[cpv] - except KeyError: - return self.db_ro[cpv] # raises a KeyError when necessary - except CacheCorruption: - del self.db_rw[cpv] - return self.db_ro[cpv] # raises a KeyError when necessary - if self._is_whiteout(value): - if self._is_whiteout_valid(cpv, value): - raise KeyError(cpv) - else: - del self.db_rw[cpv] - return self.db_ro[cpv] # raises a KeyError when necessary - else: - return value - - def _setitem(self, name, values): - try: - value_ro = self.db_ro.get(name) - except CacheCorruption: - value_ro = None - if value_ro is not None and \ - self._are_values_identical(value_ro, values): - # we have matching values in the underlying db_ro - # so it is unnecessary to store data in db_rw - try: - del self.db_rw[name] # delete unwanted whiteout when necessary - except KeyError: - pass - return - self.db_rw[name] = values - - def _delitem(self, cpv): - value = self[cpv] # validates whiteout and/or raises a KeyError when necessary - if cpv in self.db_ro: - self.db_rw[cpv] = self._create_whiteout(value) - else: - del self.db_rw[cpv] - - def __contains__(self, cpv): - try: - self[cpv] # validates whiteout when necessary - except KeyError: - return False - return True - - def __iter__(self): - s = set() - for cpv in self.db_rw: - if cpv in self: # validates whiteout when necessary - yield cpv - # set includes whiteouts so they won't be yielded later - s.add(cpv) - for cpv in self.db_ro: - if cpv not in s: - yield cpv - - def _is_whiteout(self, value): - return value["EAPI"] == "whiteout" - - def _create_whiteout(self, value): - return {"EAPI":"whiteout","_eclasses_":value["_eclasses_"],"_mtime_":value["_mtime_"]} - - def _is_whiteout_valid(self, name, value_rw): - try: - value_ro = self.db_ro[name] - return self._are_values_identical(value_rw,value_ro) - except KeyError: - return False - - def _are_values_identical(self, value1, value2): - if value1['_mtime_'] != value2['_mtime_']: - return False - return value1["_eclasses_"] == value2["_eclasses_"] diff --git a/portage_with_autodep/pym/portage/cache/sql_template.pyo b/portage_with_autodep/pym/portage/cache/sql_template.pyo Binary files differnew file mode 100644 index 0000000..e2c5974 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/sql_template.pyo diff --git a/portage_with_autodep/pym/portage/cache/sqlite.pyo b/portage_with_autodep/pym/portage/cache/sqlite.pyo Binary files differnew file mode 100644 index 0000000..a82d25f --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/sqlite.pyo diff --git a/portage_with_autodep/pym/portage/cache/template.py b/portage_with_autodep/pym/portage/cache/template.py index f84d8f4..cf1e8ae 100644 --- a/portage_with_autodep/pym/portage/cache/template.py +++ b/portage_with_autodep/pym/portage/cache/template.py @@ -1,4 +1,4 @@ -# Copyright: 2005 Gentoo Foundation +# Copyright: 2005-2012 Gentoo Foundation # Author(s): Brian Harring (ferringb@gentoo.org) # License: GPL2 @@ -7,10 +7,14 @@ from portage.cache.cache_errors import InvalidRestriction from portage.cache.mappings import ProtectedDict import sys import warnings +import operator if sys.hexversion >= 0x3000000: + _unicode = str basestring = str long = int +else: + _unicode = unicode class database(object): # this is for metadata/cache transfer. @@ -21,6 +25,8 @@ class database(object): autocommits = False cleanse_keys = False serialize_eclasses = True + validation_chf = 'mtime' + store_eclass_paths = True def __init__(self, location, label, auxdbkeys, readonly=False): """ initialize the derived class; specifically, store label/keys""" @@ -40,9 +46,15 @@ class database(object): self.updates = 0 d=self._getitem(cpv) if self.serialize_eclasses and "_eclasses_" in d: - d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"]) + d["_eclasses_"] = reconstruct_eclasses(cpv, d["_eclasses_"], + self.validation_chf, paths=self.store_eclass_paths) elif "_eclasses_" not in d: d["_eclasses_"] = {} + # Never return INHERITED, since portdbapi.aux_get() will + # generate it automatically from _eclasses_, and we want + # to omit it in comparisons between cache entries like + # those that egencache uses to avoid redundant writes. + d.pop("INHERITED", None) mtime = d.get('_mtime_') if mtime is None: raise cache_errors.CacheCorruption(cpv, @@ -60,22 +72,46 @@ class database(object): override this in derived classess""" raise NotImplementedError + @staticmethod + def _internal_eclasses(extern_ec_dict, chf_type, paths): + """ + When serialize_eclasses is False, we have to convert an external + eclass dict containing hashed_path objects into an appropriate + internal dict containing values of chf_type (and eclass dirs + if store_eclass_paths is True). + """ + if not extern_ec_dict: + return extern_ec_dict + chf_getter = operator.attrgetter(chf_type) + if paths: + intern_ec_dict = dict((k, (v.eclass_dir, chf_getter(v))) + for k, v in extern_ec_dict.items()) + else: + intern_ec_dict = dict((k, chf_getter(v)) + for k, v in extern_ec_dict.items()) + return intern_ec_dict + def __setitem__(self, cpv, values): """set a cpv to values This shouldn't be overriden in derived classes since it handles the readonly checks""" if self.readonly: raise cache_errors.ReadOnlyRestriction() + d = None if self.cleanse_keys: d=ProtectedDict(values) for k, v in list(d.items()): if not v: del d[k] - if self.serialize_eclasses and "_eclasses_" in values: - d["_eclasses_"] = serialize_eclasses(d["_eclasses_"]) - elif self.serialize_eclasses and "_eclasses_" in values: - d = ProtectedDict(values) - d["_eclasses_"] = serialize_eclasses(d["_eclasses_"]) - else: + if "_eclasses_" in values: + if d is None: + d = ProtectedDict(values) + if self.serialize_eclasses: + d["_eclasses_"] = serialize_eclasses(d["_eclasses_"], + self.validation_chf, paths=self.store_eclass_paths) + else: + d["_eclasses_"] = self._internal_eclasses(d["_eclasses_"], + self.validation_chf, self.store_eclass_paths) + elif d is None: d = values self._setitem(cpv, d) if not self.autocommits: @@ -159,6 +195,23 @@ class database(object): except KeyError: return x + def validate_entry(self, entry, ebuild_hash, eclass_db): + hash_key = '_%s_' % self.validation_chf + try: + entry_hash = entry[hash_key] + except KeyError: + return False + else: + if entry_hash != getattr(ebuild_hash, self.validation_chf): + return False + update = eclass_db.validate_and_rewrite_cache(entry['_eclasses_'], self.validation_chf, + self.store_eclass_paths) + if update is None: + return False + if update: + entry['_eclasses_'] = update + return True + def get_matches(self, match_dict): """generic function for walking the entire cache db, matching restrictions to filter what cpv's are returned. Derived classes should override this if they @@ -195,7 +248,9 @@ class database(object): keys = __iter__ items = iteritems -def serialize_eclasses(eclass_dict): +_keysorter = operator.itemgetter(0) + +def serialize_eclasses(eclass_dict, chf_type='mtime', paths=True): """takes a dict, returns a string representing said dict""" """The "new format", which causes older versions of <portage-2.1.2 to traceback with a ValueError due to failed long() conversion. This format @@ -206,27 +261,40 @@ def serialize_eclasses(eclass_dict): """ if not eclass_dict: return "" - return "\t".join(k + "\t%s\t%s" % eclass_dict[k] \ - for k in sorted(eclass_dict)) + getter = operator.attrgetter(chf_type) + if paths: + return "\t".join("%s\t%s\t%s" % (k, v.eclass_dir, getter(v)) + for k, v in sorted(eclass_dict.items(), key=_keysorter)) + return "\t".join("%s\t%s" % (k, getter(v)) + for k, v in sorted(eclass_dict.items(), key=_keysorter)) + -def reconstruct_eclasses(cpv, eclass_string): +def reconstruct_eclasses(cpv, eclass_string, chf_type='mtime', paths=True): """returns a dict when handed a string generated by serialize_eclasses""" eclasses = eclass_string.rstrip().lstrip().split("\t") if eclasses == [""]: # occasionally this occurs in the fs backends. they suck. return {} - - if len(eclasses) % 2 != 0 and len(eclasses) % 3 != 0: + + converter = _unicode + if chf_type == 'mtime': + converter = long + + if paths: + if len(eclasses) % 3 != 0: + raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses)) + elif len(eclasses) % 2 != 0: raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses)) d={} try: - if eclasses[1].isdigit(): - for x in range(0, len(eclasses), 2): - d[eclasses[x]] = ("", long(eclasses[x + 1])) - else: + i = iter(eclasses) + if paths: # The old format contains paths that will be discarded. - for x in range(0, len(eclasses), 3): - d[eclasses[x]] = (eclasses[x + 1], long(eclasses[x + 2])) + for name, path, val in zip(i, i, i): + d[name] = (path, converter(val)) + else: + for name, val in zip(i, i): + d[name] = converter(val) except IndexError: raise cache_errors.CacheCorruption(cpv, "_eclasses_ was of invalid len %i" % len(eclasses)) diff --git a/portage_with_autodep/pym/portage/cache/template.pyo b/portage_with_autodep/pym/portage/cache/template.pyo Binary files differnew file mode 100644 index 0000000..45da015 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/template.pyo diff --git a/portage_with_autodep/pym/portage/cache/util.py b/portage_with_autodep/pym/portage/cache/util.py deleted file mode 100644 index b824689..0000000 --- a/portage_with_autodep/pym/portage/cache/util.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright: 2005 Gentoo Foundation -# Author(s): Brian Harring (ferringb@gentoo.org) -# License: GPL2 - -from __future__ import print_function - -__all__ = ["mirror_cache", "non_quiet_mirroring", "quiet_mirroring"] - -from itertools import chain -from portage.cache import cache_errors -from portage.localization import _ - -def mirror_cache(valid_nodes_iterable, src_cache, trg_cache, eclass_cache=None, verbose_instance=None): - - from portage import eapi_is_supported, \ - _validate_cache_for_unsupported_eapis - if not src_cache.complete_eclass_entries and not eclass_cache: - raise Exception("eclass_cache required for cache's of class %s!" % src_cache.__class__) - - if verbose_instance == None: - noise=quiet_mirroring() - else: - noise=verbose_instance - - dead_nodes = set(trg_cache) - count=0 - - if not trg_cache.autocommits: - trg_cache.sync(100) - - for x in valid_nodes_iterable: -# print "processing x=",x - count+=1 - dead_nodes.discard(x) - try: - entry = src_cache[x] - except KeyError as e: - noise.missing_entry(x) - del e - continue - except cache_errors.CacheError as ce: - noise.exception(x, ce) - del ce - continue - - eapi = entry.get('EAPI') - if not eapi: - eapi = '0' - eapi = eapi.lstrip('-') - eapi_supported = eapi_is_supported(eapi) - if not eapi_supported: - if not _validate_cache_for_unsupported_eapis: - noise.misc(x, _("unable to validate cache for EAPI='%s'") % eapi) - continue - - write_it = True - trg = None - try: - trg = trg_cache[x] - except (KeyError, cache_errors.CacheError): - pass - else: - if trg['_mtime_'] == entry['_mtime_'] and \ - eclass_cache.is_eclass_data_valid(trg['_eclasses_']) and \ - set(trg['_eclasses_']) == set(entry['_eclasses_']): - write_it = False - - for d in (entry, trg): - if d is not None and d.get('EAPI') in ('', '0'): - del d['EAPI'] - - if trg and not write_it: - """ We don't want to skip the write unless we're really sure that - the existing cache is identical, so don't trust _mtime_ and - _eclasses_ alone.""" - for k in set(chain(entry, trg)).difference( - ("_mtime_", "_eclasses_")): - if trg.get(k, "") != entry.get(k, ""): - write_it = True - break - - if write_it: - try: - inherited = entry.get("INHERITED", "") - eclasses = entry.get("_eclasses_") - except cache_errors.CacheError as ce: - noise.exception(x, ce) - del ce - continue - - if eclasses is not None: - if not eclass_cache.is_eclass_data_valid(entry["_eclasses_"]): - noise.eclass_stale(x) - continue - inherited = eclasses - else: - inherited = inherited.split() - - if inherited: - if src_cache.complete_eclass_entries and eclasses is None: - noise.corruption(x, "missing _eclasses_ field") - continue - - # Even if _eclasses_ already exists, replace it with data from - # eclass_cache, in order to insert local eclass paths. - try: - eclasses = eclass_cache.get_eclass_data(inherited) - except KeyError: - # INHERITED contains a non-existent eclass. - noise.eclass_stale(x) - continue - - if eclasses is None: - noise.eclass_stale(x) - continue - entry["_eclasses_"] = eclasses - - if not eapi_supported: - for k in set(entry).difference(("_mtime_", "_eclasses_")): - entry[k] = "" - entry["EAPI"] = "-" + eapi - - # by this time, if it reaches here, the eclass has been validated, and the entry has - # been updated/translated (if needs be, for metadata/cache mainly) - try: - trg_cache[x] = entry - except cache_errors.CacheError as ce: - noise.exception(x, ce) - del ce - continue - if count >= noise.call_update_min: - noise.update(x) - count = 0 - - if not trg_cache.autocommits: - trg_cache.commit() - - # ok. by this time, the trg_cache is up to date, and we have a dict - # with a crapload of cpv's. we now walk the target db, removing stuff if it's in the list. - for key in dead_nodes: - try: - del trg_cache[key] - except KeyError: - pass - except cache_errors.CacheError as ce: - noise.exception(ce) - del ce - noise.finish() - - -class quiet_mirroring(object): - # call_update_every is used by mirror_cache to determine how often to call in. - # quiet defaults to 2^24 -1. Don't call update, 'cept once every 16 million or so :) - call_update_min = 0xffffff - def update(self,key,*arg): pass - def exception(self,key,*arg): pass - def eclass_stale(self,*arg): pass - def missing_entry(self, key): pass - def misc(self,key,*arg): pass - def corruption(self, key, s): pass - def finish(self, *arg): pass - -class non_quiet_mirroring(quiet_mirroring): - call_update_min=1 - def update(self,key,*arg): print("processed",key) - def exception(self, key, *arg): print("exec",key,arg) - def missing(self,key): print("key %s is missing", key) - def corruption(self,key,*arg): print("corrupt %s:" % key,arg) - def eclass_stale(self,key,*arg):print("stale %s:"%key,arg) - diff --git a/portage_with_autodep/pym/portage/cache/volatile.py b/portage_with_autodep/pym/portage/cache/volatile.py index 0bf6bab..5516745 100644 --- a/portage_with_autodep/pym/portage/cache/volatile.py +++ b/portage_with_autodep/pym/portage/cache/volatile.py @@ -8,18 +8,23 @@ class database(template.database): autocommits = True serialize_eclasses = False + store_eclass_paths = False def __init__(self, *args, **config): config.pop("gid", None) config.pop("perms", None) super(database, self).__init__(*args, **config) self._data = {} - self.__iter__ = self._data.__iter__ self._delitem = self._data.__delitem__ - self.__contains__ = self._data.__contains__ def _setitem(self, name, values): self._data[name] = copy.deepcopy(values) - def _getitem(self, cpv): + def __getitem__(self, cpv): return copy.deepcopy(self._data[cpv]) + + def __iter__(self): + return iter(self._data) + + def __contains__(self, key): + return key in self._data diff --git a/portage_with_autodep/pym/portage/cache/volatile.pyo b/portage_with_autodep/pym/portage/cache/volatile.pyo Binary files differnew file mode 100644 index 0000000..fac5d55 --- /dev/null +++ b/portage_with_autodep/pym/portage/cache/volatile.pyo diff --git a/portage_with_autodep/pym/portage/checksum.py b/portage_with_autodep/pym/portage/checksum.py index 9e7e455..bd416ac 100644 --- a/portage_with_autodep/pym/portage/checksum.py +++ b/portage_with_autodep/pym/portage/checksum.py @@ -1,5 +1,5 @@ # checksum.py -- core Portage functionality -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import portage @@ -16,8 +16,31 @@ import tempfile hashfunc_map = {} hashorigin_map = {} -def _generate_hash_function(hashtype, hashobject, origin="unknown"): - def pyhash(filename): +def _open_file(filename): + try: + return open(_unicode_encode(filename, + encoding=_encodings['fs'], errors='strict'), 'rb') + except IOError as e: + func_call = "open('%s')" % filename + if e.errno == errno.EPERM: + raise portage.exception.OperationNotPermitted(func_call) + elif e.errno == errno.EACCES: + raise portage.exception.PermissionDenied(func_call) + elif e.errno == errno.ENOENT: + raise portage.exception.FileNotFound(filename) + else: + raise + +class _generate_hash_function(object): + + __slots__ = ("_hashobject",) + + def __init__(self, hashtype, hashobject, origin="unknown"): + self._hashobject = hashobject + hashfunc_map[hashtype] = self + hashorigin_map[hashtype] = origin + + def __call__(self, filename): """ Run a checksum against a file. @@ -25,23 +48,11 @@ def _generate_hash_function(hashtype, hashobject, origin="unknown"): @type filename: String @return: The hash and size of the data """ - try: - f = open(_unicode_encode(filename, - encoding=_encodings['fs'], errors='strict'), 'rb') - except IOError as e: - func_call = "open('%s')" % filename - if e.errno == errno.EPERM: - raise portage.exception.OperationNotPermitted(func_call) - elif e.errno == errno.EACCES: - raise portage.exception.PermissionDenied(func_call) - elif e.errno == errno.ENOENT: - raise portage.exception.FileNotFound(filename) - else: - raise + f = _open_file(filename) blocksize = HASHING_BLOCKSIZE data = f.read(blocksize) size = 0 - checksum = hashobject() + checksum = self._hashobject() while data: checksum.update(data) size = size + len(data) @@ -49,9 +60,6 @@ def _generate_hash_function(hashtype, hashobject, origin="unknown"): f.close() return (checksum.hexdigest(), size) - hashfunc_map[hashtype] = pyhash - hashorigin_map[hashtype] = origin - return pyhash # Define hash functions, try to use the best module available. Later definitions # override earlier ones @@ -71,40 +79,72 @@ except ImportError: sha1hash = _generate_hash_function("SHA1", _new_sha1, origin="internal") +# Try to use mhash if available +# mhash causes GIL presently, so it gets less priority than hashlib and +# pycrypto. However, it might be the only accelerated implementation of +# WHIRLPOOL available. +try: + import mhash, functools + md5hash = _generate_hash_function("MD5", functools.partial(mhash.MHASH, mhash.MHASH_MD5), origin="mhash") + sha1hash = _generate_hash_function("SHA1", functools.partial(mhash.MHASH, mhash.MHASH_SHA1), origin="mhash") + sha256hash = _generate_hash_function("SHA256", functools.partial(mhash.MHASH, mhash.MHASH_SHA256), origin="mhash") + sha512hash = _generate_hash_function("SHA512", functools.partial(mhash.MHASH, mhash.MHASH_SHA512), origin="mhash") + for local_name, hash_name in (("rmd160", "ripemd160"), ("whirlpool", "whirlpool")): + if hasattr(mhash, 'MHASH_%s' % local_name.upper()): + globals()['%shash' % local_name] = \ + _generate_hash_function(local_name.upper(), \ + functools.partial(mhash.MHASH, getattr(mhash, 'MHASH_%s' % hash_name.upper())), \ + origin='mhash') +except ImportError: + pass + # Use pycrypto when available, prefer it over the internal fallbacks +# Check for 'new' attributes, since they can be missing if the module +# is broken somehow. try: from Crypto.Hash import SHA256, RIPEMD - sha256hash = _generate_hash_function("SHA256", SHA256.new, origin="pycrypto") - rmd160hash = _generate_hash_function("RMD160", RIPEMD.new, origin="pycrypto") -except ImportError as e: + sha256hash = getattr(SHA256, 'new', None) + if sha256hash is not None: + sha256hash = _generate_hash_function("SHA256", + sha256hash, origin="pycrypto") + rmd160hash = getattr(RIPEMD, 'new', None) + if rmd160hash is not None: + rmd160hash = _generate_hash_function("RMD160", + rmd160hash, origin="pycrypto") +except ImportError: pass # Use hashlib from python-2.5 if available and prefer it over pycrypto and internal fallbacks. -# Need special handling for RMD160 as it may not always be provided by hashlib. +# Need special handling for RMD160/WHIRLPOOL as they may not always be provided by hashlib. try: - import hashlib + import hashlib, functools md5hash = _generate_hash_function("MD5", hashlib.md5, origin="hashlib") sha1hash = _generate_hash_function("SHA1", hashlib.sha1, origin="hashlib") sha256hash = _generate_hash_function("SHA256", hashlib.sha256, origin="hashlib") - try: - hashlib.new('ripemd160') - except ValueError: - pass - else: - def rmd160(): - return hashlib.new('ripemd160') - rmd160hash = _generate_hash_function("RMD160", rmd160, origin="hashlib") -except ImportError as e: + sha512hash = _generate_hash_function("SHA512", hashlib.sha512, origin="hashlib") + for local_name, hash_name in (("rmd160", "ripemd160"), ("whirlpool", "whirlpool")): + try: + hashlib.new(hash_name) + except ValueError: + pass + else: + globals()['%shash' % local_name] = \ + _generate_hash_function(local_name.upper(), \ + functools.partial(hashlib.new, hash_name), \ + origin='hashlib') + +except ImportError: pass - + +if "WHIRLPOOL" not in hashfunc_map: + # Bundled WHIRLPOOL implementation + from portage.util.whirlpool import new as _new_whirlpool + whirlpoolhash = _generate_hash_function("WHIRLPOOL", _new_whirlpool, origin="bundled") # Use python-fchksum if available, prefer it over all other MD5 implementations try: - import fchksum - - def md5hash(filename): - return fchksum.fmd5t(filename) + from fchksum import fmd5t as md5hash hashfunc_map["MD5"] = md5hash hashorigin_map["MD5"] = "python-fchksum" @@ -127,6 +167,15 @@ if os.path.exists(PRELINK_BINARY): prelink_capable=1 del results +def is_prelinkable_elf(filename): + f = _open_file(filename) + try: + magic = f.read(17) + finally: + f.close() + return (len(magic) == 17 and magic.startswith(b'\x7fELF') and + magic[16] in (b'\x02', b'\x03')) # 2=ET_EXEC, 3=ET_DYN + def perform_md5(x, calc_prelink=0): return perform_checksum(x, "MD5", calc_prelink)[0] @@ -137,7 +186,7 @@ def _perform_md5_merge(x, **kwargs): def perform_all(x, calc_prelink=0): mydict = {} for k in hashfunc_map: - mydict[k] = perform_checksum(x, hashfunc_map[k], calc_prelink)[0] + mydict[k] = perform_checksum(x, k, calc_prelink)[0] return mydict def get_valid_checksum_keys(): @@ -234,7 +283,8 @@ def perform_checksum(filename, hashname="MD5", calc_prelink=0): myfilename = filename prelink_tmpfile = None try: - if calc_prelink and prelink_capable: + if (calc_prelink and prelink_capable and + is_prelinkable_elf(filename)): # Create non-prelinked temporary file to checksum. # Files rejected by prelink are summed in place. try: @@ -255,8 +305,10 @@ def perform_checksum(filename, hashname="MD5", calc_prelink=0): " hash function not available (needs dev-python/pycrypto)") myhash, mysize = hashfunc_map[hashname](myfilename) except (OSError, IOError) as e: - if e.errno == errno.ENOENT: + if e.errno in (errno.ENOENT, errno.ESTALE): raise portage.exception.FileNotFound(myfilename) + elif e.errno == portage.exception.PermissionDenied.errno: + raise portage.exception.PermissionDenied(myfilename) raise return myhash, mysize finally: diff --git a/portage_with_autodep/pym/portage/checksum.pyo b/portage_with_autodep/pym/portage/checksum.pyo Binary files differnew file mode 100644 index 0000000..00231af --- /dev/null +++ b/portage_with_autodep/pym/portage/checksum.pyo diff --git a/portage_with_autodep/pym/portage/const.py b/portage_with_autodep/pym/portage/const.py index 2a391db..614dcdb 100644 --- a/portage_with_autodep/pym/portage/const.py +++ b/portage_with_autodep/pym/portage/const.py @@ -67,8 +67,7 @@ FAKEROOT_BINARY = "/usr/bin/fakeroot" BASH_BINARY = "/bin/bash" MOVE_BINARY = "/bin/mv" PRELINK_BINARY = "/usr/sbin/prelink" -AUTODEP_LIBRARY = "/usr/lib/file_hook.so" -#AUTODEP_LIBRARY = "/home/bay/autodep/src/hook_lib/file_hook.so" +AUTODEP_LIBRARY = "/usr/lib/file_hook.so" INVALID_ENV_FILE = "/etc/spork/is/not/valid/profile.env" @@ -89,12 +88,12 @@ EBUILD_PHASES = ("pretend", "setup", "unpack", "prepare", "configure" "package", "preinst", "postinst","prerm", "postrm", "nofetch", "config", "info", "other") SUPPORTED_FEATURES = frozenset([ - "allow-missing-manifests", "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", - "ccache", "chflags", "collision-protect", "compress-build-logs", - "depcheck", "depcheckstrict", + "ccache", "chflags", "clean-logs", + "collision-protect", "compress-build-logs", "compressdebug", + "config-protect-if-modified", "depcheck", "depcheckstrict", "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", - "fail-clean", "fixpackages", "force-mirror", "getbinpkg", + "fail-clean", "force-mirror", "force-prefix", "getbinpkg", "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", "metadata-transfer", "mirror", "multilib-strict", "news", "noauto", "noclean", "nodoc", "noinfo", "noman", @@ -107,18 +106,57 @@ SUPPORTED_FEATURES = frozenset([ "strict", "stricter", "suidctl", "test", "test-fail-continue", "unknown-features-filter", "unknown-features-warn", "unmerge-logs", "unmerge-orphans", "userfetch", "userpriv", - "usersandbox", "usersync", "webrsync-gpg"]) + "usersandbox", "usersync", "webrsync-gpg", "xattr"]) EAPI = 4 HASHING_BLOCKSIZE = 32768 MANIFEST1_HASH_FUNCTIONS = ("MD5", "SHA256", "RMD160") -MANIFEST2_HASH_FUNCTIONS = ("SHA1", "SHA256", "RMD160") - MANIFEST1_REQUIRED_HASH = "MD5" -MANIFEST2_REQUIRED_HASH = "SHA1" + +# Future events: +# +# After WHIRLPOOL is supported in stable portage: +# - Add SHA256 and WHIRLPOOL to MANIFEST2_HASH_DEFAULTS. +# - Remove SHA1 and RMD160 from MANIFEST2_HASH_*. +# - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows: +# manifest-hashes = SHA256 SHA512 WHIRLPOOL +# +# After WHIRLPOOL is supported in stable portage for at least 1 year: +# - Change MANIFEST2_REQUIRED_HASH to WHIRLPOOL. +# - Remove SHA256 from MANIFEST2_HASH_*. +# - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows: +# manifest-hashes = SHA512 WHIRLPOOL +# +# After SHA-3 is approved: +# - Add new hashes to MANIFEST2_HASH_*. +# +# After SHA-3 is supported in stable portage: +# - Set manifest-hashes in gentoo-x86/metadata/layout.conf as follows: +# manifest-hashes = SHA3 SHA512 WHIRLPOOL +# +# After layout.conf settings correspond to defaults in stable portage: +# - Remove redundant settings from gentoo-x86/metadata/layout.conf. + +MANIFEST2_HASH_FUNCTIONS = ("RMD160", "SHA1", "SHA256", "SHA512", "WHIRLPOOL") +MANIFEST2_HASH_DEFAULTS = frozenset(["SHA1", "SHA256", "RMD160"]) +MANIFEST2_REQUIRED_HASH = "SHA256" MANIFEST2_IDENTIFIERS = ("AUX", "MISC", "DIST", "EBUILD") + +# The EPREFIX for the current install is hardcoded here, but access to this +# constant should be minimal, in favor of access via the EPREFIX setting of +# a config instance (since it's possible to contruct a config instance with +# a different EPREFIX). Therefore, the EPREFIX constant should *NOT* be used +# in the definition of any other constants within this file. +EPREFIX="" + +# pick up EPREFIX from the environment if set +if "PORTAGE_OVERRIDE_EPREFIX" in os.environ: + EPREFIX = os.environ["PORTAGE_OVERRIDE_EPREFIX"] + if EPREFIX: + EPREFIX = os.path.normpath(EPREFIX) + # =========================================================================== # END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANTS -- END OF CONSTANT # =========================================================================== @@ -129,7 +167,6 @@ _ENABLE_DYN_LINK_MAP = True _ENABLE_PRESERVE_LIBS = True _ENABLE_REPO_NAME_WARN = True _ENABLE_SET_CONFIG = True -_SANDBOX_COMPAT_LEVEL = "22" # The definitions above will differ between branches, so it's useful to have diff --git a/portage_with_autodep/pym/portage/const.py.rej b/portage_with_autodep/pym/portage/const.py.rej new file mode 100644 index 0000000..9fe70f8 --- /dev/null +++ b/portage_with_autodep/pym/portage/const.py.rej @@ -0,0 +1,12 @@ +--- pym/portage/const.py ++++ pym/portage/const.py +@@ -90,7 +92,8 @@ + SUPPORTED_FEATURES = frozenset([ + "allow-missing-manifests", + "assume-digests", "binpkg-logs", "buildpkg", "buildsyspkg", "candy", +- "ccache", "chflags", "collision-protect", "compress-build-logs", ++ "ccache", "chflags", "collision-protect", "compress-build-logs", ++ "depcheck", "depcheckstrict", + "digest", "distcc", "distcc-pump", "distlocks", "ebuild-locks", "fakeroot", + "fail-clean", "fixpackages", "force-mirror", "getbinpkg", + "installsources", "keeptemp", "keepwork", "fixlafiles", "lmirror", diff --git a/portage_with_autodep/pym/portage/const.pyo b/portage_with_autodep/pym/portage/const.pyo Binary files differnew file mode 100644 index 0000000..804420f --- /dev/null +++ b/portage_with_autodep/pym/portage/const.pyo diff --git a/portage_with_autodep/pym/portage/cvstree.py b/portage_with_autodep/pym/portage/cvstree.py index 9ba22f3..3680ae4 100644 --- a/portage_with_autodep/pym/portage/cvstree.py +++ b/portage_with_autodep/pym/portage/cvstree.py @@ -248,11 +248,13 @@ def getentries(mydir,recursive=0): if entries["files"][mysplit[1]]["revision"][0]=="-": entries["files"][mysplit[1]]["status"]+=["removed"] - for file in apply_cvsignore_filter(os.listdir(mydir)): + for file in os.listdir(mydir): if file=="CVS": continue if os.path.isdir(mydir+"/"+file): if file not in entries["dirs"]: + if ignore_list.match(file) is not None: + continue entries["dirs"][file]={"dirs":{},"files":{}} # It's normal for a directory to be unlisted in Entries # when checked out without -P (see bug #257660). @@ -266,6 +268,8 @@ def getentries(mydir,recursive=0): entries["dirs"][file]["status"]=["exists"] elif os.path.isfile(mydir+"/"+file): if file not in entries["files"]: + if ignore_list.match(file) is not None: + continue entries["files"][file]={"revision":"","date":"","flags":"","tags":""} if "status" in entries["files"][file]: if "exists" not in entries["files"][file]["status"]: @@ -285,7 +289,9 @@ def getentries(mydir,recursive=0): print("failed to stat",file) print(e) return - + + elif ignore_list.match(file) is not None: + pass else: print() print("File of unknown type:",mydir+"/"+file) diff --git a/portage_with_autodep/pym/portage/cvstree.pyo b/portage_with_autodep/pym/portage/cvstree.pyo Binary files differnew file mode 100644 index 0000000..4719daf --- /dev/null +++ b/portage_with_autodep/pym/portage/cvstree.pyo diff --git a/portage_with_autodep/pym/portage/data.py b/portage_with_autodep/pym/portage/data.py index c38fa17..c4d967a 100644 --- a/portage_with_autodep/pym/portage/data.py +++ b/portage_with_autodep/pym/portage/data.py @@ -1,5 +1,5 @@ # data.py -- Calculated/Discovered Data Values -# Copyright 1998-2010 Gentoo Foundation +# Copyright 1998-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os, pwd, grp, platform @@ -58,65 +58,165 @@ def portage_group_warning(): # If the "wheel" group does not exist then wheelgid falls back to 0. # If the "portage" group does not exist then portage_uid falls back to wheelgid. -secpass=0 - uid=os.getuid() wheelgid=0 -if uid==0: - secpass=2 try: wheelgid=grp.getgrnam("wheel")[2] except KeyError: pass -# Allow the overriding of the user used for 'userpriv' and 'userfetch' -_portage_uname = os.environ.get('PORTAGE_USERNAME', 'portage') -_portage_grpname = os.environ.get('PORTAGE_GRPNAME', 'portage') +# The portage_uid and portage_gid global constants, and others that +# depend on them are initialized lazily, in order to allow configuration +# via make.conf. Eventually, these constants may be deprecated in favor +# of config attributes, since it's conceivable that multiple +# configurations with different constants could be used simultaneously. +_initialized_globals = set() -#Discover the uid and gid of the portage user/group -try: - portage_uid = pwd.getpwnam(_portage_uname)[2] - portage_gid = grp.getgrnam(_portage_grpname)[2] - if secpass < 1 and portage_gid in os.getgroups(): - secpass=1 -except KeyError: - portage_uid=0 - portage_gid=0 - userpriv_groups = [portage_gid] - writemsg(colorize("BAD", - _("portage: 'portage' user or group missing.")) + "\n", noiselevel=-1) - writemsg(_( - " For the defaults, line 1 goes into passwd, " - "and 2 into group.\n"), noiselevel=-1) - writemsg(colorize("GOOD", - " portage:x:250:250:portage:/var/tmp/portage:/bin/false") \ - + "\n", noiselevel=-1) - writemsg(colorize("GOOD", " portage::250:portage") + "\n", - noiselevel=-1) - portage_group_warning() -else: - userpriv_groups = [portage_gid] - if secpass >= 2: - class _LazyUserprivGroups(portage.proxy.objectproxy.ObjectProxy): - def _get_target(self): - global userpriv_groups - if userpriv_groups is not self: - return userpriv_groups - userpriv_groups = _userpriv_groups - # Get a list of group IDs for the portage user. Do not use - # grp.getgrall() since it is known to trigger spurious - # SIGPIPE problems with nss_ldap. - mystatus, myoutput = \ - portage.subprocess_getstatusoutput("id -G %s" % _portage_uname) - if mystatus == os.EX_OK: - for x in myoutput.split(): - try: - userpriv_groups.append(int(x)) - except ValueError: - pass - userpriv_groups[:] = sorted(set(userpriv_groups)) - return userpriv_groups - - _userpriv_groups = userpriv_groups - userpriv_groups = _LazyUserprivGroups() +def _get_global(k): + if k in _initialized_globals: + return globals()[k] + + if k in ('portage_gid', 'portage_uid', 'secpass'): + global portage_gid, portage_uid, secpass + secpass = 0 + if uid == 0: + secpass = 2 + elif portage.const.EPREFIX: + secpass = 2 + #Discover the uid and gid of the portage user/group + try: + portage_uid = pwd.getpwnam(_get_global('_portage_username')).pw_uid + _portage_grpname = _get_global('_portage_grpname') + if platform.python_implementation() == 'PyPy': + # Somehow this prevents "TypeError: expected string" errors + # from grp.getgrnam() with PyPy 1.7 + _portage_grpname = str(_portage_grpname) + portage_gid = grp.getgrnam(_portage_grpname).gr_gid + if secpass < 1 and portage_gid in os.getgroups(): + secpass = 1 + except KeyError: + portage_uid = 0 + portage_gid = 0 + writemsg(colorize("BAD", + _("portage: 'portage' user or group missing.")) + "\n", noiselevel=-1) + writemsg(_( + " For the defaults, line 1 goes into passwd, " + "and 2 into group.\n"), noiselevel=-1) + writemsg(colorize("GOOD", + " portage:x:250:250:portage:/var/tmp/portage:/bin/false") \ + + "\n", noiselevel=-1) + writemsg(colorize("GOOD", " portage::250:portage") + "\n", + noiselevel=-1) + portage_group_warning() + + _initialized_globals.add('portage_gid') + _initialized_globals.add('portage_uid') + _initialized_globals.add('secpass') + + if k == 'portage_gid': + return portage_gid + elif k == 'portage_uid': + return portage_uid + elif k == 'secpass': + return secpass + else: + raise AssertionError('unknown name: %s' % k) + + elif k == 'userpriv_groups': + v = [portage_gid] + if secpass >= 2: + # Get a list of group IDs for the portage user. Do not use + # grp.getgrall() since it is known to trigger spurious + # SIGPIPE problems with nss_ldap. + mystatus, myoutput = \ + portage.subprocess_getstatusoutput("id -G %s" % _portage_username) + if mystatus == os.EX_OK: + for x in myoutput.split(): + try: + v.append(int(x)) + except ValueError: + pass + v = sorted(set(v)) + + # Avoid instantiating portage.settings when the desired + # variable is set in os.environ. + elif k in ('_portage_grpname', '_portage_username'): + v = None + if k == '_portage_grpname': + env_key = 'PORTAGE_GRPNAME' + else: + env_key = 'PORTAGE_USERNAME' + + if env_key in os.environ: + v = os.environ[env_key] + elif hasattr(portage, 'settings'): + v = portage.settings.get(env_key) + elif portage.const.EPREFIX: + # For prefix environments, default to the UID and GID of + # the top-level EROOT directory. The config class has + # equivalent code, but we also need to do it here if + # _disable_legacy_globals() has been called. + eroot = os.path.join(os.environ.get('ROOT', os.sep), + portage.const.EPREFIX.lstrip(os.sep)) + try: + eroot_st = os.stat(eroot) + except OSError: + pass + else: + if k == '_portage_grpname': + try: + grp_struct = grp.getgrgid(eroot_st.st_gid) + except KeyError: + pass + else: + v = grp_struct.gr_name + else: + try: + pwd_struct = pwd.getpwuid(eroot_st.st_uid) + except KeyError: + pass + else: + v = pwd_struct.pw_name + + if v is None: + v = 'portage' + else: + raise AssertionError('unknown name: %s' % k) + + globals()[k] = v + _initialized_globals.add(k) + return v + +class _GlobalProxy(portage.proxy.objectproxy.ObjectProxy): + + __slots__ = ('_name',) + + def __init__(self, name): + portage.proxy.objectproxy.ObjectProxy.__init__(self) + object.__setattr__(self, '_name', name) + + def _get_target(self): + return _get_global(object.__getattribute__(self, '_name')) + +for k in ('portage_gid', 'portage_uid', 'secpass', 'userpriv_groups', + '_portage_grpname', '_portage_username'): + globals()[k] = _GlobalProxy(k) +del k + +def _init(settings): + """ + Use config variables like PORTAGE_GRPNAME and PORTAGE_USERNAME to + initialize global variables. This allows settings to come from make.conf + instead of requiring them to be set in the calling environment. + """ + if '_portage_grpname' not in _initialized_globals and \ + '_portage_username' not in _initialized_globals: + + v = settings.get('PORTAGE_GRPNAME', 'portage') + globals()['_portage_grpname'] = v + _initialized_globals.add('_portage_grpname') + + v = settings.get('PORTAGE_USERNAME', 'portage') + globals()['_portage_username'] = v + _initialized_globals.add('_portage_username') diff --git a/portage_with_autodep/pym/portage/data.pyo b/portage_with_autodep/pym/portage/data.pyo Binary files differnew file mode 100644 index 0000000..7f749e0 --- /dev/null +++ b/portage_with_autodep/pym/portage/data.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py index 34ed031..b5f6a0b 100644 --- a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.py @@ -1,22 +1,16 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io -import shutil import signal -import tempfile +import sys import traceback import errno import fcntl import portage from portage import os, _unicode_decode -from portage.const import PORTAGE_PACKAGE_ATOM -from portage.dep import match_from_list import portage.elog.messages -from portage.elog import _preload_elog_modules -from portage.util import ensure_dirs -from _emerge.PollConstants import PollConstants from _emerge.SpawnProcess import SpawnProcess class MergeProcess(SpawnProcess): @@ -26,7 +20,7 @@ class MergeProcess(SpawnProcess): """ __slots__ = ('mycat', 'mypkg', 'settings', 'treetype', - 'vartree', 'scheduler', 'blockers', 'pkgloc', 'infloc', 'myebuild', + 'vartree', 'blockers', 'pkgloc', 'infloc', 'myebuild', 'mydbapi', 'prev_mtimes', 'unmerge', '_elog_reader_fd', '_elog_reg_id', '_buf', '_elog_keys', '_locked_vdb') @@ -46,8 +40,12 @@ class MergeProcess(SpawnProcess): settings.reset() settings.setcpv(cpv, mydb=self.mydbapi) - if not self.unmerge: - self._handle_self_reinstall() + # Inherit stdin by default, so that the pdb SIGUSR1 + # handler is usable for the subprocess. + if self.fd_pipes is None: + self.fd_pipes = {} + self.fd_pipes.setdefault(0, sys.stdin.fileno()) + super(MergeProcess, self)._start() def _lock_vdb(self): @@ -69,59 +67,9 @@ class MergeProcess(SpawnProcess): self.vartree.dbapi.unlock() self._locked_vdb = False - def _handle_self_reinstall(self): - """ - If portage is reinstalling itself, create temporary - copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH in order - to avoid relying on the new versions which may be - incompatible. Register an atexit hook to clean up the - temporary directories. Pre-load elog modules here since - we won't be able to later if they get unmerged (happens - when namespace changes). - """ - - settings = self.settings - cpv = settings.mycpv - reinstall_self = False - if self.settings["ROOT"] == "/" and \ - match_from_list(PORTAGE_PACKAGE_ATOM, [cpv]): - inherited = frozenset(self.settings.get('INHERITED', '').split()) - if not self.vartree.dbapi.cpv_exists(cpv) or \ - '9999' in cpv or \ - 'git' in inherited or \ - 'git-2' in inherited: - reinstall_self = True - - if reinstall_self: - # Load lazily referenced portage submodules into memory, - # so imports won't fail during portage upgrade/downgrade. - _preload_elog_modules(self.settings) - portage.proxy.lazyimport._preload_portage_submodules() - - # Make the temp directory inside $PORTAGE_TMPDIR/portage, since - # it's common for /tmp and /var/tmp to be mounted with the - # "noexec" option (see bug #346899). - build_prefix = os.path.join(settings["PORTAGE_TMPDIR"], "portage") - ensure_dirs(build_prefix) - base_path_tmp = tempfile.mkdtemp( - "", "._portage_reinstall_.", build_prefix) - portage.process.atexit_register(shutil.rmtree, base_path_tmp) - dir_perms = 0o755 - for subdir in "bin", "pym": - var_name = "PORTAGE_%s_PATH" % subdir.upper() - var_orig = settings[var_name] - var_new = os.path.join(base_path_tmp, subdir) - settings[var_name] = var_new - settings.backup_changes(var_name) - shutil.copytree(var_orig, var_new, symlinks=True) - os.chmod(var_new, dir_perms) - portage._bin_path = settings['PORTAGE_BIN_PATH'] - portage._pym_path = settings['PORTAGE_PYM_PATH'] - os.chmod(base_path_tmp, dir_perms) - def _elog_output_handler(self, fd, event): output = None - if event & PollConstants.POLLIN: + if event & self.scheduler.IO_IN: try: output = os.read(fd, self._bufsize) except OSError as e: @@ -141,6 +89,15 @@ class MergeProcess(SpawnProcess): reporter = getattr(portage.elog.messages, funcname) reporter(msg, phase=phase, key=key, out=out) + if event & self.scheduler.IO_HUP: + self.scheduler.unregister(self._elog_reg_id) + self._elog_reg_id = None + os.close(self._elog_reader_fd) + self._elog_reader_fd = None + return False + + return True + def _spawn(self, args, fd_pipes, **kwargs): """ Fork a subprocess, apply local settings, and call @@ -178,6 +135,10 @@ class MergeProcess(SpawnProcess): pid = os.fork() if pid != 0: + if not isinstance(pid, int): + raise AssertionError( + "fork returned non-integer: %s" % (repr(pid),)) + os.close(elog_writer_fd) self._elog_reader_fd = elog_reader_fd self._buf = "" @@ -193,7 +154,9 @@ class MergeProcess(SpawnProcess): return [pid] os.close(elog_reader_fd) - portage.process._setup_pipes(fd_pipes) + portage.locks._close_fds() + # Disable close_fds since we don't exec (see _setup_pipes docstring). + portage.process._setup_pipes(fd_pipes, close_fds=False) # Use default signal handlers since the ones inherited # from the parent process are irrelevant here. @@ -270,7 +233,7 @@ class MergeProcess(SpawnProcess): if self._elog_reg_id is not None: self.scheduler.unregister(self._elog_reg_id) self._elog_reg_id = None - if self._elog_reader_fd: + if self._elog_reader_fd is not None: os.close(self._elog_reader_fd) self._elog_reader_fd = None if self._elog_keys is not None: diff --git a/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo Binary files differnew file mode 100644 index 0000000..5839ad8 --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/_MergeProcess.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_SyncfsProcess.py b/portage_with_autodep/pym/portage/dbapi/_SyncfsProcess.py new file mode 100644 index 0000000..7518214 --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/_SyncfsProcess.py @@ -0,0 +1,53 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from portage.util._ctypes import find_library, LoadLibrary +from portage.util._async.ForkProcess import ForkProcess + +class SyncfsProcess(ForkProcess): + """ + Isolate ctypes usage in a subprocess, in order to avoid + potential problems with stale cached libraries as + described in bug #448858, comment #14 (also see + http://bugs.python.org/issue14597). + """ + + __slots__ = ('paths',) + + @staticmethod + def _get_syncfs(): + + filename = find_library("c") + if filename is not None: + library = LoadLibrary(filename) + if library is not None: + try: + return library.syncfs + except AttributeError: + pass + + return None + + def _run(self): + + syncfs_failed = False + syncfs = self._get_syncfs() + + if syncfs is not None: + for path in self.paths: + try: + fd = os.open(path, os.O_RDONLY) + except OSError: + pass + else: + try: + if syncfs(fd) != 0: + # Happens with PyPy (bug #446610) + syncfs_failed = True + finally: + os.close(fd) + + if syncfs is None or syncfs_failed: + return 1 + return os.EX_OK diff --git a/portage_with_autodep/pym/portage/dbapi/__init__.py b/portage_with_autodep/pym/portage/dbapi/__init__.py index e386faa..a1c5c56 100644 --- a/portage_with_autodep/pym/portage/dbapi/__init__.py +++ b/portage_with_autodep/pym/portage/dbapi/__init__.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ["dbapi"] @@ -11,7 +11,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.dep:match_from_list', 'portage.output:colorize', 'portage.util:cmp_sort_key,writemsg', - 'portage.versions:catsplit,catpkgsplit,vercmp', + 'portage.versions:catsplit,catpkgsplit,vercmp,_pkg_str', ) from portage import os @@ -46,7 +46,12 @@ class dbapi(object): def cp_list(self, cp, use_cache=1): raise NotImplementedError(self) - def _cpv_sort_ascending(self, cpv_list): + @staticmethod + def _cmp_cpv(cpv1, cpv2): + return vercmp(cpv1.version, cpv2.version) + + @staticmethod + def _cpv_sort_ascending(cpv_list): """ Use this to sort self.cp_list() results in ascending order. It sorts in place and returns None. @@ -55,12 +60,7 @@ class dbapi(object): # If the cpv includes explicit -r0, it has to be preserved # for consistency in findname and aux_get calls, so use a # dict to map strings back to their original values. - ver_map = {} - for cpv in cpv_list: - ver_map[cpv] = '-'.join(catpkgsplit(cpv)[2:]) - def cmp_cpv(cpv1, cpv2): - return vercmp(ver_map[cpv1], ver_map[cpv2]) - cpv_list.sort(key=cmp_sort_key(cmp_cpv)) + cpv_list.sort(key=cmp_sort_key(dbapi._cmp_cpv)) def cpv_all(self): """Return all CPVs in the db @@ -155,64 +155,74 @@ class dbapi(object): 2) Check enabled/disabled flag states. """ - iuse_implicit_match = self.settings._iuse_implicit_match + aux_keys = ["IUSE", "SLOT", "USE", "repository"] for cpv in cpv_iter: try: - iuse, slot, use = self.aux_get(cpv, ["IUSE", "SLOT", "USE"], myrepo=atom.repo) + metadata = dict(zip(aux_keys, + self.aux_get(cpv, aux_keys, myrepo=atom.repo))) except KeyError: continue - iuse = frozenset(x.lstrip('+-') for x in iuse.split()) - missing_iuse = False - for x in atom.unevaluated_atom.use.required: - if x not in iuse and not iuse_implicit_match(x): - missing_iuse = True - break - if missing_iuse: + + if not self._match_use(atom, cpv, metadata): continue - if not atom.use: - pass - elif not self._use_mutable: - # Use IUSE to validate USE settings for built packages, - # in case the package manager that built this package - # failed to do that for some reason (or in case of - # data corruption). - use = frozenset(x for x in use.split() if x in iuse or \ - iuse_implicit_match(x)) - missing_enabled = atom.use.missing_enabled.difference(iuse) - missing_disabled = atom.use.missing_disabled.difference(iuse) - - if atom.use.enabled: - if atom.use.enabled.intersection(missing_disabled): - continue - need_enabled = atom.use.enabled.difference(use) + + yield cpv + + def _match_use(self, atom, cpv, metadata): + iuse_implicit_match = self.settings._iuse_implicit_match + iuse = frozenset(x.lstrip('+-') for x in metadata["IUSE"].split()) + + for x in atom.unevaluated_atom.use.required: + if x not in iuse and not iuse_implicit_match(x): + return False + + if atom.use is None: + pass + + elif not self._use_mutable: + # Use IUSE to validate USE settings for built packages, + # in case the package manager that built this package + # failed to do that for some reason (or in case of + # data corruption). + use = frozenset(x for x in metadata["USE"].split() + if x in iuse or iuse_implicit_match(x)) + missing_enabled = atom.use.missing_enabled.difference(iuse) + missing_disabled = atom.use.missing_disabled.difference(iuse) + + if atom.use.enabled: + if atom.use.enabled.intersection(missing_disabled): + return False + need_enabled = atom.use.enabled.difference(use) + if need_enabled: + need_enabled = need_enabled.difference(missing_enabled) if need_enabled: - need_enabled = need_enabled.difference(missing_enabled) - if need_enabled: - continue + return False - if atom.use.disabled: - if atom.use.disabled.intersection(missing_enabled): - continue - need_disabled = atom.use.disabled.intersection(use) + if atom.use.disabled: + if atom.use.disabled.intersection(missing_enabled): + return False + need_disabled = atom.use.disabled.intersection(use) + if need_disabled: + need_disabled = need_disabled.difference(missing_disabled) if need_disabled: - need_disabled = need_disabled.difference(missing_disabled) - if need_disabled: - continue - else: - # Check masked and forced flags for repoman. - mysettings = getattr(self, 'settings', None) - if mysettings is not None and not mysettings.local_config: + return False - pkg = "%s:%s" % (cpv, slot) - usemask = mysettings._getUseMask(pkg) - if usemask.intersection(atom.use.enabled): - continue + elif not self.settings.local_config: + # Check masked and forced flags for repoman. + if hasattr(cpv, 'slot'): + pkg = cpv + else: + pkg = _pkg_str(cpv, slot=metadata["SLOT"], + repo=metadata.get("repository")) + usemask = self.settings._getUseMask(pkg) + if usemask.intersection(atom.use.enabled): + return False - useforce = mysettings._getUseForce(pkg).difference(usemask) - if useforce.intersection(atom.use.disabled): - continue + useforce = self.settings._getUseForce(pkg).difference(usemask) + if useforce.intersection(atom.use.disabled): + return False - yield cpv + return True def invalidentry(self, mypath): if '/-MERGING-' in mypath: diff --git a/portage_with_autodep/pym/portage/dbapi/__init__.pyo b/portage_with_autodep/pym/portage/dbapi/__init__.pyo Binary files differnew file mode 100644 index 0000000..e7b494d --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/__init__.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py index 6d6a27d..d379b4c 100644 --- a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py +++ b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.py @@ -63,7 +63,8 @@ def expand_new_virt(vardb, atom): success, atoms = portage.dep_check(rdepend, None, vardb.settings, myuse=valid_use, - myroot=vardb.root, trees={vardb.root:{"porttree":vardb.vartree, + myroot=vardb.settings['EROOT'], + trees={vardb.settings['EROOT']:{"porttree":vardb.vartree, "vartree":vardb.vartree}}) if success: diff --git a/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo Binary files differnew file mode 100644 index 0000000..6c23a7e --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/_expand_new_virt.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/_similar_name_search.py b/portage_with_autodep/pym/portage/dbapi/_similar_name_search.py new file mode 100644 index 0000000..b6e4a1f --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/_similar_name_search.py @@ -0,0 +1,57 @@ +# Copyright 2011-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import difflib + +from portage.versions import catsplit + +def similar_name_search(dbs, atom): + + cp_lower = atom.cp.lower() + cat, pkg = catsplit(cp_lower) + if cat == "null": + cat = None + + all_cp = set() + for db in dbs: + all_cp.update(db.cp_all()) + + # discard dir containing no ebuilds + all_cp.discard(atom.cp) + + orig_cp_map = {} + for cp_orig in all_cp: + orig_cp_map.setdefault(cp_orig.lower(), []).append(cp_orig) + all_cp = set(orig_cp_map) + + if cat: + matches = difflib.get_close_matches(cp_lower, all_cp) + else: + pkg_to_cp = {} + for other_cp in list(all_cp): + other_pkg = catsplit(other_cp)[1] + if other_pkg == pkg: + # Check for non-identical package that + # differs only by upper/lower case. + identical = True + for cp_orig in orig_cp_map[other_cp]: + if catsplit(cp_orig)[1] != \ + catsplit(atom.cp)[1]: + identical = False + break + if identical: + # discard dir containing no ebuilds + all_cp.discard(other_cp) + continue + pkg_to_cp.setdefault(other_pkg, set()).add(other_cp) + + pkg_matches = difflib.get_close_matches(pkg, pkg_to_cp) + matches = [] + for pkg_match in pkg_matches: + matches.extend(pkg_to_cp[pkg_match]) + + matches_orig_case = [] + for cp in matches: + matches_orig_case.extend(orig_cp_map[cp]) + + return matches_orig_case diff --git a/portage_with_autodep/pym/portage/dbapi/bintree.py b/portage_with_autodep/pym/portage/dbapi/bintree.py index 62fc623..a8027ee 100644 --- a/portage_with_autodep/pym/portage/dbapi/bintree.py +++ b/portage_with_autodep/pym/portage/dbapi/bintree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ["bindbapi", "binarytree"] @@ -11,19 +11,20 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.output:EOutput,colorize', 'portage.locks:lockfile,unlockfile', 'portage.package.ebuild.doebuild:_vdb_use_conditional_atoms', - 'portage.package.ebuild.fetch:_check_distfile', + 'portage.package.ebuild.fetch:_check_distfile,_hide_url_passwd', 'portage.update:update_dbentries', 'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \ 'writemsg,writemsg_stdout', 'portage.util.listdir:listdir', - 'portage.versions:best,catpkgsplit,catsplit', + 'portage.util._urlopen:urlopen@_urlopen', + 'portage.versions:best,catpkgsplit,catsplit,_pkg_str', ) from portage.cache.mappings import slot_dict_class from portage.const import CACHE_PATH from portage.dbapi.virtual import fakedbapi from portage.dep import Atom, use_reduce, paren_enclose -from portage.exception import AlarmSignal, InvalidPackageName, \ +from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \ PermissionDenied, PortageException from portage.localization import _ from portage import _movefile @@ -35,19 +36,17 @@ from portage import _unicode_encode import codecs import errno import io -import re import stat import subprocess import sys import tempfile import textwrap +import warnings from itertools import chain try: from urllib.parse import urlparse - from urllib.request import urlopen as urllib_request_urlopen except ImportError: from urlparse import urlparse - from urllib import urlopen as urllib_request_urlopen if sys.hexversion >= 0x3000000: basestring = str @@ -67,7 +66,7 @@ class bindbapi(fakedbapi): ["BUILD_TIME", "CHOST", "DEPEND", "EAPI", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES", - "REQUIRED_USE"]) + ]) self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys) self._aux_cache = {} @@ -177,6 +176,34 @@ class bindbapi(fakedbapi): self.bintree.populate() return fakedbapi.cpv_all(self) + def getfetchsizes(self, pkg): + """ + This will raise MissingSignature if SIZE signature is not available, + or InvalidSignature if SIZE signature is invalid. + """ + + if not self.bintree.populated: + self.bintree.populate() + + pkg = getattr(pkg, 'cpv', pkg) + + filesdict = {} + if not self.bintree.isremote(pkg): + pass + else: + metadata = self.bintree._remotepkgs[pkg] + try: + size = int(metadata["SIZE"]) + except KeyError: + raise portage.exception.MissingSignature("SIZE") + except ValueError: + raise portage.exception.InvalidSignature( + "SIZE: %s" % metadata["SIZE"]) + else: + filesdict[os.path.basename(self.bintree.getname(pkg))] = size + + return filesdict + def _pkgindex_cpv_map_latest_build(pkgindex): """ Given a PackageIndex instance, create a dict of cpv -> metadata map. @@ -185,13 +212,20 @@ def _pkgindex_cpv_map_latest_build(pkgindex): @param pkgindex: A PackageIndex instance. @type pkgindex: PackageIndex @rtype: dict - @returns: a dict containing entry for the give cpv. + @return: a dict containing entry for the give cpv. """ cpv_map = {} for d in pkgindex.packages: cpv = d["CPV"] + try: + cpv = _pkg_str(cpv) + except InvalidData: + writemsg(_("!!! Invalid remote binary package: %s\n") % cpv, + noiselevel=-1) + continue + btime = d.get('BUILD_TIME', '') try: btime = int(btime) @@ -208,16 +242,35 @@ def _pkgindex_cpv_map_latest_build(pkgindex): if other_btime and (not btime or other_btime > btime): continue - cpv_map[cpv] = d + cpv_map[_pkg_str(cpv)] = d return cpv_map class binarytree(object): "this tree scans for a list of all packages available in PKGDIR" - def __init__(self, root, pkgdir, virtual=None, settings=None): + def __init__(self, _unused=None, pkgdir=None, + virtual=DeprecationWarning, settings=None): + + if pkgdir is None: + raise TypeError("pkgdir parameter is required") + + if settings is None: + raise TypeError("settings parameter is required") + + if _unused is not None and _unused != settings['ROOT']: + warnings.warn("The root parameter of the " + "portage.dbapi.bintree.binarytree" + " constructor is now unused. Use " + "settings['ROOT'] instead.", + DeprecationWarning, stacklevel=2) + + if virtual is not DeprecationWarning: + warnings.warn("The 'virtual' parameter of the " + "portage.dbapi.bintree.binarytree" + " constructor is unused", + DeprecationWarning, stacklevel=2) + if True: - self.root = root - #self.pkgdir=settings["PKGDIR"] self.pkgdir = normalize_path(pkgdir) self.dbapi = bindbapi(self, settings=settings) self.update_ents = self.dbapi.update_ents @@ -242,7 +295,7 @@ class binarytree(object): ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES", - "REQUIRED_USE", "BASE_URI"] + "BASE_URI"] self._pkgindex_aux_keys = list(self._pkgindex_aux_keys) self._pkgindex_use_evaluated_keys = \ ("LICENSE", "RDEPEND", "DEPEND", @@ -268,7 +321,6 @@ class binarytree(object): "SLOT" : "0", "USE" : "", "DEFINED_PHASES" : "", - "REQUIRED_USE" : "" } self._pkgindex_inherited_keys = ["CHOST", "repository"] @@ -302,6 +354,15 @@ class binarytree(object): chain(*self._pkgindex_translated_keys) )) + @property + def root(self): + warnings.warn("The root attribute of " + "portage.dbapi.bintree.binarytree" + " is deprecated. Use " + "settings['ROOT'] instead.", + DeprecationWarning, stacklevel=3) + return self.settings['ROOT'] + def move_ent(self, mylist, repo_match=None): if not self.populated: self.populate() @@ -603,6 +664,7 @@ class binarytree(object): if mycpv in pkg_paths: # discard duplicates (All/ is preferred) continue + mycpv = _pkg_str(mycpv) pkg_paths[mycpv] = mypath # update the path if the package has been moved oldpath = d.get("PATH") @@ -678,6 +740,7 @@ class binarytree(object): (mycpv, self.settings["PORTAGE_CONFIGROOT"]), noiselevel=-1) continue + mycpv = _pkg_str(mycpv) pkg_paths[mycpv] = mypath self.dbapi.cpv_inject(mycpv) update_pkgindex = True @@ -787,7 +850,7 @@ class binarytree(object): # slash, so join manually... url = base_url.rstrip("/") + "/Packages" try: - f = urllib_request_urlopen(url) + f = _urlopen(url) except IOError: path = parsed_url.path.rstrip("/") + "/Packages" @@ -859,7 +922,7 @@ class binarytree(object): noiselevel=-1) except EnvironmentError as e: writemsg(_("\n\n!!! Error fetching binhost package" \ - " info from '%s'\n") % base_url) + " info from '%s'\n") % _hide_url_passwd(base_url)) writemsg("!!! %s\n\n" % str(e)) del e pkgindex = None @@ -935,7 +998,7 @@ class binarytree(object): writemsg_stdout("\n") writemsg_stdout( colorize("GOOD", _("Fetching bininfo from ")) + \ - re.sub(r'//(.+):.+@(.+)/', r'//\1:*password*@\2/', base_url) + "\n") + _hide_url_passwd(base_url) + "\n") remotepkgs = portage.getbinpkg.dir_get_metadata( base_url, chunk_size=chunk_size) @@ -947,7 +1010,12 @@ class binarytree(object): noiselevel=-1) continue mycat = mycat.strip() - fullpkg = mycat+"/"+mypkg[:-5] + try: + fullpkg = _pkg_str(mycat+"/"+mypkg[:-5]) + except InvalidData: + writemsg(_("!!! Invalid remote binary package: %s\n") % mypkg, + noiselevel=-1) + continue if fullpkg in metadata: # When using this old protocol, comparison with the remote @@ -1101,7 +1169,7 @@ class binarytree(object): Performs checksums and evaluates USE flag conditionals. Raises InvalidDependString if necessary. @rtype: dict - @returns: a dict containing entry for the give cpv. + @return: a dict containing entry for the give cpv. """ pkg_path = self.getname(cpv) @@ -1307,7 +1375,7 @@ class binarytree(object): Verify digests for the given package and raise DigestException if verification fails. @rtype: bool - @returns: True if digests could be located, False otherwise. + @return: True if digests could be located, False otherwise. """ cpv = pkg if not isinstance(cpv, basestring): diff --git a/portage_with_autodep/pym/portage/dbapi/bintree.pyo b/portage_with_autodep/pym/portage/dbapi/bintree.pyo Binary files differnew file mode 100644 index 0000000..f99f377 --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/bintree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo b/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo Binary files differnew file mode 100644 index 0000000..cf1a428 --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/cpv_expand.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo b/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo Binary files differnew file mode 100644 index 0000000..b323f5b --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/dep_expand.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/porttree.py b/portage_with_autodep/pym/portage/dbapi/porttree.py index ecf275c..c5ee770 100644 --- a/portage_with_autodep/pym/portage/dbapi/porttree.py +++ b/portage_with_autodep/pym/portage/dbapi/porttree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -14,20 +14,19 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.doebuild:doebuild', 'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 'portage.util.listdir:listdir', - 'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp', + 'portage.versions:best,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str', ) -from portage.cache import metadata_overlay, volatile +from portage.cache import volatile from portage.cache.cache_errors import CacheError from portage.cache.mappings import Mapping from portage.dbapi import dbapi from portage.exception import PortageException, \ FileNotFound, InvalidAtom, InvalidDependString, InvalidPackageName from portage.localization import _ -from portage.manifest import Manifest -from portage import eclass_cache, auxdbkeys, \ - eapi_is_supported, dep_check, \ +from portage import eclass_cache, \ + eapi_is_supported, \ _eapi_is_deprecated from portage import os from portage import _encodings @@ -37,8 +36,6 @@ from _emerge.EbuildMetadataPhase import EbuildMetadataPhase from _emerge.PollScheduler import PollScheduler import os as _os -import io -import stat import sys import traceback import warnings @@ -47,15 +44,6 @@ if sys.hexversion >= 0x3000000: basestring = str long = int -class _repo_info(object): - __slots__ = ('name', 'path', 'eclass_db', 'portdir', 'portdir_overlay') - def __init__(self, name, path, eclass_db): - self.name = name - self.path = path - self.eclass_db = eclass_db - self.portdir = eclass_db.porttrees[0] - self.portdir_overlay = ' '.join(eclass_db.porttrees[1:]) - class portdbapi(dbapi): """this tree will scan a portage directory located at root (passed to init)""" portdbapi_instances = [] @@ -69,6 +57,13 @@ class portdbapi(dbapi): def porttree_root(self): return self.settings.repositories.mainRepoLocation() + @property + def eclassdb(self): + main_repo = self.repositories.mainRepo() + if main_repo is None: + return None + return main_repo.eclass_db + def __init__(self, _unused_param=None, mysettings=None): """ @param _unused_param: deprecated, use mysettings['PORTDIR'] instead @@ -100,6 +95,7 @@ class portdbapi(dbapi): # this purpose because doebuild makes many changes to the config # instance that is passed in. self.doebuild_settings = config(clone=self.settings) + self._scheduler = PollScheduler().sched_iface self.depcachedir = os.path.realpath(self.settings.depcachedir) if os.environ.get("SANDBOX_ON") == "1": @@ -112,7 +108,6 @@ class portdbapi(dbapi): ":".join(filter(None, sandbox_write)) self.porttrees = list(self.settings.repositories.repoLocationList()) - self.eclassdb = eclass_cache.cache(self.settings.repositories.mainRepoLocation()) # This is used as sanity check for aux_get(). If there is no # root eclass dir, we assume that PORTDIR is invalid or @@ -121,86 +116,74 @@ class portdbapi(dbapi): self._have_root_eclass_dir = os.path.isdir( os.path.join(self.settings.repositories.mainRepoLocation(), "eclass")) - self.metadbmodule = self.settings.load_best_module("portdbapi.metadbmodule") - #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening) self.xcache = {} self.frozen = 0 - #Create eclass dbs - self._repo_info = {} - eclass_dbs = {self.settings.repositories.mainRepoLocation() : self.eclassdb} - for repo in self.repositories: - if repo.location in self._repo_info: - continue - - eclass_db = None - for eclass_location in repo.eclass_locations: - tree_db = eclass_dbs.get(eclass_location) - if tree_db is None: - tree_db = eclass_cache.cache(eclass_location) - eclass_dbs[eclass_location] = tree_db - if eclass_db is None: - eclass_db = tree_db.copy() - else: - eclass_db.append(tree_db) - - self._repo_info[repo.location] = _repo_info(repo.name, repo.location, eclass_db) - #Keep a list of repo names, sorted by priority (highest priority first). self._ordered_repo_name_list = tuple(reversed(self.repositories.prepos_order)) self.auxdbmodule = self.settings.load_best_module("portdbapi.auxdbmodule") self.auxdb = {} self._pregen_auxdb = {} + # If the current user doesn't have depcachedir write permission, + # then the depcachedir cache is kept here read-only access. + self._ro_auxdb = {} self._init_cache_dirs() - depcachedir_w_ok = os.access(self.depcachedir, os.W_OK) - cache_kwargs = { - 'gid' : portage_gid, - 'perms' : 0o664 - } - - # XXX: REMOVE THIS ONCE UNUSED_0 IS YANKED FROM auxdbkeys - # ~harring - filtered_auxdbkeys = [x for x in auxdbkeys if not x.startswith("UNUSED_0")] - filtered_auxdbkeys.sort() + try: + depcachedir_st = os.stat(self.depcachedir) + depcachedir_w_ok = os.access(self.depcachedir, os.W_OK) + except OSError: + depcachedir_st = None + depcachedir_w_ok = False + + cache_kwargs = {} + + depcachedir_unshared = False + if portage.data.secpass < 1 and \ + depcachedir_w_ok and \ + depcachedir_st is not None and \ + os.getuid() == depcachedir_st.st_uid and \ + os.getgid() == depcachedir_st.st_gid: + # If this user owns depcachedir and is not in the + # portage group, then don't bother to set permissions + # on cache entries. This makes it possible to run + # egencache without any need to be a member of the + # portage group. + depcachedir_unshared = True + else: + cache_kwargs.update({ + 'gid' : portage_gid, + 'perms' : 0o664 + }) + # If secpass < 1, we don't want to write to the cache # since then we won't be able to apply group permissions # to the cache entries/directories. - if secpass < 1 or not depcachedir_w_ok: + if (secpass < 1 and not depcachedir_unshared) or not depcachedir_w_ok: for x in self.porttrees: + self.auxdb[x] = volatile.database( + self.depcachedir, x, self._known_keys, + **cache_kwargs) try: - db_ro = self.auxdbmodule(self.depcachedir, x, - filtered_auxdbkeys, readonly=True, **cache_kwargs) + self._ro_auxdb[x] = self.auxdbmodule(self.depcachedir, x, + self._known_keys, readonly=True, **cache_kwargs) except CacheError: - self.auxdb[x] = volatile.database( - self.depcachedir, x, filtered_auxdbkeys, - **cache_kwargs) - else: - self.auxdb[x] = metadata_overlay.database( - self.depcachedir, x, filtered_auxdbkeys, - db_rw=volatile.database, db_ro=db_ro, - **cache_kwargs) + pass else: for x in self.porttrees: if x in self.auxdb: continue # location, label, auxdbkeys self.auxdb[x] = self.auxdbmodule( - self.depcachedir, x, filtered_auxdbkeys, **cache_kwargs) - if self.auxdbmodule is metadata_overlay.database: - self.auxdb[x].db_ro.ec = self._repo_info[x].eclass_db + self.depcachedir, x, self._known_keys, **cache_kwargs) if "metadata-transfer" not in self.settings.features: for x in self.porttrees: if x in self._pregen_auxdb: continue - if os.path.isdir(os.path.join(x, "metadata", "cache")): - self._pregen_auxdb[x] = self.metadbmodule( - x, "metadata/cache", filtered_auxdbkeys, readonly=True) - try: - self._pregen_auxdb[x].ec = self._repo_info[x].eclass_db - except AttributeError: - pass + cache = self._create_pregen_cache(x) + if cache is not None: + self._pregen_auxdb[x] = cache # Selectively cache metadata in order to optimize dep matching. self._aux_cache_keys = set( ["DEPEND", "EAPI", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", @@ -210,18 +193,28 @@ class portdbapi(dbapi): self._aux_cache = {} self._broken_ebuilds = set() + def _create_pregen_cache(self, tree): + conf = self.repositories.get_repo_for_location(tree) + cache = conf.get_pregenerated_cache( + self._known_keys, readonly=True) + if cache is not None: + try: + cache.ec = self.repositories.get_repo_for_location(tree).eclass_db + except AttributeError: + pass + return cache + def _init_cache_dirs(self): """Create /var/cache/edb/dep and adjust permissions for the portage group.""" dirmode = 0o2070 - filemode = 0o60 modemask = 0o2 try: ensure_dirs(self.depcachedir, gid=portage_gid, mode=dirmode, mask=modemask) - except PortageException as e: + except PortageException: pass def close_caches(self): @@ -260,7 +253,7 @@ class portdbapi(dbapi): @param canonical_repo_path: the canonical path of a repository, as resolved by os.path.realpath() @type canonical_repo_path: String - @returns: The repo_name for the corresponding repository, or None + @return: The repo_name for the corresponding repository, or None if the path does not correspond a known repository @rtype: String or None """ @@ -332,63 +325,33 @@ class portdbapi(dbapi): return (filename, x) return (None, 0) - def _metadata_process(self, cpv, ebuild_path, repo_path): - """ - Create an EbuildMetadataPhase instance to generate metadata for the - give ebuild. - @rtype: EbuildMetadataPhase - @returns: A new EbuildMetadataPhase instance, or None if the - metadata cache is already valid. - """ - metadata, st, emtime = self._pull_valid_cache(cpv, ebuild_path, repo_path) - if metadata is not None: - return None - - process = EbuildMetadataPhase(cpv=cpv, ebuild_path=ebuild_path, - ebuild_mtime=emtime, metadata_callback=self._metadata_callback, - portdb=self, repo_path=repo_path, settings=self.doebuild_settings) - return process - - def _metadata_callback(self, cpv, ebuild_path, repo_path, metadata, mtime): - - i = metadata - if hasattr(metadata, "items"): - i = iter(metadata.items()) - metadata = dict(i) - - if metadata.get("INHERITED", False): - metadata["_eclasses_"] = self._repo_info[repo_path - ].eclass_db.get_eclass_data(metadata["INHERITED"].split()) - else: - metadata["_eclasses_"] = {} - - metadata.pop("INHERITED", None) - metadata["_mtime_"] = mtime - - eapi = metadata.get("EAPI") - if not eapi or not eapi.strip(): - eapi = "0" - metadata["EAPI"] = eapi - if not eapi_is_supported(eapi): - for k in set(metadata).difference(("_mtime_", "_eclasses_")): - metadata[k] = "" - metadata["EAPI"] = "-" + eapi.lstrip("-") + def _write_cache(self, cpv, repo_path, metadata, ebuild_hash): try: - self.auxdb[repo_path][cpv] = metadata + cache = self.auxdb[repo_path] + chf = cache.validation_chf + metadata['_%s_' % chf] = getattr(ebuild_hash, chf) except CacheError: # Normally this shouldn't happen, so we'll show # a traceback for debugging purposes. traceback.print_exc() - return metadata + cache = None + + if cache is not None: + try: + cache[cpv] = metadata + except CacheError: + # Normally this shouldn't happen, so we'll show + # a traceback for debugging purposes. + traceback.print_exc() def _pull_valid_cache(self, cpv, ebuild_path, repo_path): try: - # Don't use unicode-wrapped os module, for better performance. - st = _os.stat(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict')) - emtime = st[stat.ST_MTIME] - except OSError: + ebuild_hash = eclass_cache.hashed_path(ebuild_path) + # snag mtime since we use it later, and to trigger stat failure + # if it doesn't exist + ebuild_hash.mtime + except FileNotFound: writemsg(_("!!! aux_get(): ebuild for " \ "'%s' does not exist at:\n") % (cpv,), noiselevel=-1) writemsg("!!! %s\n" % ebuild_path, noiselevel=-1) @@ -401,39 +364,39 @@ class portdbapi(dbapi): pregen_auxdb = self._pregen_auxdb.get(repo_path) if pregen_auxdb is not None: auxdbs.append(pregen_auxdb) + ro_auxdb = self._ro_auxdb.get(repo_path) + if ro_auxdb is not None: + auxdbs.append(ro_auxdb) auxdbs.append(self.auxdb[repo_path]) - eclass_db = self._repo_info[repo_path].eclass_db + eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db - doregen = True for auxdb in auxdbs: try: metadata = auxdb[cpv] except KeyError: - pass + continue except CacheError: - if auxdb is not pregen_auxdb: + if not auxdb.readonly: try: del auxdb[cpv] - except KeyError: - pass - except CacheError: + except (KeyError, CacheError): pass - else: - eapi = metadata.get('EAPI', '').strip() - if not eapi: - eapi = '0' - if not (eapi[:1] == '-' and eapi_is_supported(eapi[1:])) and \ - emtime == metadata['_mtime_'] and \ - eclass_db.is_eclass_data_valid(metadata['_eclasses_']): - doregen = False - - if not doregen: + continue + eapi = metadata.get('EAPI', '').strip() + if not eapi: + eapi = '0' + metadata['EAPI'] = eapi + if not eapi_is_supported(eapi): + # Since we're supposed to be able to efficiently obtain the + # EAPI from _parse_eapi_ebuild_head, we disregard cache entries + # for unsupported EAPIs. + continue + if auxdb.validate_entry(metadata, ebuild_hash, eclass_db): break - - if doregen: + else: metadata = None - return (metadata, st, emtime) + return (metadata, ebuild_hash) def aux_get(self, mycpv, mylist, mytree=None, myrepo=None): "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc." @@ -445,15 +408,22 @@ class portdbapi(dbapi): if mytree is None: raise KeyError(myrepo) - if not mytree: + if mytree is not None and len(self.porttrees) == 1 \ + and mytree == self.porttrees[0]: + # mytree matches our only tree, so it's safe to + # ignore mytree and cache the result + mytree = None + myrepo = None + + if mytree is None: cache_me = True - if not mytree and not self._known_keys.intersection( + if mytree is None and not self._known_keys.intersection( mylist).difference(self._aux_cache_keys): aux_cache = self._aux_cache.get(mycpv) if aux_cache is not None: return [aux_cache.get(x, "") for x in mylist] cache_me = True - global auxdbkeys, auxdbkeylen + try: cat, pkg = mycpv.split("/", 1) except ValueError: @@ -467,60 +437,35 @@ class portdbapi(dbapi): _("ebuild not found for '%s'") % mycpv, noiselevel=1) raise KeyError(mycpv) - mydata, st, emtime = self._pull_valid_cache(mycpv, myebuild, mylocation) + mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation) doregen = mydata is None if doregen: if myebuild in self._broken_ebuilds: raise KeyError(mycpv) - self.doebuild_settings.setcpv(mycpv) - eapi = None - - if eapi is None and \ - 'parse-eapi-ebuild-head' in self.doebuild_settings.features: - eapi = portage._parse_eapi_ebuild_head(io.open( - _unicode_encode(myebuild, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], - errors='replace')) - - if eapi is not None: - self.doebuild_settings.configdict['pkg']['EAPI'] = eapi - - if eapi is not None and not portage.eapi_is_supported(eapi): - mydata = self._metadata_callback( - mycpv, myebuild, mylocation, {'EAPI':eapi}, emtime) - else: - proc = EbuildMetadataPhase(cpv=mycpv, ebuild_path=myebuild, - ebuild_mtime=emtime, - metadata_callback=self._metadata_callback, portdb=self, - repo_path=mylocation, - scheduler=PollScheduler().sched_iface, - settings=self.doebuild_settings) + proc = EbuildMetadataPhase(cpv=mycpv, + ebuild_hash=ebuild_hash, portdb=self, + repo_path=mylocation, scheduler=self._scheduler, + settings=self.doebuild_settings) - proc.start() - proc.wait() + proc.start() + proc.wait() - if proc.returncode != os.EX_OK: - self._broken_ebuilds.add(myebuild) - raise KeyError(mycpv) + if proc.returncode != os.EX_OK: + self._broken_ebuilds.add(myebuild) + raise KeyError(mycpv) - mydata = proc.metadata + mydata = proc.metadata - # do we have a origin repository name for the current package mydata["repository"] = self.repositories.get_name_for_location(mylocation) - mydata["INHERITED"] = ' '.join(mydata.get("_eclasses_", [])) - mydata["_mtime_"] = st[stat.ST_MTIME] - + mydata["_mtime_"] = ebuild_hash.mtime eapi = mydata.get("EAPI") if not eapi: eapi = "0" mydata["EAPI"] = eapi - if not eapi_is_supported(eapi): - for k in set(mydata).difference(("_mtime_", "_eclasses_")): - mydata[k] = "" - mydata["EAPI"] = "-" + eapi.lstrip("-") + if eapi_is_supported(eapi): + mydata["INHERITED"] = " ".join(mydata.get("_eclasses_", [])) #finally, we look at our internal cache entry and return the requested data. returnme = [mydata.get(x, "") for x in mylist] @@ -546,7 +491,7 @@ class portdbapi(dbapi): @param mytree: The canonical path of the tree in which the ebuild is located, or None for automatic lookup @type mypkg: String - @returns: A dict which maps each file name to a set of alternative + @return: A dict which maps each file name to a set of alternative URIs. @rtype: dict """ @@ -565,7 +510,7 @@ class portdbapi(dbapi): # since callers already handle it. raise portage.exception.InvalidDependString( "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ - (mypkg, eapi.lstrip("-"))) + (mypkg, eapi)) return _parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris}, use=useflags) @@ -576,7 +521,9 @@ class portdbapi(dbapi): if myebuild is None: raise AssertionError(_("ebuild not found for '%s'") % mypkg) pkgdir = os.path.dirname(myebuild) - mf = Manifest(pkgdir, self.settings["DISTDIR"]) + mf = self.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))).load_manifest( + pkgdir, self.settings["DISTDIR"]) checksums = mf.getDigests() if not checksums: if debug: @@ -597,7 +544,7 @@ class portdbapi(dbapi): mystat = None try: mystat = os.stat(file_path) - except OSError as e: + except OSError: pass if mystat is None: existing_size = 0 @@ -644,7 +591,9 @@ class portdbapi(dbapi): if myebuild is None: raise AssertionError(_("ebuild not found for '%s'") % mypkg) pkgdir = os.path.dirname(myebuild) - mf = Manifest(pkgdir, self.settings["DISTDIR"]) + mf = self.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + mf = mf.load_manifest(pkgdir, self.settings["DISTDIR"]) mysums = mf.getDigests() failures = {} @@ -706,15 +655,22 @@ class portdbapi(dbapi): return l def cp_list(self, mycp, use_cache=1, mytree=None): + # NOTE: Cache can be safely shared with the match cache, since the + # match cache uses the result from dep_expand for the cache_key. + if self.frozen and mytree is not None \ + and len(self.porttrees) == 1 \ + and mytree == self.porttrees[0]: + # mytree matches our only tree, so it's safe to + # ignore mytree and cache the result + mytree = None + if self.frozen and mytree is None: cachelist = self.xcache["cp-list"].get(mycp) if cachelist is not None: # Try to propagate this to the match-all cache here for # repoman since he uses separate match-all caches for each - # profile (due to old-style virtuals). Do not propagate - # old-style virtuals since cp_list() doesn't expand them. - if not (not cachelist and mycp.startswith("virtual/")): - self.xcache["match-all"][mycp] = cachelist + # profile (due to differences in _get_implicit_iuse). + self.xcache["match-all"][(mycp, mycp)] = cachelist return cachelist[:] mysplit = mycp.split("/") invalid_category = mysplit[0] not in self._categories @@ -752,7 +708,7 @@ class portdbapi(dbapi): writemsg(_("\nInvalid ebuild version: %s\n") % \ os.path.join(oroot, mycp, x), noiselevel=-1) continue - d[mysplit[0]+"/"+pf] = None + d[_pkg_str(mysplit[0]+"/"+pf)] = None if invalid_category and d: writemsg(_("\n!!! '%s' has a category that is not listed in " \ "%setc/portage/categories\n") % \ @@ -766,14 +722,11 @@ class portdbapi(dbapi): if self.frozen and mytree is None: cachelist = mylist[:] self.xcache["cp-list"][mycp] = cachelist - # Do not propagate old-style virtuals since - # cp_list() doesn't expand them. - if not (not cachelist and mycp.startswith("virtual/")): - self.xcache["match-all"][mycp] = cachelist + self.xcache["match-all"][(mycp, mycp)] = cachelist return mylist def freeze(self): - for x in "bestmatch-visible", "cp-list", "list-visible", "match-all", \ + for x in "bestmatch-visible", "cp-list", "match-all", \ "match-all-cpv-only", "match-visible", "minimum-all", \ "minimum-visible": self.xcache[x]={} @@ -785,12 +738,12 @@ class portdbapi(dbapi): def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None): "caching match function; very trick stuff" - #if no updates are being made to the tree, we can consult our xcache... - if self.frozen: - try: - return self.xcache[level][origdep][:] - except KeyError: - pass + if level == "list-visible": + level = "match-visible" + warnings.warn("The 'list-visible' mode of " + "portage.dbapi.porttree.portdbapi.xmatch " + "has been renamed to match-visible", + DeprecationWarning, stacklevel=2) if mydep is None: #this stuff only runs on first call of xmatch() @@ -798,12 +751,24 @@ class portdbapi(dbapi): mydep = dep_expand(origdep, mydb=self, settings=self.settings) mykey = mydep.cp + #if no updates are being made to the tree, we can consult our xcache... + cache_key = None + if self.frozen: + cache_key = (mydep, mydep.unevaluated_atom) + try: + return self.xcache[level][cache_key][:] + except KeyError: + pass + myval = None mytree = None if mydep.repo is not None: mytree = self.treemap.get(mydep.repo) if mytree is None: - myval = [] + if level.startswith("match-"): + myval = [] + else: + myval = "" if myval is not None: # Unknown repo, empty result. @@ -822,27 +787,8 @@ class portdbapi(dbapi): myval = match_from_list(mydep, self.cp_list(mykey, mytree=mytree)) - elif level == "list-visible": - #a list of all visible packages, not called directly (just by xmatch()) - #myval = self.visible(self.cp_list(mykey)) - - myval = self.gvisible(self.visible( - self.cp_list(mykey, mytree=mytree))) - elif level == "minimum-all": - # Find the minimum matching version. This is optimized to - # minimize the number of metadata accesses (improves performance - # especially in cases where metadata needs to be generated). - if mydep == mykey: - cpv_iter = iter(self.cp_list(mykey, mytree=mytree)) - else: - cpv_iter = self._iter_match(mydep, - self.cp_list(mykey, mytree=mytree)) - try: - myval = next(cpv_iter) - except StopIteration: - myval = "" - - elif level in ("minimum-visible", "bestmatch-visible"): + elif level in ("bestmatch-visible", "match-all", "match-visible", + "minimum-all", "minimum-visible"): # Find the minimum matching visible version. This is optimized to # minimize the number of metadata accesses (improves performance # especially in cases where metadata needs to be generated). @@ -851,158 +797,172 @@ class portdbapi(dbapi): else: mylist = match_from_list(mydep, self.cp_list(mykey, mytree=mytree)) - myval = "" - settings = self.settings - local_config = settings.local_config + + visibility_filter = level not in ("match-all", "minimum-all") + single_match = level not in ("match-all", "match-visible") + myval = [] aux_keys = list(self._aux_cache_keys) - if level == "minimum-visible": + if level == "bestmatch-visible": + iterfunc = reversed + else: iterfunc = iter + + if mydep.repo is not None: + repos = [mydep.repo] else: - iterfunc = reversed + # We iterate over self.porttrees, since it's common to + # tweak this attribute in order to adjust match behavior. + repos = [] + for tree in reversed(self.porttrees): + repos.append(self.repositories.get_name_for_location(tree)) + for cpv in iterfunc(mylist): - try: - metadata = dict(zip(aux_keys, - self.aux_get(cpv, aux_keys))) - except KeyError: - # ebuild masked by corruption - continue - if not eapi_is_supported(metadata["EAPI"]): - continue - if mydep.slot and mydep.slot != metadata["SLOT"]: - continue - if settings._getMissingKeywords(cpv, metadata): - continue - if settings._getMaskAtom(cpv, metadata): - continue - if settings._getProfileMaskAtom(cpv, metadata): - continue - if local_config: - metadata["USE"] = "" - if "?" in metadata["LICENSE"] or "?" in metadata["PROPERTIES"]: - self.doebuild_settings.setcpv(cpv, mydb=metadata) - metadata["USE"] = self.doebuild_settings.get("USE", "") + for repo in repos: try: - if settings._getMissingLicenses(cpv, metadata): - continue - if settings._getMissingProperties(cpv, metadata): - continue - except InvalidDependString: + metadata = dict(zip(aux_keys, + self.aux_get(cpv, aux_keys, myrepo=repo))) + except KeyError: + # ebuild not in this repo, or masked by corruption continue - if mydep.use: - has_iuse = False - for has_iuse in self._iter_match_use(mydep, [cpv]): - break - if not has_iuse: + + if visibility_filter and not self._visible(cpv, metadata): continue - myval = cpv - break + + if mydep.slot is not None and \ + mydep.slot != metadata["SLOT"]: + continue + + if mydep.unevaluated_atom.use is not None and \ + not self._match_use(mydep, cpv, metadata): + continue + + myval.append(cpv) + # only yield a given cpv once + break + + if myval and single_match: + break + + if single_match: + if myval: + myval = myval[0] + else: + myval = "" + elif level == "bestmatch-list": #dep match -- find best match but restrict search to sublist - #no point in calling xmatch again since we're not caching list deps - + warnings.warn("The 'bestmatch-list' mode of " + "portage.dbapi.porttree.portdbapi.xmatch is deprecated", + DeprecationWarning, stacklevel=2) myval = best(list(self._iter_match(mydep, mylist))) elif level == "match-list": #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible()) - + warnings.warn("The 'match-list' mode of " + "portage.dbapi.porttree.portdbapi.xmatch is deprecated", + DeprecationWarning, stacklevel=2) myval = list(self._iter_match(mydep, mylist)) - elif level == "match-visible": - #dep match -- find all visible matches - #get all visible packages, then get the matching ones - myval = list(self._iter_match(mydep, - self.xmatch("list-visible", mykey, mydep=Atom(mykey), mykey=mykey))) - elif level == "match-all": - #match *all* visible *and* masked packages - if mydep == mykey: - myval = self.cp_list(mykey, mytree=mytree) - else: - myval = list(self._iter_match(mydep, - self.cp_list(mykey, mytree=mytree))) else: raise AssertionError( "Invalid level argument: '%s'" % level) - if self.frozen and (level not in ["match-list", "bestmatch-list"]): - self.xcache[level][mydep] = myval - if origdep and origdep != mydep: - self.xcache[level][origdep] = myval - return myval[:] + if self.frozen: + xcache_this_level = self.xcache.get(level) + if xcache_this_level is not None: + xcache_this_level[cache_key] = myval + if not isinstance(myval, _pkg_str): + myval = myval[:] + + return myval def match(self, mydep, use_cache=1): return self.xmatch("match-visible", mydep) - def visible(self, mylist): - """two functions in one. Accepts a list of cpv values and uses the package.mask *and* - packages file to remove invisible entries, returning remaining items. This function assumes - that all entries in mylist have the same category and package name.""" - if not mylist: - return [] - - db_keys = ["SLOT"] - visible = [] - getMaskAtom = self.settings._getMaskAtom - getProfileMaskAtom = self.settings._getProfileMaskAtom - for cpv in mylist: - try: - metadata = dict(zip(db_keys, self.aux_get(cpv, db_keys))) - except KeyError: - # masked by corruption - continue - if not metadata["SLOT"]: - continue - if getMaskAtom(cpv, metadata): - continue - if getProfileMaskAtom(cpv, metadata): - continue - visible.append(cpv) - return visible - - def gvisible(self,mylist): - "strip out group-masked (not in current group) entries" + def gvisible(self, mylist): + warnings.warn("The 'gvisible' method of " + "portage.dbapi.porttree.portdbapi " + "is deprecated", + DeprecationWarning, stacklevel=2) + return list(self._iter_visible(iter(mylist))) - if mylist is None: + def visible(self, cpv_iter): + warnings.warn("The 'visible' method of " + "portage.dbapi.porttree.portdbapi " + "is deprecated", + DeprecationWarning, stacklevel=2) + if cpv_iter is None: return [] - newlist=[] + return list(self._iter_visible(iter(cpv_iter))) + + def _iter_visible(self, cpv_iter, myrepo=None): + """ + Return a new list containing only visible packages. + """ aux_keys = list(self._aux_cache_keys) metadata = {} - local_config = self.settings.local_config - chost = self.settings.get('CHOST', '') - accept_chost = self.settings._accept_chost - for mycpv in mylist: - metadata.clear() - try: - metadata.update(zip(aux_keys, self.aux_get(mycpv, aux_keys))) - except KeyError: - continue - except PortageException as e: - writemsg("!!! Error: aux_get('%s', %s)\n" % (mycpv, aux_keys), - noiselevel=-1) - writemsg("!!! %s\n" % (e,), noiselevel=-1) - del e - continue - eapi = metadata["EAPI"] - if not eapi_is_supported(eapi): - continue - if _eapi_is_deprecated(eapi): - continue - if self.settings._getMissingKeywords(mycpv, metadata): - continue - if local_config: - metadata['CHOST'] = chost - if not accept_chost(mycpv, metadata): - continue - metadata["USE"] = "" - if "?" in metadata["LICENSE"] or "?" in metadata["PROPERTIES"]: - self.doebuild_settings.setcpv(mycpv, mydb=metadata) - metadata['USE'] = self.doebuild_settings['PORTAGE_USE'] + + if myrepo is not None: + repos = [myrepo] + else: + # We iterate over self.porttrees, since it's common to + # tweak this attribute in order to adjust match behavior. + repos = [] + for tree in reversed(self.porttrees): + repos.append(self.repositories.get_name_for_location(tree)) + + for mycpv in cpv_iter: + for repo in repos: + metadata.clear() try: - if self.settings._getMissingLicenses(mycpv, metadata): - continue - if self.settings._getMissingProperties(mycpv, metadata): - continue - except InvalidDependString: + metadata.update(zip(aux_keys, + self.aux_get(mycpv, aux_keys, myrepo=repo))) + except KeyError: + continue + except PortageException as e: + writemsg("!!! Error: aux_get('%s', %s)\n" % + (mycpv, aux_keys), noiselevel=-1) + writemsg("!!! %s\n" % (e,), noiselevel=-1) + del e continue - newlist.append(mycpv) - return newlist + + if not self._visible(mycpv, metadata): + continue + + yield mycpv + # only yield a given cpv once + break + + def _visible(self, cpv, metadata): + eapi = metadata["EAPI"] + if not eapi_is_supported(eapi): + return False + if _eapi_is_deprecated(eapi): + return False + if not metadata["SLOT"]: + return False + + settings = self.settings + if settings._getMaskAtom(cpv, metadata): + return False + if settings._getMissingKeywords(cpv, metadata): + return False + if settings.local_config: + metadata['CHOST'] = settings.get('CHOST', '') + if not settings._accept_chost(cpv, metadata): + return False + metadata["USE"] = "" + if "?" in metadata["LICENSE"] or \ + "?" in metadata["PROPERTIES"]: + self.doebuild_settings.setcpv(cpv, mydb=metadata) + metadata['USE'] = self.doebuild_settings['PORTAGE_USE'] + try: + if settings._getMissingLicenses(cpv, metadata): + return False + if settings._getMissingProperties(cpv, metadata): + return False + except InvalidDependString: + return False + + return True def close_portdbapi_caches(): for i in portdbapi.portdbapi_instances: @@ -1011,7 +971,7 @@ def close_portdbapi_caches(): portage.process.atexit_register(portage.portageexit) class portagetree(object): - def __init__(self, root=None, virtual=None, settings=None): + def __init__(self, root=None, virtual=DeprecationWarning, settings=None): """ Constructor for a PortageTree @@ -1034,8 +994,14 @@ class portagetree(object): "settings['ROOT'] instead.", DeprecationWarning, stacklevel=2) + if virtual is not DeprecationWarning: + warnings.warn("The 'virtual' parameter of the " + "portage.dbapi.porttree.portagetree" + " constructor is unused", + DeprecationWarning, stacklevel=2) + self.portroot = settings["PORTDIR"] - self.virtual = virtual + self.__virtual = virtual self.dbapi = portdbapi(mysettings=settings) @property @@ -1044,9 +1010,17 @@ class portagetree(object): "portage.dbapi.porttree.portagetree" + \ " is deprecated. Use " + \ "settings['ROOT'] instead.", - DeprecationWarning, stacklevel=2) + DeprecationWarning, stacklevel=3) return self.settings['ROOT'] + @property + def virtual(self): + warnings.warn("The 'virtual' attribute of " + \ + "portage.dbapi.porttree.portagetree" + \ + " is deprecated.", + DeprecationWarning, stacklevel=3) + return self.__virtual + def dep_bestmatch(self,mydep): "compatibility method" mymatch = self.dbapi.xmatch("bestmatch-visible",mydep) @@ -1077,17 +1051,14 @@ class portagetree(object): psplit = pkgsplit(mysplit[1]) return "/".join([self.portroot, mysplit[0], psplit[0], mysplit[1]])+".ebuild" - def depcheck(self, mycheck, use="yes", myusesplit=None): - return dep_check(mycheck, self.dbapi, use=use, myuse=myusesplit) - def getslot(self,mycatpkg): "Get a slot for a catpkg; assume it exists." myslot = "" try: myslot = self.dbapi.aux_get(mycatpkg, ["SLOT"])[0] - except SystemExit as e: + except SystemExit: raise - except Exception as e: + except Exception: pass return myslot @@ -1148,7 +1119,7 @@ def _parse_uri_map(cpv, metadata, use=None): while myuris: uri = myuris.pop() if myuris and myuris[-1] == "->": - operator = myuris.pop() + myuris.pop() distfile = myuris.pop() else: distfile = os.path.basename(uri) @@ -1163,6 +1134,5 @@ def _parse_uri_map(cpv, metadata, use=None): uri_map[distfile] = uri_set uri_set.add(uri) uri = None - operator = None return uri_map diff --git a/portage_with_autodep/pym/portage/dbapi/porttree.pyo b/portage_with_autodep/pym/portage/dbapi/porttree.pyo Binary files differnew file mode 100644 index 0000000..fb57919 --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/porttree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/vartree.py b/portage_with_autodep/pym/portage/dbapi/vartree.py index 7f7873b..517c873 100644 --- a/portage_with_autodep/pym/portage/dbapi/vartree.py +++ b/portage_with_autodep/pym/portage/dbapi/vartree.py @@ -1,4 +1,4 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -18,7 +18,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.locks:lockdir,unlockdir,lockfile,unlockfile', 'portage.output:bold,colorize', 'portage.package.ebuild.doebuild:doebuild_environment,' + \ - '_spawn_phase', + '_merge_unicode_error', '_spawn_phase', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', 'portage.update:fixdbentries', 'portage.util:apply_secpass_permissions,ConfigProtect,ensure_dirs,' + \ @@ -27,10 +27,13 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.util.digraph:digraph', 'portage.util.env_update:env_update', 'portage.util.listdir:dircache,listdir', + 'portage.util.movefile:movefile', 'portage.util._dyn_libs.PreservedLibsRegistry:PreservedLibsRegistry', 'portage.util._dyn_libs.LinkageMapELF:LinkageMapELF@LinkageMap', - 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,pkgcmp,' + \ - '_pkgsplit@pkgsplit', + 'portage.versions:best,catpkgsplit,catsplit,cpv_getkey,vercmp,' + \ + '_pkgsplit@pkgsplit,_pkg_str', + 'subprocess', + 'tarfile', ) from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, \ @@ -41,12 +44,12 @@ from portage.exception import CommandNotFound, \ InvalidData, InvalidLocation, InvalidPackageName, \ FileNotFound, PermissionDenied, UnsupportedAPIException from portage.localization import _ -from portage.util.movefile import movefile from portage import abssymlink, _movefile, bsd_chflags # This is a special version of the os module, wrapped for unicode support. from portage import os +from portage import shutil from portage import _encodings from portage import _os_merge from portage import _selinux_merge @@ -60,13 +63,15 @@ from _emerge.PollScheduler import PollScheduler from _emerge.MiscFunctionsProcess import MiscFunctionsProcess import errno +import fnmatch import gc +import grp import io from itertools import chain import logging import os as _os +import pwd import re -import shutil import stat import sys import tempfile @@ -82,6 +87,9 @@ except ImportError: if sys.hexversion >= 0x3000000: basestring = str long = int + _unicode = str +else: + _unicode = unicode class vardbapi(dbapi): @@ -129,12 +137,11 @@ class vardbapi(dbapi): if settings is None: settings = portage.settings self.settings = settings - self.root = settings['ROOT'] - if _unused_param is not None and _unused_param != self.root: - warnings.warn("The first parameter of the " + \ - "portage.dbapi.vartree.vardbapi" + \ - " constructor is now unused. Use " + \ + if _unused_param is not None and _unused_param != settings['ROOT']: + warnings.warn("The first parameter of the " + "portage.dbapi.vartree.vardbapi" + " constructor is now unused. Use " "settings['ROOT'] instead.", DeprecationWarning, stacklevel=2) @@ -148,14 +155,14 @@ class vardbapi(dbapi): self._fs_lock_count = 0 if vartree is None: - vartree = portage.db[self.root]["vartree"] + vartree = portage.db[settings['EROOT']]['vartree'] self.vartree = vartree self._aux_cache_keys = set( ["BUILD_TIME", "CHOST", "COUNTER", "DEPEND", "DESCRIPTION", "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", "RESTRICT" , "SLOT", "USE", "DEFINED_PHASES", - "REQUIRED_USE"]) + ]) self._aux_cache_obj = None self._aux_cache_filename = os.path.join(self._eroot, CACHE_PATH, "vdb_metadata.pickle") @@ -164,7 +171,7 @@ class vardbapi(dbapi): self._plib_registry = None if _ENABLE_PRESERVE_LIBS: - self._plib_registry = PreservedLibsRegistry(self.root, + self._plib_registry = PreservedLibsRegistry(settings["ROOT"], os.path.join(self._eroot, PRIVATE_PATH, "preserved_libs_registry")) @@ -175,6 +182,15 @@ class vardbapi(dbapi): self._cached_counter = None + @property + def root(self): + warnings.warn("The root attribute of " + "portage.dbapi.vartree.vardbapi" + " is deprecated. Use " + "settings['ROOT'] instead.", + DeprecationWarning, stacklevel=3) + return self.settings['ROOT'] + def getpath(self, mykey, filename=None): # This is an optimized hotspot, so don't use unicode-wrapped # os module and don't use os.path.join(). @@ -373,7 +389,7 @@ class vardbapi(dbapi): continue if len(mysplit) > 1: if ps[0] == mysplit[1]: - returnme.append(mysplit[0]+"/"+x) + returnme.append(_pkg_str(mysplit[0]+"/"+x)) self._cpv_sort_ascending(returnme) if use_cache: self.cpcache[mycp] = [mystat, returnme[:]] @@ -472,6 +488,7 @@ class vardbapi(dbapi): "caching match function" mydep = dep_expand( origdep, mydb=self, use_cache=use_cache, settings=self.settings) + cache_key = (mydep, mydep.unevaluated_atom) mykey = dep_getkey(mydep) mycat = catsplit(mykey)[0] if not use_cache: @@ -493,8 +510,8 @@ class vardbapi(dbapi): if mydep not in self.matchcache[mycat]: mymatch = list(self._iter_match(mydep, self.cp_list(mydep.cp, use_cache=use_cache))) - self.matchcache[mycat][mydep] = mymatch - return self.matchcache[mycat][mydep][:] + self.matchcache[mycat][cache_key] = mymatch + return self.matchcache[mycat][cache_key][:] def findname(self, mycpv, myrepo=None): return self.getpath(str(mycpv), filename=catsplit(mycpv)[1]+".ebuild") @@ -555,8 +572,11 @@ class vardbapi(dbapi): aux_cache = mypickle.load() f.close() del f - except (IOError, OSError, EOFError, ValueError, pickle.UnpicklingError) as e: - if isinstance(e, pickle.UnpicklingError): + except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError) as e: + if isinstance(e, EnvironmentError) and \ + getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): + pass + else: writemsg(_unicode_decode(_("!!! Error loading '%s': %s\n")) % \ (self._aux_cache_filename, e), noiselevel=-1) del e @@ -610,7 +630,8 @@ class vardbapi(dbapi): cache_these_wants.add(x) if not cache_these_wants: - return self._aux_get(mycpv, wants) + mydata = self._aux_get(mycpv, wants) + return [mydata[x] for x in wants] cache_these = set(self._aux_cache_keys) cache_these.update(cache_these_wants) @@ -655,16 +676,15 @@ class vardbapi(dbapi): if pull_me: # pull any needed data and cache it aux_keys = list(pull_me) - for k, v in zip(aux_keys, - self._aux_get(mycpv, aux_keys, st=mydir_stat)): - mydata[k] = v + mydata.update(self._aux_get(mycpv, aux_keys, st=mydir_stat)) if not cache_valid or cache_these.difference(metadata): cache_data = {} if cache_valid and metadata: cache_data.update(metadata) for aux_key in cache_these: cache_data[aux_key] = mydata[aux_key] - self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data) + self._aux_cache["packages"][_unicode(mycpv)] = \ + (mydir_mtime, cache_data) self._aux_cache["modified"].add(mycpv) if _slot_re.match(mydata['SLOT']) is None: @@ -688,10 +708,11 @@ class vardbapi(dbapi): raise if not stat.S_ISDIR(st.st_mode): raise KeyError(mycpv) - results = [] + results = {} + env_keys = [] for x in wants: if x == "_mtime_": - results.append(st[stat.ST_MTIME]) + results[x] = st[stat.ST_MTIME] continue try: myf = io.open( @@ -703,16 +724,103 @@ class vardbapi(dbapi): myd = myf.read() finally: myf.close() - # Preserve \n for metadata that is known to - # contain multiple lines. - if self._aux_multi_line_re.match(x) is None: - myd = " ".join(myd.split()) except IOError: + if x not in self._aux_cache_keys and \ + self._aux_cache_keys_re.match(x) is None: + env_keys.append(x) + continue myd = _unicode_decode('') - if x == "EAPI" and not myd: - results.append(_unicode_decode('0')) - else: - results.append(myd) + + # Preserve \n for metadata that is known to + # contain multiple lines. + if self._aux_multi_line_re.match(x) is None: + myd = " ".join(myd.split()) + + results[x] = myd + + if env_keys: + env_results = self._aux_env_search(mycpv, env_keys) + for k in env_keys: + v = env_results.get(k) + if v is None: + v = _unicode_decode('') + if self._aux_multi_line_re.match(k) is None: + v = " ".join(v.split()) + results[k] = v + + if results.get("EAPI") == "": + results[_unicode_decode("EAPI")] = _unicode_decode('0') + + return results + + def _aux_env_search(self, cpv, variables): + """ + Search environment.bz2 for the specified variables. Returns + a dict mapping variables to values, and any variables not + found in the environment will not be included in the dict. + This is useful for querying variables like ${SRC_URI} and + ${A}, which are not saved in separate files but are available + in environment.bz2 (see bug #395463). + """ + env_file = self.getpath(cpv, filename="environment.bz2") + if not os.path.isfile(env_file): + return {} + bunzip2_cmd = portage.util.shlex_split( + self.settings.get("PORTAGE_BUNZIP2_COMMAND", "")) + if not bunzip2_cmd: + bunzip2_cmd = portage.util.shlex_split( + self.settings["PORTAGE_BZIP2_COMMAND"]) + bunzip2_cmd.append("-d") + args = bunzip2_cmd + ["-c", env_file] + try: + proc = subprocess.Popen(args, stdout=subprocess.PIPE) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + raise portage.exception.CommandNotFound(args[0]) + + # Parts of the following code are borrowed from + # filter-bash-environment.py (keep them in sync). + var_assign_re = re.compile(r'(^|^declare\s+-\S+\s+|^declare\s+|^export\s+)([^=\s]+)=("|\')?(.*)$') + close_quote_re = re.compile(r'(\\"|"|\')\s*$') + def have_end_quote(quote, line): + close_quote_match = close_quote_re.search(line) + return close_quote_match is not None and \ + close_quote_match.group(1) == quote + + variables = frozenset(variables) + results = {} + for line in proc.stdout: + line = _unicode_decode(line, + encoding=_encodings['content'], errors='replace') + var_assign_match = var_assign_re.match(line) + if var_assign_match is not None: + key = var_assign_match.group(2) + quote = var_assign_match.group(3) + if quote is not None: + if have_end_quote(quote, + line[var_assign_match.end(2)+2:]): + value = var_assign_match.group(4) + else: + value = [var_assign_match.group(4)] + for line in proc.stdout: + line = _unicode_decode(line, + encoding=_encodings['content'], + errors='replace') + value.append(line) + if have_end_quote(quote, line): + break + value = ''.join(value) + # remove trailing quote and whitespace + value = value.rstrip()[:-1] + else: + value = var_assign_match.group(4).rstrip() + + if key in variables: + results[key] = value + + proc.wait() + proc.stdout.close() return results def aux_update(self, cpv, values): @@ -758,8 +866,7 @@ class vardbapi(dbapi): @param myroot: ignored, self._eroot is used instead """ - myroot = None - new_vdb = False + del myroot counter = -1 try: cfile = io.open( @@ -768,8 +875,9 @@ class vardbapi(dbapi): mode='r', encoding=_encodings['repo.content'], errors='replace') except EnvironmentError as e: - new_vdb = not bool(self.cpv_all()) - if not new_vdb: + # Silently allow ENOENT since files under + # /var/cache/ are allowed to disappear. + if e.errno != errno.ENOENT: writemsg(_("!!! Unable to read COUNTER file: '%s'\n") % \ self._counter_path, noiselevel=-1) writemsg("!!! %s\n" % str(e), noiselevel=-1) @@ -806,10 +914,6 @@ class vardbapi(dbapi): if pkg_counter > max_counter: max_counter = pkg_counter - if counter < 0 and not new_vdb: - writemsg(_("!!! Initializing COUNTER to " \ - "value of %d\n") % max_counter, noiselevel=-1) - return max_counter + 1 def counter_tick_core(self, myroot=None, incrementing=1, mycpv=None): @@ -823,7 +927,7 @@ class vardbapi(dbapi): @param myroot: ignored, self._eroot is used instead @param mycpv: ignored @rtype: int - @returns: new counter value + @return: new counter value """ myroot = None mycpv = None @@ -959,7 +1063,7 @@ class vardbapi(dbapi): counter = int(counter) except ValueError: counter = 0 - return (cpv, counter, mtime) + return (_unicode(cpv), counter, mtime) class _owners_db(object): @@ -1149,24 +1253,38 @@ class vardbapi(dbapi): class vartree(object): "this tree will scan a var/db/pkg database located at root (passed to init)" - def __init__(self, root=None, virtual=None, categories=None, + def __init__(self, root=None, virtual=DeprecationWarning, categories=None, settings=None): if settings is None: settings = portage.settings - self.root = settings['ROOT'] - if root is not None and root != self.root: - warnings.warn("The 'root' parameter of the " + \ - "portage.dbapi.vartree.vartree" + \ - " constructor is now unused. Use " + \ + if root is not None and root != settings['ROOT']: + warnings.warn("The 'root' parameter of the " + "portage.dbapi.vartree.vartree" + " constructor is now unused. Use " "settings['ROOT'] instead.", DeprecationWarning, stacklevel=2) + if virtual is not DeprecationWarning: + warnings.warn("The 'virtual' parameter of the " + "portage.dbapi.vartree.vartree" + " constructor is unused", + DeprecationWarning, stacklevel=2) + self.settings = settings self.dbapi = vardbapi(settings=settings, vartree=self) self.populated = 1 + @property + def root(self): + warnings.warn("The root attribute of " + "portage.dbapi.vartree.vartree" + " is deprecated. Use " + "settings['ROOT'] instead.", + DeprecationWarning, stacklevel=3) + return self.settings['ROOT'] + def getpath(self, mykey, filename=None): return self.dbapi.getpath(mykey, filename=filename) @@ -1276,6 +1394,20 @@ class dblink(object): r')$' ) + # These files are generated by emerge, so we need to remove + # them when they are the only thing left in a directory. + _infodir_cleanup = frozenset(["dir", "dir.old"]) + + _ignored_unlink_errnos = ( + errno.EBUSY, errno.ENOENT, + errno.ENOTDIR, errno.EISDIR) + + _ignored_rmdir_errnos = ( + errno.EEXIST, errno.ENOTEMPTY, + errno.EBUSY, errno.ENOENT, + errno.ENOTDIR, errno.EISDIR, + errno.EPERM) + def __init__(self, cat, pkg, myroot=None, settings=None, treetype=None, vartree=None, blockers=None, scheduler=None, pipe=None): """ @@ -1300,22 +1432,23 @@ class dblink(object): raise TypeError("settings argument is required") mysettings = settings - myroot = settings['ROOT'] + self._eroot = mysettings['EROOT'] self.cat = cat self.pkg = pkg self.mycpv = self.cat + "/" + self.pkg - self.mysplit = list(catpkgsplit(self.mycpv)[1:]) - self.mysplit[0] = "%s/%s" % (self.cat, self.mysplit[0]) + if self.mycpv == settings.mycpv and \ + isinstance(settings.mycpv, _pkg_str): + self.mycpv = settings.mycpv + else: + self.mycpv = _pkg_str(self.mycpv) + self.mysplit = list(self.mycpv.cpv_split[1:]) + self.mysplit[0] = self.mycpv.cp self.treetype = treetype if vartree is None: - vartree = portage.db[myroot]["vartree"] + vartree = portage.db[self._eroot]["vartree"] self.vartree = vartree self._blockers = blockers self._scheduler = scheduler - - # WARNING: EROOT support is experimental and may be incomplete - # for cases in which EPREFIX is non-empty. - self._eroot = mysettings['EROOT'] self.dbroot = normalize_path(os.path.join(self._eroot, VDB_PATH)) self.dbcatdir = self.dbroot+"/"+cat self.dbpkgdir = self.dbcatdir+"/"+pkg @@ -1324,14 +1457,14 @@ class dblink(object): self.settings = mysettings self._verbose = self.settings.get("PORTAGE_VERBOSE") == "1" - self.myroot=myroot + self.myroot = self.settings['ROOT'] self._installed_instance = None self.contentscache = None self._contents_inodes = None self._contents_basenames = None self._linkmap_broken = False - self._md5_merge_map = {} - self._hash_key = (self.myroot, self.mycpv) + self._hardlink_merge_map = {} + self._hash_key = (self._eroot, self.mycpv) self._protect_obj = None self._pipe = pipe @@ -1610,7 +1743,7 @@ class dblink(object): PreservedLibsRegistry yet. @type preserve_paths: set @rtype: Integer - @returns: + @return: 1. os.EX_OK if everything went well. 2. return code of the failed phase (for prerm, postrm, cleanrm) """ @@ -1839,16 +1972,19 @@ class dblink(object): else: self.settings.pop("PORTAGE_LOG_FILE", None) - # Lock the config memory file to prevent symlink creation - # in merge_contents from overlapping with env-update. - self.vartree.dbapi._fs_lock() - try: - env_update(target_root=self.settings['ROOT'], - prev_mtimes=ldpath_mtimes, - contents=contents, env=self.settings.environ(), - writemsg_level=self._display_merge) - finally: - self.vartree.dbapi._fs_unlock() + env_update(target_root=self.settings['ROOT'], + prev_mtimes=ldpath_mtimes, + contents=contents, env=self.settings, + writemsg_level=self._display_merge, vardbapi=self.vartree.dbapi) + + unmerge_with_replacement = preserve_paths is not None + if not unmerge_with_replacement: + # When there's a replacement package which calls us via treewalk, + # treewalk will automatically call _prune_plib_registry for us. + # Otherwise, we need to call _prune_plib_registry ourselves. + # Don't pass in the "unmerge=True" flag here, since that flag + # is intended to be used _prior_ to unmerge, not after. + self._prune_plib_registry() return os.EX_OK @@ -1871,6 +2007,10 @@ class dblink(object): log_path=log_path, background=background, level=level, noiselevel=noiselevel) + def _show_unmerge(self, zing, desc, file_type, file_name): + self._display_merge("%s %s %s %s\n" % \ + (zing, desc.ljust(8), file_type, file_name)) + def _unmerge_pkgfiles(self, pkgfiles, others_in_slot): """ @@ -1887,6 +2027,9 @@ class dblink(object): os = _os_merge perf_md5 = perform_md5 showMessage = self._display_merge + show_unmerge = self._show_unmerge + ignored_unlink_errnos = self._ignored_unlink_errnos + ignored_rmdir_errnos = self._ignored_rmdir_errnos if not pkgfiles: showMessage(_("No package files given... Grabbing a set.\n")) @@ -1904,9 +2047,6 @@ class dblink(object): settings=self.settings, vartree=self.vartree, treetype="vartree", pipe=self._pipe)) - dest_root = self._eroot - dest_root_len = len(dest_root) - 1 - cfgfiledict = grabdict(self.vartree.dbapi._conf_mem_file) stale_confmem = [] protected_symlinks = {} @@ -1922,14 +2062,6 @@ class dblink(object): #process symlinks second-to-last, directories last. mydirs = set() - ignored_unlink_errnos = ( - errno.EBUSY, errno.ENOENT, - errno.ENOTDIR, errno.EISDIR) - ignored_rmdir_errnos = ( - errno.EEXIST, errno.ENOTEMPTY, - errno.EBUSY, errno.ENOENT, - errno.ENOTDIR, errno.EISDIR, - errno.EPERM) modprotect = os.path.join(self._eroot, "lib/modules/") def unlink(file_name, lstatobj): @@ -1965,10 +2097,6 @@ class dblink(object): # Restore the parent flags we saved before unlinking bsd_chflags.chflags(parent_name, pflags) - def show_unmerge(zing, desc, file_type, file_name): - showMessage("%s %s %s %s\n" % \ - (zing, desc.ljust(8), file_type, file_name)) - unmerge_desc = {} unmerge_desc["cfgpro"] = _("cfgpro") unmerge_desc["replaced"] = _("replaced") @@ -1980,14 +2108,12 @@ class dblink(object): unmerge_desc["!mtime"] = _("!mtime") unmerge_desc["!obj"] = _("!obj") unmerge_desc["!sym"] = _("!sym") + unmerge_desc["!prefix"] = _("!prefix") real_root = self.settings['ROOT'] real_root_len = len(real_root) - 1 - eroot_split_len = len(self.settings["EROOT"].split(os.sep)) - 1 + eroot = self.settings["EROOT"] - # These files are generated by emerge, so we need to remove - # them when they are the only thing left in a directory. - infodir_cleanup = frozenset(["dir", "dir.old"]) infodirs = frozenset(infodir for infodir in chain( self.settings.get("INFOPATH", "").split(":"), self.settings.get("INFODIR", "").split(":")) if infodir) @@ -2023,6 +2149,12 @@ class dblink(object): file_data = pkgfiles[objkey] file_type = file_data[0] + + # don't try to unmerge the prefix offset itself + if len(obj) <= len(eroot) or not obj.startswith(eroot): + show_unmerge("---", unmerge_desc["!prefix"], file_type, obj) + continue + statobj = None try: statobj = os.stat(obj) @@ -2216,78 +2348,13 @@ class dblink(object): elif pkgfiles[objkey][0] == "dev": show_unmerge("---", "", file_type, obj) - mydirs = sorted(mydirs) - mydirs.reverse() + self._unmerge_dirs(mydirs, infodirs_inodes, + protected_symlinks, unmerge_desc, unlink, os) + mydirs.clear() - for obj, inode_key in mydirs: - # Treat any directory named "info" as a candidate here, - # since it might have been in INFOPATH previously even - # though it may not be there now. - if inode_key in infodirs_inodes or \ - os.path.basename(obj) == "info": - try: - remaining = os.listdir(obj) - except OSError: - pass - else: - cleanup_info_dir = () - if remaining and \ - len(remaining) <= len(infodir_cleanup): - if not set(remaining).difference(infodir_cleanup): - cleanup_info_dir = remaining - - for child in cleanup_info_dir: - child = os.path.join(obj, child) - try: - lstatobj = os.lstat(child) - if stat.S_ISREG(lstatobj.st_mode): - unlink(child, lstatobj) - show_unmerge("<<<", "", "obj", child) - except EnvironmentError as e: - if e.errno not in ignored_unlink_errnos: - raise - del e - show_unmerge("!!!", "", "obj", child) - try: - if bsd_chflags: - lstatobj = os.lstat(obj) - if lstatobj.st_flags != 0: - bsd_chflags.lchflags(obj, 0) - parent_name = os.path.dirname(obj) - # Use normal stat/chflags for the parent since we want to - # follow any symlinks to the real parent directory. - pflags = os.stat(parent_name).st_flags - if pflags != 0: - bsd_chflags.chflags(parent_name, 0) - try: - os.rmdir(obj) - finally: - if bsd_chflags and pflags != 0: - # Restore the parent flags we saved before unlinking - bsd_chflags.chflags(parent_name, pflags) - show_unmerge("<<<", "", "dir", obj) - except EnvironmentError as e: - if e.errno not in ignored_rmdir_errnos: - raise - if e.errno != errno.ENOENT: - show_unmerge("---", unmerge_desc["!empty"], "dir", obj) - del e - else: - # When a directory is successfully removed, there's - # no need to protect symlinks that point to it. - unmerge_syms = protected_symlinks.pop(inode_key, None) - if unmerge_syms is not None: - for relative_path in unmerge_syms: - obj = os.path.join(real_root, - relative_path.lstrip(os.sep)) - try: - unlink(obj, os.lstat(obj)) - show_unmerge("<<<", "", "sym", obj) - except (OSError, IOError) as e: - if e.errno not in ignored_unlink_errnos: - raise - del e - show_unmerge("!!!", "", "sym", obj) + if protected_symlinks: + self._unmerge_protected_symlinks(others_in_slot, infodirs_inodes, + protected_symlinks, unmerge_desc, unlink, os) if protected_symlinks: msg = "One or more symlinks to directories have been " + \ @@ -2313,6 +2380,168 @@ class dblink(object): #remove self from vartree database so that our own virtual gets zapped if we're the last node self.vartree.zap(self.mycpv) + def _unmerge_protected_symlinks(self, others_in_slot, infodirs_inodes, + protected_symlinks, unmerge_desc, unlink, os): + + real_root = self.settings['ROOT'] + show_unmerge = self._show_unmerge + ignored_unlink_errnos = self._ignored_unlink_errnos + + flat_list = set() + flat_list.update(*protected_symlinks.values()) + flat_list = sorted(flat_list) + + for f in flat_list: + for dblnk in others_in_slot: + if dblnk.isowner(f): + # If another package in the same slot installed + # a file via a protected symlink, return early + # and don't bother searching for any other owners. + return + + msg = [] + msg.append("") + msg.append(_("Directory symlink(s) may need protection:")) + msg.append("") + + for f in flat_list: + msg.append("\t%s" % \ + os.path.join(real_root, f.lstrip(os.path.sep))) + + msg.append("") + msg.append(_("Searching all installed" + " packages for files installed via above symlink(s)...")) + msg.append("") + self._elog("elog", "postrm", msg) + + self.lockdb() + try: + owners = self.vartree.dbapi._owners.get_owners(flat_list) + self.vartree.dbapi.flush_cache() + finally: + self.unlockdb() + + for owner in list(owners): + if owner.mycpv == self.mycpv: + owners.pop(owner, None) + + if not owners: + msg = [] + msg.append(_("The above directory symlink(s) are all " + "safe to remove. Removing them now...")) + msg.append("") + self._elog("elog", "postrm", msg) + dirs = set() + for unmerge_syms in protected_symlinks.values(): + for relative_path in unmerge_syms: + obj = os.path.join(real_root, + relative_path.lstrip(os.sep)) + parent = os.path.dirname(obj) + while len(parent) > len(self._eroot): + try: + lstatobj = os.lstat(parent) + except OSError: + break + else: + dirs.add((parent, + (lstatobj.st_dev, lstatobj.st_ino))) + parent = os.path.dirname(parent) + try: + unlink(obj, os.lstat(obj)) + show_unmerge("<<<", "", "sym", obj) + except (OSError, IOError) as e: + if e.errno not in ignored_unlink_errnos: + raise + del e + show_unmerge("!!!", "", "sym", obj) + + protected_symlinks.clear() + self._unmerge_dirs(dirs, infodirs_inodes, + protected_symlinks, unmerge_desc, unlink, os) + dirs.clear() + + def _unmerge_dirs(self, dirs, infodirs_inodes, + protected_symlinks, unmerge_desc, unlink, os): + + show_unmerge = self._show_unmerge + infodir_cleanup = self._infodir_cleanup + ignored_unlink_errnos = self._ignored_unlink_errnos + ignored_rmdir_errnos = self._ignored_rmdir_errnos + real_root = self.settings['ROOT'] + + dirs = sorted(dirs) + dirs.reverse() + + for obj, inode_key in dirs: + # Treat any directory named "info" as a candidate here, + # since it might have been in INFOPATH previously even + # though it may not be there now. + if inode_key in infodirs_inodes or \ + os.path.basename(obj) == "info": + try: + remaining = os.listdir(obj) + except OSError: + pass + else: + cleanup_info_dir = () + if remaining and \ + len(remaining) <= len(infodir_cleanup): + if not set(remaining).difference(infodir_cleanup): + cleanup_info_dir = remaining + + for child in cleanup_info_dir: + child = os.path.join(obj, child) + try: + lstatobj = os.lstat(child) + if stat.S_ISREG(lstatobj.st_mode): + unlink(child, lstatobj) + show_unmerge("<<<", "", "obj", child) + except EnvironmentError as e: + if e.errno not in ignored_unlink_errnos: + raise + del e + show_unmerge("!!!", "", "obj", child) + try: + if bsd_chflags: + lstatobj = os.lstat(obj) + if lstatobj.st_flags != 0: + bsd_chflags.lchflags(obj, 0) + parent_name = os.path.dirname(obj) + # Use normal stat/chflags for the parent since we want to + # follow any symlinks to the real parent directory. + pflags = os.stat(parent_name).st_flags + if pflags != 0: + bsd_chflags.chflags(parent_name, 0) + try: + os.rmdir(obj) + finally: + if bsd_chflags and pflags != 0: + # Restore the parent flags we saved before unlinking + bsd_chflags.chflags(parent_name, pflags) + show_unmerge("<<<", "", "dir", obj) + except EnvironmentError as e: + if e.errno not in ignored_rmdir_errnos: + raise + if e.errno != errno.ENOENT: + show_unmerge("---", unmerge_desc["!empty"], "dir", obj) + del e + else: + # When a directory is successfully removed, there's + # no need to protect symlinks that point to it. + unmerge_syms = protected_symlinks.pop(inode_key, None) + if unmerge_syms is not None: + for relative_path in unmerge_syms: + obj = os.path.join(real_root, + relative_path.lstrip(os.sep)) + try: + unlink(obj, os.lstat(obj)) + show_unmerge("<<<", "", "sym", obj) + except (OSError, IOError) as e: + if e.errno not in ignored_unlink_errnos: + raise + del e + show_unmerge("!!!", "", "sym", obj) + def isowner(self, filename, destroot=None): """ Check if a file belongs to this package. This may @@ -2328,7 +2557,7 @@ class dblink(object): @param destroot: @type destroot: @rtype: Boolean - @returns: + @return: 1. True if this package owns the file. 2. False if this package does not own the file. """ @@ -2857,9 +3086,13 @@ class dblink(object): os = _os_merge - collision_ignore = set([normalize_path(myignore) for myignore in \ - portage.util.shlex_split( - self.settings.get("COLLISION_IGNORE", ""))]) + collision_ignore = [] + for x in portage.util.shlex_split( + self.settings.get("COLLISION_IGNORE", "")): + if os.path.isdir(os.path.join(self._eroot, x.lstrip(os.sep))): + x = normalize_path(x) + x += "/*" + collision_ignore.append(x) # For collisions with preserved libraries, the current package # will assume ownership and the libraries will be unregistered. @@ -2960,15 +3193,12 @@ class dblink(object): if not isowned and self.isprotected(full_path): isowned = True if not isowned: + f_match = full_path[len(self._eroot)-1:] stopmerge = True - if collision_ignore: - if f in collision_ignore: + for pattern in collision_ignore: + if fnmatch.fnmatch(f_match, pattern): stopmerge = False - else: - for myignore in collision_ignore: - if f.startswith(myignore + os.path.sep): - stopmerge = False - break + break if stopmerge: collisions.append(f) return collisions, symlink_collisions, plib_collisions @@ -3121,9 +3351,10 @@ class dblink(object): if isinstance(lines, basestring): lines = [lines] for line in lines: - fields = (funcname, phase, cpv, line.rstrip('\n')) - str_buffer.append(' '.join(fields)) - str_buffer.append('\n') + for line in line.split('\n'): + fields = (funcname, phase, cpv, line) + str_buffer.append(' '.join(fields)) + str_buffer.append('\n') if str_buffer: os.write(self._pipe, _unicode_encode(''.join(str_buffer))) @@ -3157,7 +3388,7 @@ class dblink(object): @param prev_mtimes: { Filename:mtime } mapping for env_update @type prev_mtimes: Dictionary @rtype: Boolean - @returns: + @return: 1. 0 on success 2. 1 on failure @@ -3192,17 +3423,22 @@ class dblink(object): pass continue + f = None try: - val = io.open(_unicode_encode( + f = io.open(_unicode_encode( os.path.join(inforoot, var_name), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace').readline().strip() + errors='replace') + val = f.readline().strip() except EnvironmentError as e: if e.errno != errno.ENOENT: raise del e val = '' + finally: + if f is not None: + f.close() if var_name == 'SLOT': slot = val @@ -3226,10 +3462,6 @@ class dblink(object): if not os.path.exists(self.dbcatdir): ensure_dirs(self.dbcatdir) - otherversions = [] - for v in self.vartree.dbapi.cp_list(self.mysplit[0]): - otherversions.append(v.split("/")[1]) - cp = self.mysplit[0] slot_atom = "%s:%s" % (cp, slot) @@ -3270,22 +3502,49 @@ class dblink(object): max_dblnk = dblnk self._installed_instance = max_dblnk + if self.settings.get("INSTALL_MASK") or \ + "nodoc" in self.settings.features or \ + "noinfo" in self.settings.features or \ + "noman" in self.settings.features: + # Apply INSTALL_MASK before collision-protect, since it may + # be useful to avoid collisions in some scenarios. + phase = MiscFunctionsProcess(background=False, + commands=["preinst_mask"], phase="preinst", + scheduler=self._scheduler, settings=self.settings) + phase.start() + phase.wait() + # We check for unicode encoding issues after src_install. However, # the check must be repeated here for binary packages (it's # inexpensive since we call os.walk() here anyway). unicode_errors = [] + line_ending_re = re.compile('[\n\r]') + srcroot_len = len(srcroot) + ed_len = len(self.settings["ED"]) while True: unicode_error = False + eagain_error = False myfilelist = [] mylinklist = [] paths_with_newlines = [] - srcroot_len = len(srcroot) def onerror(e): raise - for parent, dirs, files in os.walk(srcroot, onerror=onerror): + walk_iter = os.walk(srcroot, onerror=onerror) + while True: + try: + parent, dirs, files = next(walk_iter) + except StopIteration: + break + except OSError as e: + if e.errno != errno.EAGAIN: + raise + # Observed with PyPy 1.8. + eagain_error = True + break + try: parent = _unicode_decode(parent, encoding=_encodings['merge'], errors='strict') @@ -3293,12 +3552,12 @@ class dblink(object): new_parent = _unicode_decode(parent, encoding=_encodings['merge'], errors='replace') new_parent = _unicode_encode(new_parent, - encoding=_encodings['merge'], errors='backslashreplace') + encoding='ascii', errors='backslashreplace') new_parent = _unicode_decode(new_parent, encoding=_encodings['merge'], errors='replace') os.rename(parent, new_parent) unicode_error = True - unicode_errors.append(new_parent[srcroot_len:]) + unicode_errors.append(new_parent[ed_len:]) break for fname in files: @@ -3311,13 +3570,13 @@ class dblink(object): new_fname = _unicode_decode(fname, encoding=_encodings['merge'], errors='replace') new_fname = _unicode_encode(new_fname, - encoding=_encodings['merge'], errors='backslashreplace') + encoding='ascii', errors='backslashreplace') new_fname = _unicode_decode(new_fname, encoding=_encodings['merge'], errors='replace') new_fpath = os.path.join(parent, new_fname) os.rename(fpath, new_fpath) unicode_error = True - unicode_errors.append(new_fpath[srcroot_len:]) + unicode_errors.append(new_fpath[ed_len:]) fname = new_fname fpath = new_fpath else: @@ -3325,7 +3584,7 @@ class dblink(object): relative_path = fpath[srcroot_len:] - if "\n" in relative_path: + if line_ending_re.search(relative_path) is not None: paths_with_newlines.append(relative_path) file_mode = os.lstat(fpath).st_mode @@ -3340,19 +3599,20 @@ class dblink(object): if unicode_error: break - if not unicode_error: + if not (unicode_error or eagain_error): break if unicode_errors: - eerror(portage._merge_unicode_error(unicode_errors)) + self._elog("eqawarn", "preinst", + _merge_unicode_error(unicode_errors)) if paths_with_newlines: msg = [] - msg.append(_("This package installs one or more files containing a newline (\\n) character:")) + msg.append(_("This package installs one or more files containing line ending characters:")) msg.append("") paths_with_newlines.sort() for f in paths_with_newlines: - msg.append("\t/%s" % (f.replace("\n", "\\n"))) + msg.append("\t/%s" % (f.replace("\n", "\\n").replace("\r", "\\r"))) msg.append("") msg.append(_("package %s NOT merged") % self.mycpv) msg.append("") @@ -3394,14 +3654,6 @@ class dblink(object): if installed_files: return 1 - # check for package collisions - blockers = self._blockers - if blockers is None: - blockers = [] - collisions, symlink_collisions, plib_collisions = \ - self._collision_protect(srcroot, destroot, - others_in_slot + blockers, myfilelist, mylinklist) - # Make sure the ebuild environment is initialized and that ${T}/elog # exists for logging of collision-protect eerror messages. if myebuild is None: @@ -3413,6 +3665,29 @@ class dblink(object): for other in others_in_slot]) prepare_build_dirs(settings=self.settings, cleanup=cleanup) + # check for package collisions + blockers = self._blockers + if blockers is None: + blockers = [] + collisions, symlink_collisions, plib_collisions = \ + self._collision_protect(srcroot, destroot, + others_in_slot + blockers, myfilelist, mylinklist) + + if symlink_collisions: + # Symlink collisions need to be distinguished from other types + # of collisions, in order to avoid confusion (see bug #409359). + msg = _("Package '%s' has one or more collisions " + "between symlinks and directories, which is explicitly " + "forbidden by PMS section 13.4 (see bug #326685):") % \ + (self.settings.mycpv,) + msg = textwrap.wrap(msg, 70) + msg.append("") + for f in symlink_collisions: + msg.append("\t%s" % os.path.join(destroot, + f.lstrip(os.path.sep))) + msg.append("") + self._elog("eerror", "preinst", msg) + if collisions: collision_protect = "collision-protect" in self.settings.features protect_owned = "protect-owned" in self.settings.features @@ -3494,12 +3769,20 @@ class dblink(object): eerror([_("None of the installed" " packages claim the file(s)."), ""]) + symlink_abort_msg =_("Package '%s' NOT merged since it has " + "one or more collisions between symlinks and directories, " + "which is explicitly forbidden by PMS section 13.4 " + "(see bug #326685).") + # The explanation about the collision and how to solve # it may not be visible via a scrollback buffer, especially # if the number of file collisions is large. Therefore, # show a summary at the end. abort = False - if collision_protect: + if symlink_collisions: + abort = True + msg = symlink_abort_msg % (self.settings.mycpv,) + elif collision_protect: abort = True msg = _("Package '%s' NOT merged due to file collisions.") % \ self.settings.mycpv @@ -3507,12 +3790,6 @@ class dblink(object): abort = True msg = _("Package '%s' NOT merged due to file collisions.") % \ self.settings.mycpv - elif symlink_collisions: - abort = True - msg = _("Package '%s' NOT merged due to collision " + \ - "between a symlink and a directory which is explicitly " + \ - "forbidden by PMS (see bug #326685).") % \ - (self.settings.mycpv,) else: msg = _("Package '%s' merged despite file collisions.") % \ self.settings.mycpv @@ -3558,10 +3835,12 @@ class dblink(object): # write local package counter for recording if counter is None: counter = self.vartree.dbapi.counter_tick(mycpv=self.mycpv) - io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), + f = io.open(_unicode_encode(os.path.join(self.dbtmpdir, 'COUNTER'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='backslashreplace').write(_unicode_decode(str(counter))) + errors='backslashreplace') + f.write(_unicode_decode(str(counter))) + f.close() self.updateprotect() @@ -3577,9 +3856,8 @@ class dblink(object): # Always behave like --noconfmem is enabled for downgrades # so that people who don't know about this option are less # likely to get confused when doing upgrade/downgrade cycles. - pv_split = catpkgsplit(self.mycpv)[1:] for other in others_in_slot: - if pkgcmp(pv_split, catpkgsplit(other.mycpv)[1:]) < 0: + if vercmp(self.mycpv.version, other.mycpv.version) < 0: cfgfiledict["IGNORE"] = 1 break @@ -3798,22 +4076,11 @@ class dblink(object): showMessage(_("!!! FAILED postinst: ")+str(a)+"\n", level=logging.ERROR, noiselevel=-1) - downgrade = False - for v in otherversions: - if pkgcmp(catpkgsplit(self.pkg)[1:], catpkgsplit(v)[1:]) < 0: - downgrade = True - - # Lock the config memory file to prevent symlink creation - # in merge_contents from overlapping with env-update. - self.vartree.dbapi._fs_lock() - try: - #update environment settings, library paths. DO NOT change symlinks. - env_update(makelinks=(not downgrade), - target_root=self.settings['ROOT'], prev_mtimes=prev_mtimes, - contents=contents, env=self.settings.environ(), - writemsg_level=self._display_merge) - finally: - self.vartree.dbapi._fs_unlock() + #update environment settings, library paths. DO NOT change symlinks. + env_update( + target_root=self.settings['ROOT'], prev_mtimes=prev_mtimes, + contents=contents, env=self.settings, + writemsg_level=self._display_merge, vardbapi=self.vartree.dbapi) # For gcc upgrades, preserved libs have to be removed after the # the library path has been updated. @@ -3867,7 +4134,8 @@ class dblink(object): # we do a first merge; this will recurse through all files in our srcroot but also build up a # "second hand" of symlinks to merge later - if self.mergeme(srcroot, destroot, outfile, secondhand, "", cfgfiledict, mymtime): + if self.mergeme(srcroot, destroot, outfile, secondhand, + self.settings["EPREFIX"].lstrip(os.sep), cfgfiledict, mymtime): return 1 # now, it's time for dealing our second hand; we'll loop until we can't merge anymore. The rest are @@ -3936,7 +4204,7 @@ class dblink(object): @param thismtime: The current time (typically long(time.time()) @type thismtime: Long @rtype: None or Boolean - @returns: + @return: 1. True on failure 2. None otherwise @@ -3952,6 +4220,10 @@ class dblink(object): destroot = normalize_path(destroot).rstrip(sep) + sep calc_prelink = "prelink-checksums" in self.settings.features + protect_if_modified = \ + "config-protect-if-modified" in self.settings.features and \ + self._installed_instance is not None + # this is supposed to merge a list of files. There will be 2 forms of argument passing. if isinstance(stufftomerge, basestring): #A directory is specified. Figure out protection paths, listdir() it and process it. @@ -3985,14 +4257,37 @@ class dblink(object): if stat.S_ISLNK(mymode): # we are merging a symbolic link - myabsto = abssymlink(mysrc) + # The file name of mysrc and the actual file that it points to + # will have earlier been forcefully converted to the 'merge' + # encoding if necessary, but the content of the symbolic link + # may need to be forcefully converted here. + myto = _os.readlink(_unicode_encode(mysrc, + encoding=_encodings['merge'], errors='strict')) + try: + myto = _unicode_decode(myto, + encoding=_encodings['merge'], errors='strict') + except UnicodeDecodeError: + myto = _unicode_decode(myto, encoding=_encodings['merge'], + errors='replace') + myto = _unicode_encode(myto, encoding='ascii', + errors='backslashreplace') + myto = _unicode_decode(myto, encoding=_encodings['merge'], + errors='replace') + os.unlink(mysrc) + os.symlink(myto, mysrc) + + # Pass in the symlink target in order to bypass the + # os.readlink() call inside abssymlink(), since that + # call is unsafe if the merge encoding is not ascii + # or utf_8 (see bug #382021). + myabsto = abssymlink(mysrc, target=myto) + if myabsto.startswith(srcroot): myabsto = myabsto[len(srcroot):] myabsto = myabsto.lstrip(sep) - myto = os.readlink(mysrc) if self.settings and self.settings["D"]: if myto.startswith(self.settings["D"]): - myto = myto[len(self.settings["D"]):] + myto = myto[len(self.settings["D"])-1:] # myrealto contains the path of the real file to which this symlink points. # we can simply test for existence of this file to see if the target has been merged yet myrealto = normalize_path(os.path.join(destroot, myabsto)) @@ -4170,9 +4465,18 @@ class dblink(object): # now, config file management may come into play. # we only need to tweak mydest if cfg file management is in play. if protected: + destmd5 = perform_md5(mydest, calc_prelink=calc_prelink) + if protect_if_modified: + contents_key = \ + self._installed_instance._match_contents(myrealdest) + if contents_key: + inst_info = self._installed_instance.getcontents()[contents_key] + if inst_info[0] == "obj" and inst_info[2] == destmd5: + protected = False + + if protected: # we have a protection path; enable config file management. cfgprot = 0 - destmd5 = perform_md5(mydest, calc_prelink=calc_prelink) if mymd5 == destmd5: #file already in place; simply update mtimes of destination moveme = 1 @@ -4207,10 +4511,10 @@ class dblink(object): # as hardlinks (having identical st_dev and st_ino). hardlink_key = (mystat.st_dev, mystat.st_ino) - hardlink_candidates = self._md5_merge_map.get(hardlink_key) + hardlink_candidates = self._hardlink_merge_map.get(hardlink_key) if hardlink_candidates is None: hardlink_candidates = [] - self._md5_merge_map[hardlink_key] = hardlink_candidates + self._hardlink_merge_map[hardlink_key] = hardlink_candidates mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings, @@ -4218,8 +4522,7 @@ class dblink(object): encoding=_encodings['merge']) if mymtime is None: return 1 - if hardlink_candidates is not None: - hardlink_candidates.append(mydest) + hardlink_candidates.append(mydest) zing = ">>>" if mymtime != None: @@ -4445,6 +4748,7 @@ def write_contents(contents, root, f): def tar_contents(contents, root, tar, protect=None, onProgress=None): os = _os_merge + encoding = _encodings['merge'] try: for x in contents: @@ -4464,7 +4768,9 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): pass else: os = portage.os + encoding = _encodings['fs'] + tar.encoding = encoding root = normalize_path(root).rstrip(os.path.sep) + os.path.sep id_strings = {} maxval = len(contents) @@ -4486,7 +4792,7 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): continue contents_type = contents[path][0] if path.startswith(root): - arcname = path[len(root):] + arcname = "./" + path[len(root):] else: raise ValueError("invalid root argument: '%s'" % root) live_path = path @@ -4498,7 +4804,51 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): # recorded as a real directory in the tar file to ensure that tar # can properly extract it's children. live_path = os.path.realpath(live_path) - tarinfo = tar.gettarinfo(live_path, arcname) + lst = os.lstat(live_path) + + # Since os.lstat() inside TarFile.gettarinfo() can trigger a + # UnicodeEncodeError when python has something other than utf_8 + # return from sys.getfilesystemencoding() (as in bug #388773), + # we implement the needed functionality here, using the result + # of our successful lstat call. An alternative to this would be + # to pass in the fileobj argument to TarFile.gettarinfo(), so + # that it could use fstat instead of lstat. However, that would + # have the unwanted effect of dereferencing symlinks. + + tarinfo = tar.tarinfo() + tarinfo.name = arcname + tarinfo.mode = lst.st_mode + tarinfo.uid = lst.st_uid + tarinfo.gid = lst.st_gid + tarinfo.size = 0 + tarinfo.mtime = lst.st_mtime + tarinfo.linkname = "" + if stat.S_ISREG(lst.st_mode): + inode = (lst.st_ino, lst.st_dev) + if (lst.st_nlink > 1 and + inode in tar.inodes and + arcname != tar.inodes[inode]): + tarinfo.type = tarfile.LNKTYPE + tarinfo.linkname = tar.inodes[inode] + else: + tar.inodes[inode] = arcname + tarinfo.type = tarfile.REGTYPE + tarinfo.size = lst.st_size + elif stat.S_ISDIR(lst.st_mode): + tarinfo.type = tarfile.DIRTYPE + elif stat.S_ISLNK(lst.st_mode): + tarinfo.type = tarfile.SYMTYPE + tarinfo.linkname = os.readlink(live_path) + else: + continue + try: + tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0] + except KeyError: + pass + try: + tarinfo.gname = grp.getgrgid(tarinfo.gid)[0] + except KeyError: + pass if stat.S_ISREG(lst.st_mode): if protect and protect(path): @@ -4515,7 +4865,7 @@ def tar_contents(contents, root, tar, protect=None, onProgress=None): f.close() else: f = open(_unicode_encode(path, - encoding=object.__getattribute__(os, '_encoding'), + encoding=encoding, errors='strict'), 'rb') try: tar.addfile(tarinfo, f) diff --git a/portage_with_autodep/pym/portage/dbapi/vartree.pyo b/portage_with_autodep/pym/portage/dbapi/vartree.pyo Binary files differnew file mode 100644 index 0000000..7c186cf --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/vartree.pyo diff --git a/portage_with_autodep/pym/portage/dbapi/virtual.py b/portage_with_autodep/pym/portage/dbapi/virtual.py index ec97ffe..da15983 100644 --- a/portage_with_autodep/pym/portage/dbapi/virtual.py +++ b/portage_with_autodep/pym/portage/dbapi/virtual.py @@ -1,9 +1,10 @@ -# Copyright 1998-2007 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.dbapi import dbapi -from portage import cpv_getkey +from portage.dbapi.dep_expand import dep_expand +from portage.versions import cpv_getkey, _pkg_str class fakedbapi(dbapi): """A fake dbapi that allows consumers to inject/remove packages to/from it @@ -31,27 +32,30 @@ class fakedbapi(dbapi): self._match_cache = {} def match(self, origdep, use_cache=1): - result = self._match_cache.get(origdep, None) + atom = dep_expand(origdep, mydb=self, settings=self.settings) + cache_key = (atom, atom.unevaluated_atom) + result = self._match_cache.get(cache_key) if result is not None: return result[:] - result = dbapi.match(self, origdep, use_cache=use_cache) - self._match_cache[origdep] = result + result = list(self._iter_match(atom, self.cp_list(atom.cp))) + self._match_cache[cache_key] = result return result[:] def cpv_exists(self, mycpv, myrepo=None): return mycpv in self.cpvdict def cp_list(self, mycp, use_cache=1, myrepo=None): - cachelist = self._match_cache.get(mycp) - # cp_list() doesn't expand old-style virtuals - if cachelist and cachelist[0].startswith(mycp): + # NOTE: Cache can be safely shared with the match cache, since the + # match cache uses the result from dep_expand for the cache_key. + cache_key = (mycp, mycp) + cachelist = self._match_cache.get(cache_key) + if cachelist is not None: return cachelist[:] cpv_list = self.cpdict.get(mycp) if cpv_list is None: cpv_list = [] self._cpv_sort_ascending(cpv_list) - if not (not cpv_list and mycp.startswith("virtual/")): - self._match_cache[mycp] = cpv_list + self._match_cache[cache_key] = cpv_list return cpv_list[:] def cp_all(self): @@ -70,7 +74,13 @@ class fakedbapi(dbapi): @param metadata: dict """ self._clear_cache() - mycp = cpv_getkey(mycpv) + if not hasattr(mycpv, 'cp'): + if metadata is None: + mycpv = _pkg_str(mycpv) + else: + mycpv = _pkg_str(mycpv, slot=metadata.get('SLOT'), + repo=metadata.get('repository')) + mycp = mycpv.cp self.cpvdict[mycpv] = metadata myslot = None if self._exclusive_slots and metadata: diff --git a/portage_with_autodep/pym/portage/dbapi/virtual.pyo b/portage_with_autodep/pym/portage/dbapi/virtual.pyo Binary files differnew file mode 100644 index 0000000..9f7c667 --- /dev/null +++ b/portage_with_autodep/pym/portage/dbapi/virtual.pyo diff --git a/portage_with_autodep/pym/portage/debug.py b/portage_with_autodep/pym/portage/debug.py index ce642fe..ebf1a13 100644 --- a/portage_with_autodep/pym/portage/debug.py +++ b/portage_with_autodep/pym/portage/debug.py @@ -1,4 +1,4 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import os @@ -26,7 +26,7 @@ class trace_handler(object): def __init__(self): python_system_paths = [] for x in sys.path: - if os.path.basename(x).startswith("python2."): + if os.path.basename(x) == "python%s.%s" % sys.version_info[:2]: python_system_paths.append(x) self.ignore_prefixes = [] diff --git a/portage_with_autodep/pym/portage/debug.pyo b/portage_with_autodep/pym/portage/debug.pyo Binary files differnew file mode 100644 index 0000000..82a5e8f --- /dev/null +++ b/portage_with_autodep/pym/portage/debug.pyo diff --git a/portage_with_autodep/pym/portage/dep/__init__.py b/portage_with_autodep/pym/portage/dep/__init__.py index fd5ad30..152af0a 100644 --- a/portage_with_autodep/pym/portage/dep/__init__.py +++ b/portage_with_autodep/pym/portage/dep/__init__.py @@ -1,5 +1,5 @@ # deps.py -- Portage dependency resolution functions -# Copyright 2003-2011 Gentoo Foundation +# Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -30,13 +30,20 @@ __all__ = [ import re, sys import warnings from itertools import chain + +import portage +portage.proxy.lazyimport.lazyimport(globals(), + 'portage.util:cmp_sort_key,writemsg', +) + from portage import _unicode_decode from portage.eapi import eapi_has_slot_deps, eapi_has_src_uri_arrows, \ - eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_use_dep_defaults + eapi_has_use_deps, eapi_has_strong_blocks, eapi_has_use_dep_defaults, \ + eapi_has_repo_deps, eapi_allows_dots_in_PN, eapi_allows_dots_in_use_flags from portage.exception import InvalidAtom, InvalidData, InvalidDependString from portage.localization import _ from portage.versions import catpkgsplit, catsplit, \ - pkgcmp, ververify, _cp, _cpv + vercmp, ververify, _cp, _cpv, _pkg_str, _unknown_repo import portage.cache.mappings if sys.hexversion >= 0x3000000: @@ -55,7 +62,7 @@ def cpvequal(cpv1, cpv2): @param cpv2: CategoryPackageVersion (no operators) Example: "sys-apps/portage-2.1" @type cpv2: String @rtype: Boolean - @returns: + @return: 1. True if cpv1 = cpv2 2. False Otherwise 3. Throws PortageException if cpv1 or cpv2 is not a CPV @@ -67,16 +74,27 @@ def cpvequal(cpv1, cpv2): """ - split1 = catpkgsplit(cpv1) - split2 = catpkgsplit(cpv2) - - if not split1 or not split2: + try: + try: + split1 = cpv1.cpv_split + except AttributeError: + cpv1 = _pkg_str(cpv1) + split1 = cpv1.cpv_split + + try: + split2 = cpv2.cpv_split + except AttributeError: + cpv2 = _pkg_str(cpv2) + split2 = cpv2.cpv_split + + except InvalidData: raise portage.exception.PortageException(_("Invalid data '%s, %s', parameter was not a CPV") % (cpv1, cpv2)) - - if split1[0] != split2[0]: + + if split1[0] != split2[0] or \ + split1[1] != split2[1]: return False - - return (pkgcmp(split1[1:], split2[1:]) == 0) + + return vercmp(cpv1.version, cpv2.version) == 0 def strip_empty(myarr): """ @@ -635,8 +653,8 @@ def flatten(mylist): _usedep_re = { - "0": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), - "4-python": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@.-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), + "dots_disallowed_in_use_flags": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), + "dots_allowed_in_use_flags": re.compile("^(?P<prefix>[!-]?)(?P<flag>[A-Za-z0-9][A-Za-z0-9+_@.-]*)(?P<default>(\(\+\)|\(\-\))?)(?P<suffix>[?=]?)$"), } def _get_usedep_re(eapi): @@ -649,10 +667,10 @@ def _get_usedep_re(eapi): @return: A regular expression object that matches valid USE deps for the given eapi. """ - if eapi in (None, "4-python",): - return _usedep_re["4-python"] + if eapi is None or eapi_allows_dots_in_use_flags(eapi): + return _usedep_re["dots_allowed_in_use_flags"] else: - return _usedep_re["0"] + return _usedep_re["dots_disallowed_in_use_flags"] class _use_dep(object): @@ -1068,6 +1086,10 @@ class Atom(_atom_base): _atom_base.__init__(s) + atom_re = _get_atom_re(eapi) + if eapi_has_repo_deps(eapi): + allow_repo = True + if "!" == s[:1]: blocker = self._blocker(forbid_overlap=("!" == s[1:2])) if blocker.overlap.forbid: @@ -1077,11 +1099,11 @@ class Atom(_atom_base): else: blocker = False self.__dict__['blocker'] = blocker - m = _atom_re.match(s) + m = atom_re.match(s) extended_syntax = False if m is None: if allow_wildcard: - m = _atom_wildcard_re.match(s) + m = _get_atom_wildcard_re(eapi).match(s) if m is None: raise InvalidAtom(self) op = None @@ -1096,38 +1118,44 @@ class Atom(_atom_base): else: raise InvalidAtom(self) elif m.group('op') is not None: - base = _atom_re.groupindex['op'] + base = atom_re.groupindex['op'] op = m.group(base + 1) cpv = m.group(base + 2) cp = m.group(base + 3) - slot = m.group(_atom_re.groups - 2) - repo = m.group(_atom_re.groups - 1) - use_str = m.group(_atom_re.groups) + slot = m.group(atom_re.groups - 2) + repo = m.group(atom_re.groups - 1) + use_str = m.group(atom_re.groups) if m.group(base + 4) is not None: raise InvalidAtom(self) elif m.group('star') is not None: - base = _atom_re.groupindex['star'] + base = atom_re.groupindex['star'] op = '=*' cpv = m.group(base + 1) cp = m.group(base + 2) - slot = m.group(_atom_re.groups - 2) - repo = m.group(_atom_re.groups - 1) - use_str = m.group(_atom_re.groups) + slot = m.group(atom_re.groups - 2) + repo = m.group(atom_re.groups - 1) + use_str = m.group(atom_re.groups) if m.group(base + 3) is not None: raise InvalidAtom(self) elif m.group('simple') is not None: op = None - cpv = cp = m.group(_atom_re.groupindex['simple'] + 1) - slot = m.group(_atom_re.groups - 2) - repo = m.group(_atom_re.groups - 1) - use_str = m.group(_atom_re.groups) - if m.group(_atom_re.groupindex['simple'] + 2) is not None: + cpv = cp = m.group(atom_re.groupindex['simple'] + 1) + slot = m.group(atom_re.groups - 2) + repo = m.group(atom_re.groups - 1) + use_str = m.group(atom_re.groups) + if m.group(atom_re.groupindex['simple'] + 2) is not None: raise InvalidAtom(self) else: raise AssertionError(_("required group not found in atom: '%s'") % self) self.__dict__['cp'] = cp - self.__dict__['cpv'] = cpv + try: + self.__dict__['cpv'] = _pkg_str(cpv) + self.__dict__['version'] = self.cpv.version + except InvalidData: + # plain cp, wildcard, or something + self.__dict__['cpv'] = cpv + self.__dict__['version'] = None self.__dict__['repo'] = repo self.__dict__['slot'] = slot self.__dict__['operator'] = op @@ -1216,6 +1244,23 @@ class Atom(_atom_base): return Atom(self.replace(_slot_separator + self.slot, '', 1), allow_repo=True, allow_wildcard=True) + def with_repo(self, repo): + atom = remove_slot(self) + if self.slot is not None: + atom += _slot_separator + self.slot + atom += _repo_separator + repo + if self.use is not None: + atom += str(self.use) + return Atom(atom, allow_repo=True, allow_wildcard=True) + + def with_slot(self, slot): + atom = remove_slot(self) + _slot_separator + slot + if self.repo is not None: + atom += _repo_separator + self.repo + if self.use is not None: + atom += str(self.use) + return Atom(atom, allow_repo=True, allow_wildcard=True) + def __setattr__(self, name, value): raise AttributeError("Atom instances are immutable", self.__class__, name, value) @@ -1353,10 +1398,13 @@ class ExtendedAtomDict(portage.cache.mappings.MutableMapping): yield k def iteritems(self): - for item in self._normal.items(): - yield item - for item in self._extended.items(): - yield item + try: + for item in self._normal.items(): + yield item + for item in self._extended.items(): + yield item + except AttributeError: + pass # FEATURES=python-trace def __delitem__(self, cp): if "*" in cp: @@ -1610,20 +1658,45 @@ _repo_separator = "::" _repo_name = r'[\w][\w-]*' _repo = r'(?:' + _repo_separator + '(' + _repo_name + ')' + ')?' -_atom_re = re.compile('^(?P<without_use>(?:' + - '(?P<op>' + _op + _cpv + ')|' + - '(?P<star>=' + _cpv + r'\*)|' + - '(?P<simple>' + _cp + '))' + - '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE) +_atom_re = { + "dots_disallowed_in_PN": re.compile('^(?P<without_use>(?:' + + '(?P<op>' + _op + _cpv['dots_disallowed_in_PN'] + ')|' + + '(?P<star>=' + _cpv['dots_disallowed_in_PN'] + r'\*)|' + + '(?P<simple>' + _cp['dots_disallowed_in_PN'] + '))' + + '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE), + "dots_allowed_in_PN": re.compile('^(?P<without_use>(?:' + + '(?P<op>' + _op + _cpv['dots_allowed_in_PN'] + ')|' + + '(?P<star>=' + _cpv['dots_allowed_in_PN'] + r'\*)|' + + '(?P<simple>' + _cp['dots_allowed_in_PN'] + '))' + + '(' + _slot_separator + _slot + ')?' + _repo + ')(' + _use + ')?$', re.VERBOSE), +} + +def _get_atom_re(eapi): + if eapi is None or eapi_allows_dots_in_PN(eapi): + return _atom_re["dots_allowed_in_PN"] + else: + return _atom_re["dots_disallowed_in_PN"] _extended_cat = r'[\w+*][\w+.*-]*' -_extended_pkg = r'[\w+*][\w+*-]*?' +_extended_pkg = { + "dots_disallowed_in_PN": r'[\w+*][\w+*-]*?', + "dots_allowed_in_PN": r'[\w+*][\w+.*-]*?', +} -_atom_wildcard_re = re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$') +_atom_wildcard_re = { + "dots_disallowed_in_PN": re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg['dots_disallowed_in_PN'] + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$'), + "dots_allowed_in_PN": re.compile('(?P<simple>(' + _extended_cat + ')/(' + _extended_pkg['dots_allowed_in_PN'] + '))(:(?P<slot>' + _slot + '))?(' + _repo_separator + '(?P<repo>' + _repo_name + '))?$'), +} + +def _get_atom_wildcard_re(eapi): + if eapi is None or eapi_allows_dots_in_PN(eapi): + return _atom_wildcard_re["dots_allowed_in_PN"] + else: + return _atom_wildcard_re["dots_disallowed_in_PN"] _useflag_re = { - "0": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@-]*$'), - "4-python": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@.-]*$'), + "dots_disallowed_in_use_flags": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@-]*$'), + "dots_allowed_in_use_flags": re.compile(r'^[A-Za-z0-9][A-Za-z0-9+_@.-]*$'), } def _get_useflag_re(eapi): @@ -1636,10 +1709,10 @@ def _get_useflag_re(eapi): @return: A regular expression object that matches valid USE flags for the given eapi. """ - if eapi in (None, "4-python",): - return _useflag_re["4-python"] + if eapi is None or eapi_allows_dots_in_use_flags(eapi): + return _useflag_re["dots_allowed_in_use_flags"] else: - return _useflag_re["0"] + return _useflag_re["dots_disallowed_in_use_flags"] def isvalidatom(atom, allow_blockers=False, allow_wildcard=False, allow_repo=False): """ @@ -1753,7 +1826,14 @@ def match_to_list(mypkg, mylist): @rtype: List @return: A unique list of package atoms that match the given package atom """ - return [ x for x in set(mylist) if match_from_list(x, [mypkg]) ] + matches = set() + result = [] + pkgs = [mypkg] + for x in mylist: + if x not in matches and match_from_list(x, pkgs): + matches.add(x) + result.append(x) + return result def best_match_to_list(mypkg, mylist): """ @@ -1781,6 +1861,7 @@ def best_match_to_list(mypkg, mylist): '>':2, '<':2, '>=':2, '<=':2, None:1} maxvalue = -2 bestm = None + mypkg_cpv = None for x in match_to_list(mypkg, mylist): if x.extended_syntax: if dep_getslot(x) is not None: @@ -1800,6 +1881,31 @@ def best_match_to_list(mypkg, mylist): if op_val > maxvalue: maxvalue = op_val bestm = x + elif op_val == maxvalue and op_val == 2: + # For >, <, >=, and <=, the one with the version + # closest to mypkg is the best match. + if mypkg_cpv is None: + try: + mypkg_cpv = mypkg.cpv + except AttributeError: + mypkg_cpv = _pkg_str(remove_slot(mypkg)) + if bestm.cpv == mypkg_cpv or bestm.cpv == x.cpv: + pass + elif x.cpv == mypkg_cpv: + bestm = x + else: + # Sort the cpvs to find the one closest to mypkg_cpv + cpv_list = [bestm.cpv, mypkg_cpv, x.cpv] + def cmp_cpv(cpv1, cpv2): + return vercmp(cpv1.version, cpv2.version) + cpv_list.sort(key=cmp_sort_key(cmp_cpv)) + if cpv_list[0] is mypkg_cpv or cpv_list[-1] is mypkg_cpv: + if cpv_list[1] is x.cpv: + bestm = x + else: + # TODO: handle the case where mypkg_cpv is in the middle + pass + return bestm def match_from_list(mydep, candidate_list): @@ -1817,7 +1923,6 @@ def match_from_list(mydep, candidate_list): if not candidate_list: return [] - from portage.util import writemsg if "!" == mydep[:1]: if "!" == mydep[1:2]: mydep = mydep[2:] @@ -1882,7 +1987,7 @@ def match_from_list(mydep, candidate_list): myver = mysplit[2].lstrip("0") if not myver or not myver[0].isdigit(): myver = "0"+myver - mycpv = mysplit[0]+"/"+mysplit[1]+"-"+myver + mycpv_cmp = mysplit[0]+"/"+mysplit[1]+"-"+myver for x in candidate_list: xs = getattr(x, "cpv_split", None) if xs is None: @@ -1891,7 +1996,7 @@ def match_from_list(mydep, candidate_list): if not myver or not myver[0].isdigit(): myver = "0"+myver xcpv = xs[0]+"/"+xs[1]+"-"+myver - if xcpv.startswith(mycpv): + if xcpv.startswith(mycpv_cmp): mylist.append(x) elif operator == "~": # version, any revision, match @@ -1908,15 +2013,19 @@ def match_from_list(mydep, candidate_list): mylist.append(x) elif operator in [">", ">=", "<", "<="]: - mysplit = ["%s/%s" % (cat, pkg), ver, rev] for x in candidate_list: - xs = getattr(x, "cpv_split", None) - if xs is None: - xs = catpkgsplit(remove_slot(x)) - xcat, xpkg, xver, xrev = xs - xs = ["%s/%s" % (xcat, xpkg), xver, xrev] + if hasattr(x, 'cp'): + pkg = x + else: + try: + pkg = _pkg_str(remove_slot(x)) + except InvalidData: + continue + + if pkg.cp != mydep.cp: + continue try: - result = pkgcmp(xs, mysplit) + result = vercmp(pkg.version, mydep.version) except ValueError: # pkgcmp may return ValueError during int() conversion writemsg(_("\nInvalid package name: %s\n") % x, noiselevel=-1) raise @@ -1993,7 +2102,8 @@ def match_from_list(mydep, candidate_list): repo = getattr(x, "repo", False) if repo is False: repo = dep_getrepo(x) - if repo is not None and repo != mydep.repo: + if repo is not None and repo != _unknown_repo and \ + repo != mydep.repo: continue mylist.append(x) diff --git a/portage_with_autodep/pym/portage/dep/__init__.pyo b/portage_with_autodep/pym/portage/dep/__init__.pyo Binary files differnew file mode 100644 index 0000000..c78bb23 --- /dev/null +++ b/portage_with_autodep/pym/portage/dep/__init__.pyo diff --git a/portage_with_autodep/pym/portage/dep/_slot_operator.py b/portage_with_autodep/pym/portage/dep/_slot_operator.py new file mode 100644 index 0000000..7b64444 --- /dev/null +++ b/portage_with_autodep/pym/portage/dep/_slot_operator.py @@ -0,0 +1,97 @@ +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import unicode_literals + +from portage.dep import Atom, paren_enclose, use_reduce +from portage.eapi import _get_eapi_attrs +from portage.exception import InvalidData +from _emerge.Package import Package + +def find_built_slot_operator_atoms(pkg): + atoms = {} + for k in Package._dep_keys: + atom_list = list(_find_built_slot_operator(use_reduce(pkg._metadata[k], + uselist=pkg.use.enabled, eapi=pkg.eapi, + token_class=Atom))) + if atom_list: + atoms[k] = atom_list + return atoms + +def _find_built_slot_operator(dep_struct): + for x in dep_struct: + if isinstance(x, list): + for atom in _find_built_slot_operator(x): + yield atom + elif isinstance(x, Atom) and x.slot_operator_built: + yield x + +def ignore_built_slot_operator_deps(dep_struct): + for i, x in enumerate(dep_struct): + if isinstance(x, list): + ignore_built_slot_operator_deps(x) + elif isinstance(x, Atom) and x.slot_operator_built: + # There's no way of knowing here whether the SLOT + # part of the slot/sub-slot pair should be kept, so we + # ignore both parts. + dep_struct[i] = x.without_slot + +def evaluate_slot_operator_equal_deps(settings, use, trees): + + metadata = settings.configdict['pkg'] + eapi = metadata['EAPI'] + eapi_attrs = _get_eapi_attrs(eapi) + running_vardb = trees[trees._running_eroot]["vartree"].dbapi + target_vardb = trees[trees._target_eroot]["vartree"].dbapi + vardbs = [target_vardb] + deps = {} + for k in Package._dep_keys: + deps[k] = use_reduce(metadata[k], + uselist=use, eapi=eapi, token_class=Atom) + + for k in Package._runtime_keys: + _eval_deps(deps[k], vardbs) + + if eapi_attrs.hdepend: + _eval_deps(deps["HDEPEND"], [running_vardb]) + _eval_deps(deps["DEPEND"], [target_vardb]) + else: + if running_vardb is not target_vardb: + vardbs.append(running_vardb) + _eval_deps(deps["DEPEND"], vardbs) + + result = {} + for k, v in deps.items(): + result[k] = paren_enclose(v) + + return result + +def _eval_deps(dep_struct, vardbs): + for i, x in enumerate(dep_struct): + if isinstance(x, list): + _eval_deps(x, vardbs) + elif isinstance(x, Atom) and x.slot_operator == "=": + for vardb in vardbs: + best_version = vardb.match(x) + if best_version: + best_version = best_version[-1] + try: + best_version = \ + vardb._pkg_str(best_version, None) + except (KeyError, InvalidData): + pass + else: + slot_part = "%s/%s=" % \ + (best_version.slot, best_version.sub_slot) + x = x.with_slot(slot_part) + dep_struct[i] = x + break + else: + # this dep could not be resolved, so remove the operator + # (user may be using package.provided and managing rebuilds + # manually) + if x.slot: + x = x.with_slot(x.slot) + else: + x = x.without_slot + dep_struct[i] = x diff --git a/portage_with_autodep/pym/portage/dep/dep_check.py b/portage_with_autodep/pym/portage/dep/dep_check.py index 01d5021..99a5eb0 100644 --- a/portage_with_autodep/pym/portage/dep/dep_check.py +++ b/portage_with_autodep/pym/portage/dep/dep_check.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['dep_check', 'dep_eval', 'dep_wordreduce', 'dep_zapdeps'] @@ -11,7 +11,7 @@ from portage.dep import Atom, match_from_list, use_reduce from portage.exception import InvalidDependString, ParseError from portage.localization import _ from portage.util import writemsg, writemsg_level -from portage.versions import catpkgsplit, cpv_getkey, pkgcmp +from portage.versions import vercmp, _pkg_str def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", trees=None, use_mask=None, use_force=None, **kwargs): @@ -39,14 +39,12 @@ def _expand_new_virtuals(mysplit, edebug, mydbapi, mysettings, myroot="/", parent = mytrees.get("parent") virt_parent = mytrees.get("virt_parent") graph_parent = None - eapi = None if parent is not None: if virt_parent is not None: graph_parent = virt_parent parent = virt_parent else: graph_parent = parent - eapi = parent.metadata["EAPI"] repoman = not mysettings.local_config if kwargs["use_binaries"]: portdb = trees[myroot]["bintree"].dbapi @@ -352,8 +350,14 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg = mydbapi.match(atom.without_use) if avail_pkg: avail_pkg = avail_pkg[-1] # highest (ascending order) - avail_slot = Atom("%s:%s" % (atom.cp, - mydbapi.aux_get(avail_pkg, ["SLOT"])[0])) + try: + slot = avail_pkg.slot + except AttributeError: + eapi, slot, repo = mydbapi.aux_get(avail_pkg, + ["EAPI", "SLOT", "repository"]) + avail_pkg = _pkg_str(avail_pkg, eapi=eapi, + slot=slot, repo=repo) + avail_slot = Atom("%s:%s" % (atom.cp, slot)) if not avail_pkg: all_available = False all_use_satisfied = False @@ -368,16 +372,19 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): avail_pkg_use = avail_pkg_use[-1] if avail_pkg_use != avail_pkg: avail_pkg = avail_pkg_use - avail_slot = Atom("%s:%s" % (atom.cp, - mydbapi.aux_get(avail_pkg, ["SLOT"])[0])) + try: + slot = avail_pkg.slot + except AttributeError: + eapi, slot, repo = mydbapi.aux_get(avail_pkg, + ["EAPI", "SLOT", "repository"]) + avail_pkg = _pkg_str(avail_pkg, + eapi=eapi, slot=slot, repo=repo) slot_map[avail_slot] = avail_pkg - pkg_cp = cpv_getkey(avail_pkg) - highest_cpv = cp_map.get(pkg_cp) + highest_cpv = cp_map.get(avail_pkg.cp) if highest_cpv is None or \ - pkgcmp(catpkgsplit(avail_pkg)[1:], - catpkgsplit(highest_cpv)[1:]) > 0: - cp_map[pkg_cp] = avail_pkg + vercmp(avail_pkg.version, highest_cpv.version) > 0: + cp_map[avail_pkg.cp] = avail_pkg this_choice = (atoms, slot_map, cp_map, all_available) if all_available: @@ -515,8 +522,7 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): for cp in intersecting_cps: version_1 = cp_map_1[cp] version_2 = cp_map_2[cp] - difference = pkgcmp(catpkgsplit(version_1)[1:], - catpkgsplit(version_2)[1:]) + difference = vercmp(version_1.version, version_2.version) if difference != 0: if difference > 0: has_upgrade = True @@ -539,8 +545,12 @@ def dep_zapdeps(unreduced, reduced, myroot, use_binaries=0, trees=None): assert(False) # This point should not be reachable def dep_check(depstring, mydbapi, mysettings, use="yes", mode=None, myuse=None, - use_cache=1, use_binaries=0, myroot="/", trees=None): - """Takes a depend string and parses the condition.""" + use_cache=1, use_binaries=0, myroot=None, trees=None): + """ + Takes a depend string, parses it, and selects atoms. + The myroot parameter is unused (use mysettings['EROOT'] instead). + """ + myroot = mysettings['EROOT'] edebug = mysettings.get("PORTAGE_DEBUG", None) == "1" #check_config_instance(mysettings) if trees is None: diff --git a/portage_with_autodep/pym/portage/dep/dep_check.pyo b/portage_with_autodep/pym/portage/dep/dep_check.pyo Binary files differnew file mode 100644 index 0000000..1b9e03f --- /dev/null +++ b/portage_with_autodep/pym/portage/dep/dep_check.pyo diff --git a/portage_with_autodep/pym/portage/dispatch_conf.py b/portage_with_autodep/pym/portage/dispatch_conf.py index 4991020..4c68dfc 100644 --- a/portage_with_autodep/pym/portage/dispatch_conf.py +++ b/portage_with_autodep/pym/portage/dispatch_conf.py @@ -1,5 +1,5 @@ # archive_conf.py -- functionality common to archive-conf and dispatch-conf -# Copyright 2003-2011 Gentoo Foundation +# Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -8,11 +8,12 @@ from __future__ import print_function -import os, sys, shutil +import os, shutil, subprocess, sys import portage from portage.env.loaders import KeyValuePairFileLoader from portage.localization import _ +from portage.util import shlex_split, varexpand RCS_BRANCH = '1.1.1' RCS_LOCK = 'rcs -ko -M -l' @@ -22,24 +23,29 @@ RCS_MERGE = "rcsmerge -p -r" + RCS_BRANCH + " '%s' > '%s'" DIFF3_MERGE = "diff3 -mE '%s' '%s' '%s' > '%s'" -def diffstatusoutput_len(cmd): +def diffstatusoutput(cmd, file1, file2): """ Execute the string cmd in a shell with getstatusoutput() and return a - 2-tuple (status, output_length). If getstatusoutput() raises - UnicodeDecodeError (known to happen with python3.1), return a - 2-tuple (1, 1). This provides a simple way to check for non-zero - output length of diff commands, while providing simple handling of - UnicodeDecodeError when necessary. + 2-tuple (status, output). """ - try: - status, output = portage.subprocess_getstatusoutput(cmd) - return (status, len(output)) - except UnicodeDecodeError: - return (1, 1) + # Use Popen to emulate getstatusoutput(), since getstatusoutput() may + # raise a UnicodeDecodeError which makes the output inaccessible. + args = shlex_split(cmd % (file1, file2)) + if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: + # Python 3.1 does not support bytes in Popen args. + args = [portage._unicode_encode(x, errors='strict') for x in args] + proc = subprocess.Popen(args, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output = portage._unicode_decode(proc.communicate()[0]) + if output and output[-1] == "\n": + # getstatusoutput strips one newline + output = output[:-1] + return (proc.wait(), output) def read_config(mandatory_opts): - loader = KeyValuePairFileLoader( - '/etc/dispatch-conf.conf', None) + eprefix = portage.const.EPREFIX + config_path = os.path.join(eprefix or os.sep, "etc/dispatch-conf.conf") + loader = KeyValuePairFileLoader(config_path, None) opts, errors = loader.load() if not opts: print(_('dispatch-conf: Error reading /etc/dispatch-conf.conf; fatal'), file=sys.stderr) @@ -58,6 +64,10 @@ def read_config(mandatory_opts): else: print(_('dispatch-conf: Missing option "%s" in /etc/dispatch-conf.conf; fatal') % (key,), file=sys.stderr) + # archive-dir supports ${EPREFIX} expansion, in order to avoid hardcoding + variables = {"EPREFIX": eprefix} + opts['archive-dir'] = varexpand(opts['archive-dir'], mydict=variables) + if not os.path.exists(opts['archive-dir']): os.mkdir(opts['archive-dir']) # Use restrictive permissions by default, in order to protect @@ -132,7 +142,7 @@ def file_archive(archive, curconf, newconf, mrgconf): # Archive the current config file if it isn't already saved if os.path.exists(archive) \ - and diffstatusoutput_len("diff -aq '%s' '%s'" % (curconf,archive))[1] != 0: + and len(diffstatusoutput("diff -aq '%s' '%s'", curconf, archive)[1]) != 0: suf = 1 while suf < 9 and os.path.exists(archive + '.' + str(suf)): suf += 1 diff --git a/portage_with_autodep/pym/portage/dispatch_conf.pyo b/portage_with_autodep/pym/portage/dispatch_conf.pyo Binary files differnew file mode 100644 index 0000000..6239859 --- /dev/null +++ b/portage_with_autodep/pym/portage/dispatch_conf.pyo diff --git a/portage_with_autodep/pym/portage/eapi.py b/portage_with_autodep/pym/portage/eapi.py index da5fd8c..79cf891 100644 --- a/portage_with_autodep/pym/portage/eapi.py +++ b/portage_with_autodep/pym/portage/eapi.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 def eapi_has_iuse_defaults(eapi): @@ -34,6 +34,9 @@ def eapi_exports_merge_type(eapi): def eapi_exports_replace_vars(eapi): return eapi not in ("0", "1", "2", "3") +def eapi_exports_REPOSITORY(eapi): + return eapi in ("4-python",) + def eapi_has_pkg_pretend(eapi): return eapi not in ("0", "1", "2", "3") @@ -48,3 +51,12 @@ def eapi_has_required_use(eapi): def eapi_has_use_dep_defaults(eapi): return eapi not in ("0", "1", "2", "3") + +def eapi_has_repo_deps(eapi): + return eapi in ("4-python",) + +def eapi_allows_dots_in_PN(eapi): + return eapi in ("4-python",) + +def eapi_allows_dots_in_use_flags(eapi): + return eapi in ("4-python",) diff --git a/portage_with_autodep/pym/portage/eapi.pyo b/portage_with_autodep/pym/portage/eapi.pyo Binary files differnew file mode 100644 index 0000000..ce67ab1 --- /dev/null +++ b/portage_with_autodep/pym/portage/eapi.pyo diff --git a/portage_with_autodep/pym/portage/eclass_cache.py b/portage_with_autodep/pym/portage/eclass_cache.py index 1374f1d..cb2cf8a 100644 --- a/portage_with_autodep/pym/portage/eclass_cache.py +++ b/portage_with_autodep/pym/portage/eclass_cache.py @@ -6,21 +6,59 @@ __all__ = ["cache"] import stat import sys +import operator from portage.util import normalize_path import errno -from portage.exception import PermissionDenied +from portage.exception import FileNotFound, PermissionDenied from portage import os +from portage import checksum if sys.hexversion >= 0x3000000: long = int + +class hashed_path(object): + + def __init__(self, location): + self.location = location + + def __getattr__(self, attr): + if attr == 'mtime': + # use stat.ST_MTIME; accessing .st_mtime gets you a float + # depending on the python version, and long(float) introduces + # some rounding issues that aren't present for people using + # the straight c api. + # thus use the defacto python compatibility work around; + # access via index, which guarantees you get the raw long. + try: + self.mtime = obj = os.stat(self.location)[stat.ST_MTIME] + except OSError as e: + if e.errno in (errno.ENOENT, errno.ESTALE): + raise FileNotFound(self.location) + elif e.errno == PermissionDenied.errno: + raise PermissionDenied(self.location) + raise + return obj + if not attr.islower(): + # we don't care to allow .mD5 as an alias for .md5 + raise AttributeError(attr) + hashname = attr.upper() + if hashname not in checksum.hashfunc_map: + raise AttributeError(attr) + val = checksum.perform_checksum(self.location, hashname)[0] + setattr(self, attr, val) + return val + + def __repr__(self): + return "<portage.eclass_cache.hashed_path('%s')>" % (self.location,) + class cache(object): """ Maintains the cache information about eclasses used in ebuild. """ def __init__(self, porttree_root, overlays=[]): - self.eclasses = {} # {"Name": ("location","_mtime_")} + self.eclasses = {} # {"Name": hashed_path} self._eclass_locations = {} # screw with the porttree ordering, w/out having bash inherit match it, and I'll hurt you. @@ -80,14 +118,16 @@ class cache(object): for y in eclass_filenames: if not y.endswith(".eclass"): continue + obj = hashed_path(os.path.join(x, y)) + obj.eclass_dir = x try: - mtime = os.stat(os.path.join(x, y))[stat.ST_MTIME] - except OSError: + mtime = obj.mtime + except FileNotFound: continue ys=y[:-eclass_len] if x == self._master_eclass_root: master_eclasses[ys] = mtime - self.eclasses[ys] = (x, mtime) + self.eclasses[ys] = obj self._eclass_locations[ys] = x continue @@ -98,22 +138,30 @@ class cache(object): # so prefer the master entry. continue - self.eclasses[ys] = (x, mtime) + self.eclasses[ys] = obj self._eclass_locations[ys] = x - def is_eclass_data_valid(self, ec_dict): + def validate_and_rewrite_cache(self, ec_dict, chf_type, stores_paths): + """ + This will return an empty dict if the ec_dict parameter happens + to be empty, therefore callers must take care to distinguish + between empty dict and None return values. + """ if not isinstance(ec_dict, dict): - return False - for eclass, tup in ec_dict.items(): - cached_data = self.eclasses.get(eclass, None) - """ Only use the mtime for validation since the probability of a - collision is small and, depending on the cache implementation, the - path may not be specified (cache from rsync mirrors, for example). - """ - if cached_data is None or tup[1] != cached_data[1]: - return False - - return True + return None + our_getter = operator.attrgetter(chf_type) + cache_getter = lambda x:x + if stores_paths: + cache_getter = operator.itemgetter(1) + d = {} + for eclass, ec_data in ec_dict.items(): + cached_data = self.eclasses.get(eclass) + if cached_data is None: + return None + if cache_getter(ec_data) != our_getter(cached_data): + return None + d[eclass] = cached_data + return d def get_eclass_data(self, inherits): ec_dict = {} diff --git a/portage_with_autodep/pym/portage/eclass_cache.pyo b/portage_with_autodep/pym/portage/eclass_cache.pyo Binary files differnew file mode 100644 index 0000000..ebe3463 --- /dev/null +++ b/portage_with_autodep/pym/portage/eclass_cache.pyo diff --git a/portage_with_autodep/pym/portage/elog/__init__.py b/portage_with_autodep/pym/portage/elog/__init__.py index 1a8309d..33dac17 100644 --- a/portage_with_autodep/pym/portage/elog/__init__.py +++ b/portage_with_autodep/pym/portage/elog/__init__.py @@ -1,7 +1,11 @@ # elog/__init__.py - elog core functions -# Copyright 2006-2009 Gentoo Foundation +# Copyright 2006-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +import sys +if sys.hexversion >= 0x3000000: + basestring = str + import portage portage.proxy.lazyimport.lazyimport(globals(), 'portage.util:writemsg', @@ -52,11 +56,15 @@ def _combine_logentries(logentries): for msgtype, msgcontent in logentries[phase]: if previous_type != msgtype: previous_type = msgtype - rValue.append("%s: %s\n" % (msgtype, phase)) - for line in msgcontent: - rValue.append(line) - rValue.append("\n") - return "".join(rValue) + rValue.append("%s: %s" % (msgtype, phase)) + if isinstance(msgcontent, basestring): + rValue.append(msgcontent.rstrip("\n")) + else: + for line in msgcontent: + rValue.append(line.rstrip("\n")) + if rValue: + rValue.append("") + return "\n".join(rValue) _elog_mod_imports = {} def _load_mod(name): diff --git a/portage_with_autodep/pym/portage/elog/__init__.pyo b/portage_with_autodep/pym/portage/elog/__init__.pyo Binary files differnew file mode 100644 index 0000000..39dc595 --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/__init__.pyo diff --git a/portage_with_autodep/pym/portage/elog/filtering.pyo b/portage_with_autodep/pym/portage/elog/filtering.pyo Binary files differnew file mode 100644 index 0000000..3a040cc --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/filtering.pyo diff --git a/portage_with_autodep/pym/portage/elog/messages.py b/portage_with_autodep/pym/portage/elog/messages.py index 6c1580a..a4897d8 100644 --- a/portage_with_autodep/pym/portage/elog/messages.py +++ b/portage_with_autodep/pym/portage/elog/messages.py @@ -18,6 +18,14 @@ from portage import _unicode_decode import io import sys +_log_levels = frozenset([ + "ERROR", + "INFO", + "LOG", + "QA", + "WARN", +]) + def collect_ebuild_messages(path): """ Collect elog messages generated by the bash logging function stored at 'path'. @@ -43,16 +51,21 @@ def collect_ebuild_messages(path): logentries[msgfunction] = [] lastmsgtype = None msgcontent = [] - for l in io.open(_unicode_encode(filename, + f = io.open(_unicode_encode(filename, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace'): + mode='r', encoding=_encodings['repo.content'], errors='replace') + # Use split('\n') since normal line iteration or readlines() will + # split on \r characters as shown in bug #390833. + for l in f.read().split('\n'): if not l: continue try: msgtype, msg = l.split(" ", 1) + if msgtype not in _log_levels: + raise ValueError(msgtype) except ValueError: writemsg(_("!!! malformed entry in " - "log file: '%s'\n") % filename, noiselevel=-1) + "log file: '%s': %s\n") % (filename, l), noiselevel=-1) continue if lastmsgtype is None: @@ -65,6 +78,7 @@ def collect_ebuild_messages(path): logentries[msgfunction].append((lastmsgtype, msgcontent)) msgcontent = [msg] lastmsgtype = msgtype + f.close() if msgcontent: logentries[msgfunction].append((lastmsgtype, msgcontent)) @@ -159,13 +173,17 @@ _functions = { "einfo": ("INFO", "GOOD"), "eerror": ("ERROR", "BAD"), } -def _make_msgfunction(level, color): - def _elog(msg, phase="other", key=None, out=None): - """ Display and log a message assigned to the given key/cpv - (or unassigned if no key is given). +class _make_msgfunction(object): + __slots__ = ('_color', '_level') + def __init__(self, level, color): + self._level = level + self._color = color + def __call__(self, msg, phase="other", key=None, out=None): + """ + Display and log a message assigned to the given key/cpv. """ - _elog_base(level, msg, phase=phase, key=key, color=color, out=out) - return _elog + _elog_base(self._level, msg, phase=phase, + key=key, color=self._color, out=out) for f in _functions: setattr(sys.modules[__name__], f, _make_msgfunction(_functions[f][0], _functions[f][1])) diff --git a/portage_with_autodep/pym/portage/elog/messages.pyo b/portage_with_autodep/pym/portage/elog/messages.pyo Binary files differnew file mode 100644 index 0000000..c033f55 --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/messages.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_custom.pyo b/portage_with_autodep/pym/portage/elog/mod_custom.pyo Binary files differnew file mode 100644 index 0000000..317fab4 --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/mod_custom.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_echo.py b/portage_with_autodep/pym/portage/elog/mod_echo.py index 5de25bf..59117be 100644 --- a/portage_with_autodep/pym/portage/elog/mod_echo.py +++ b/portage_with_autodep/pym/portage/elog/mod_echo.py @@ -18,6 +18,19 @@ def process(mysettings, key, logentries, fulltext): _items.append((mysettings["ROOT"], key, logentries)) def finalize(): + # For consistency, send all message types to stdout. + sys.stdout.flush() + sys.stderr.flush() + stderr = sys.stderr + try: + sys.stderr = sys.stdout + _finalize() + finally: + sys.stderr = stderr + sys.stdout.flush() + sys.stderr.flush() + +def _finalize(): global _items printer = EOutput() for root, key, logentries in _items: diff --git a/portage_with_autodep/pym/portage/elog/mod_echo.pyo b/portage_with_autodep/pym/portage/elog/mod_echo.pyo Binary files differnew file mode 100644 index 0000000..6a00d4c --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/mod_echo.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_mail.pyo b/portage_with_autodep/pym/portage/elog/mod_mail.pyo Binary files differnew file mode 100644 index 0000000..5d87aa6 --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/mod_mail.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo b/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo Binary files differnew file mode 100644 index 0000000..d7306b5 --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/mod_mail_summary.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_save.py b/portage_with_autodep/pym/portage/elog/mod_save.py index 9350a6e..c69f4a3 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save.py +++ b/portage_with_autodep/pym/portage/elog/mod_save.py @@ -4,20 +4,22 @@ import io import time +import portage from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode from portage.data import portage_gid, portage_uid from portage.package.ebuild.prepare_build_dirs import _ensure_log_subdirs -from portage.util import ensure_dirs, normalize_path +from portage.util import apply_permissions, ensure_dirs, normalize_path def process(mysettings, key, logentries, fulltext): if mysettings.get("PORT_LOGDIR"): logdir = normalize_path(mysettings["PORT_LOGDIR"]) else: - logdir = os.path.join(os.sep, "var", "log", "portage") + logdir = os.path.join(os.sep, mysettings["EPREFIX"].lstrip(os.sep), + "var", "log", "portage") if not os.path.isdir(logdir): # Only initialize group/mode if the directory doesn't @@ -25,7 +27,10 @@ def process(mysettings, key, logentries, fulltext): # were previously set by the administrator. # NOTE: These permissions should be compatible with our # default logrotate config as discussed in bug 374287. - ensure_dirs(logdir, uid=portage_uid, gid=portage_gid, mode=0o2770) + uid = -1 + if portage.data.secpass >= 2: + uid = portage_uid + ensure_dirs(logdir, uid=uid, gid=portage_gid, mode=0o2770) cat = mysettings['CATEGORY'] pf = mysettings['PF'] @@ -48,4 +53,21 @@ def process(mysettings, key, logentries, fulltext): elogfile.write(_unicode_decode(fulltext)) elogfile.close() + # Copy group permission bits from parent directory. + elogdir_st = os.stat(log_subdir) + elogdir_gid = elogdir_st.st_gid + elogdir_grp_mode = 0o060 & elogdir_st.st_mode + + # Copy the uid from the parent directory if we have privileges + # to do so, for compatibility with our default logrotate + # config (see bug 378451). With the "su portage portage" + # directive and logrotate-3.8.0, logrotate's chown call during + # the compression phase will only succeed if the log file's uid + # is portage_uid. + logfile_uid = -1 + if portage.data.secpass >= 2: + logfile_uid = elogdir_st.st_uid + apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid, + mode=elogdir_grp_mode, mask=0) + return elogfilename diff --git a/portage_with_autodep/pym/portage/elog/mod_save.pyo b/portage_with_autodep/pym/portage/elog/mod_save.pyo Binary files differnew file mode 100644 index 0000000..fb28b76 --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/mod_save.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_save_summary.py b/portage_with_autodep/pym/portage/elog/mod_save_summary.py index 4adc6f3..347f66e 100644 --- a/portage_with_autodep/pym/portage/elog/mod_save_summary.py +++ b/portage_with_autodep/pym/portage/elog/mod_save_summary.py @@ -4,6 +4,7 @@ import io import time +import portage from portage import os from portage import _encodings from portage import _unicode_decode @@ -17,7 +18,8 @@ def process(mysettings, key, logentries, fulltext): if mysettings.get("PORT_LOGDIR"): logdir = normalize_path(mysettings["PORT_LOGDIR"]) else: - logdir = os.path.join(os.sep, "var", "log", "portage") + logdir = os.path.join(os.sep, mysettings["EPREFIX"].lstrip(os.sep), + "var", "log", "portage") if not os.path.isdir(logdir): # Only initialize group/mode if the directory doesn't @@ -25,7 +27,10 @@ def process(mysettings, key, logentries, fulltext): # were previously set by the administrator. # NOTE: These permissions should be compatible with our # default logrotate config as discussed in bug 374287. - ensure_dirs(logdir, uid=portage_uid, gid=portage_gid, mode=0o2770) + logdir_uid = -1 + if portage.data.secpass >= 2: + logdir_uid = portage_uid + ensure_dirs(logdir, uid=logdir_uid, gid=portage_gid, mode=0o2770) elogdir = os.path.join(logdir, "elog") _ensure_log_subdirs(logdir, elogdir) @@ -40,7 +45,17 @@ def process(mysettings, key, logentries, fulltext): elogdir_st = os.stat(elogdir) elogdir_gid = elogdir_st.st_gid elogdir_grp_mode = 0o060 & elogdir_st.st_mode - apply_permissions(elogfilename, gid=elogdir_gid, + + # Copy the uid from the parent directory if we have privileges + # to do so, for compatibility with our default logrotate + # config (see bug 378451). With the "su portage portage" + # directive and logrotate-3.8.0, logrotate's chown call during + # the compression phase will only succeed if the log file's uid + # is portage_uid. + logfile_uid = -1 + if portage.data.secpass >= 2: + logfile_uid = elogdir_st.st_uid + apply_permissions(elogfilename, uid=logfile_uid, gid=elogdir_gid, mode=elogdir_grp_mode, mask=0) time_str = time.strftime("%Y-%m-%d %H:%M:%S %Z", diff --git a/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo b/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo Binary files differnew file mode 100644 index 0000000..8f99c51 --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/mod_save_summary.pyo diff --git a/portage_with_autodep/pym/portage/elog/mod_syslog.py b/portage_with_autodep/pym/portage/elog/mod_syslog.py index d71dab4..c8bf441 100644 --- a/portage_with_autodep/pym/portage/elog/mod_syslog.py +++ b/portage_with_autodep/pym/portage/elog/mod_syslog.py @@ -1,5 +1,5 @@ # elog/mod_syslog.py - elog dispatch module -# Copyright 2006-2007 Gentoo Foundation +# Copyright 2006-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -7,6 +7,9 @@ import syslog from portage.const import EBUILD_PHASES from portage import _encodings +if sys.hexversion >= 0x3000000: + basestring = str + _pri = { "INFO" : syslog.LOG_INFO, "WARN" : syslog.LOG_WARNING, @@ -21,12 +24,13 @@ def process(mysettings, key, logentries, fulltext): if not phase in logentries: continue for msgtype,msgcontent in logentries[phase]: - msgtext = "".join(msgcontent) - for line in msgtext.splitlines(): + if isinstance(msgcontent, basestring): + msgcontent = [msgcontent] + for line in msgcontent: line = "%s: %s: %s" % (key, phase, line) - if sys.hexversion < 0x3000000 and isinstance(msgtext, unicode): + if sys.hexversion < 0x3000000 and not isinstance(line, bytes): # Avoid TypeError from syslog.syslog() line = line.encode(_encodings['content'], 'backslashreplace') - syslog.syslog(_pri[msgtype], line) + syslog.syslog(_pri[msgtype], line.rstrip("\n")) syslog.closelog() diff --git a/portage_with_autodep/pym/portage/elog/mod_syslog.pyo b/portage_with_autodep/pym/portage/elog/mod_syslog.pyo Binary files differnew file mode 100644 index 0000000..c7b4248 --- /dev/null +++ b/portage_with_autodep/pym/portage/elog/mod_syslog.pyo diff --git a/portage_with_autodep/pym/portage/emaint/__init__.py b/portage_with_autodep/pym/portage/emaint/__init__.py new file mode 100644 index 0000000..48bc6e2 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/__init__.py @@ -0,0 +1,5 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""System health checks and maintenance utilities. +""" diff --git a/portage_with_autodep/pym/portage/emaint/defaults.py b/portage_with_autodep/pym/portage/emaint/defaults.py new file mode 100644 index 0000000..30f36af --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/defaults.py @@ -0,0 +1,25 @@ +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +# parser option data +CHECK = {"short": "-c", "long": "--check", + "help": "Check for problems (a default option for most modules)", + 'status': "Checking %s for problems", + 'action': 'store_true', + 'func': 'check' + } + +FIX = {"short": "-f", "long": "--fix", + "help": "Attempt to fix problems (a default option for most modules)", + 'status': "Attempting to fix %s", + 'action': 'store_true', + 'func': 'fix' + } + +VERSION = {"long": "--version", + "help": "show program's version number and exit", + 'action': 'store_true', + } + +# parser options +DEFAULT_OPTIONS = {'check': CHECK, 'fix': FIX, 'version': VERSION} diff --git a/portage_with_autodep/pym/portage/emaint/main.py b/portage_with_autodep/pym/portage/emaint/main.py new file mode 100644 index 0000000..9f987fa --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/main.py @@ -0,0 +1,222 @@ +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + + +import sys +import textwrap + +import portage +from portage import os +from portage.emaint.module import Modules +from portage.emaint.progress import ProgressBar +from portage.emaint.defaults import DEFAULT_OPTIONS +from portage.util._argparse import ArgumentParser + +class OptionItem(object): + """class to hold module ArgumentParser options data + """ + + def __init__(self, opt): + """ + @type opt: dictionary + @param opt: options parser options + """ + self.short = opt.get('short') + self.long = opt.get('long') + self.help = opt.get('help') + self.status = opt.get('status') + self.func = opt.get('func') + self.action = opt.get('action') + self.type = opt.get('type') + self.dest = opt.get('dest') + + @property + def pargs(self): + pargs = [] + if self.short is not None: + pargs.append(self.short) + if self.long is not None: + pargs.append(self.long) + return pargs + + @property + def kwargs(self): + # Support for keyword arguments varies depending on the action, + # so only pass in the keywords that are needed, in order + # to avoid a TypeError. + kwargs = {} + if self.help is not None: + kwargs['help'] = self.help + if self.action is not None: + kwargs['action'] = self.action + if self.type is not None: + kwargs['type'] = self.type + if self.dest is not None: + kwargs['dest'] = self.dest + return kwargs + +def usage(module_controller): + _usage = "usage: emaint [options] COMMAND" + + desc = "The emaint program provides an interface to system health " + \ + "checks and maintenance. See the emaint(1) man page " + \ + "for additional information about the following commands:" + + _usage += "\n\n" + for line in textwrap.wrap(desc, 65): + _usage += "%s\n" % line + _usage += "\nCommands:\n" + _usage += " %s" % "all".ljust(15) + \ + "Perform all supported commands\n" + textwrap.subsequent_indent = ' '.ljust(17) + for mod in module_controller.module_names: + desc = textwrap.wrap(module_controller.get_description(mod), 65) + _usage += " %s%s\n" % (mod.ljust(15), desc[0]) + for d in desc[1:]: + _usage += " %s%s\n" % (' '.ljust(15), d) + return _usage + + +def module_opts(module_controller, module): + _usage = " %s module options:\n" % module + opts = module_controller.get_func_descriptions(module) + if opts == {}: + opts = DEFAULT_OPTIONS + for opt in sorted(opts): + optd = opts[opt] + opto = " %s, %s" %(optd['short'], optd['long']) + _usage += '%s %s\n' % (opto.ljust(15),optd['help']) + _usage += '\n' + return _usage + + +class TaskHandler(object): + """Handles the running of the tasks it is given + """ + + def __init__(self, show_progress_bar=True, verbose=True, callback=None): + self.show_progress_bar = show_progress_bar + self.verbose = verbose + self.callback = callback + self.isatty = os.environ.get('TERM') != 'dumb' and sys.stdout.isatty() + self.progress_bar = ProgressBar(self.isatty, title="Emaint", max_desc_length=27) + + + def run_tasks(self, tasks, func, status=None, verbose=True, options=None): + """Runs the module tasks""" + if tasks is None or func is None: + return + for task in tasks: + inst = task() + show_progress = self.show_progress_bar and self.isatty + # check if the function is capable of progressbar + # and possibly override it off + if show_progress and hasattr(inst, 'can_progressbar'): + show_progress = inst.can_progressbar(func) + if show_progress: + self.progress_bar.reset() + self.progress_bar.set_label(func + " " + inst.name()) + onProgress = self.progress_bar.start() + else: + onProgress = None + kwargs = { + 'onProgress': onProgress, + # pass in a copy of the options so a module can not pollute or change + # them for other tasks if there is more to do. + 'options': options.copy() + } + result = getattr(inst, func)(**kwargs) + if show_progress: + # make sure the final progress is displayed + self.progress_bar.display() + print() + self.progress_bar.stop() + if self.callback: + self.callback(result) + + +def print_results(results): + if results: + print() + print("\n".join(results)) + print("\n") + + +def emaint_main(myargv): + + # Similar to emerge, emaint needs a default umask so that created + # files (such as the world file) have sane permissions. + os.umask(0o22) + + module_controller = Modules(namepath="portage.emaint.modules") + module_names = module_controller.module_names[:] + module_names.insert(0, "all") + + + parser = ArgumentParser(usage=usage(module_controller)) + # add default options + parser_options = [] + for opt in DEFAULT_OPTIONS: + parser_options.append(OptionItem(DEFAULT_OPTIONS[opt])) + for mod in module_names[1:]: + desc = module_controller.get_func_descriptions(mod) + if desc: + for opt in desc: + parser_options.append(OptionItem(desc[opt])) + for opt in parser_options: + parser.add_argument(*opt.pargs, **opt.kwargs) + + options, args = parser.parse_known_args(args=myargv) + + if options.version: + print(portage.VERSION) + return os.EX_OK + + if len(args) != 1: + parser.error("Incorrect number of arguments") + if args[0] not in module_names: + parser.error("%s target is not a known target" % args[0]) + + check_opt = None + func = status = long_action = None + for opt in parser_options: + if opt.long == '--check': + # Default action + check_opt = opt + if opt.status and getattr(options, opt.long.lstrip("-"), False): + if long_action is not None: + parser.error("--%s and %s are exclusive options" % + (long_action, opt.long)) + status = opt.status + func = opt.func + long_action = opt.long.lstrip('-') + + if long_action is None: + long_action = 'check' + func = check_opt.func + status = check_opt.status + + if args[0] == "all": + tasks = [] + for m in module_names[1:]: + #print("DEBUG: module: %s, functions: " %(m, str(module_controller.get_functions(m)))) + if long_action in module_controller.get_functions(m): + tasks.append(module_controller.get_class(m)) + elif long_action in module_controller.get_functions(args[0]): + tasks = [module_controller.get_class(args[0] )] + else: + portage.util.writemsg( + "\nERROR: module '%s' does not have option '--%s'\n\n" % + (args[0], long_action), noiselevel=-1) + portage.util.writemsg(module_opts(module_controller, args[0]), + noiselevel=-1) + sys.exit(1) + + # need to pass the parser options dict to the modules + # so they are available if needed. + task_opts = options.__dict__ + taskmaster = TaskHandler(callback=print_results) + taskmaster.run_tasks(tasks, func, status, options=task_opts) + diff --git a/portage_with_autodep/pym/portage/emaint/module.py b/portage_with_autodep/pym/portage/emaint/module.py new file mode 100644 index 0000000..64b0c64 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/module.py @@ -0,0 +1,194 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + + +from __future__ import print_function + +from portage import os +from portage.exception import PortageException +from portage.cache.mappings import ProtectedDict + + +class InvalidModuleName(PortageException): + """An invalid or unknown module name.""" + + +class Module(object): + """Class to define and hold our plug-in module + + @type name: string + @param name: the module name + @type path: the path to the new module + """ + + def __init__(self, name, namepath): + """Some variables initialization""" + self.name = name + self._namepath = namepath + self.kids_names = [] + self.kids = {} + self.initialized = self._initialize() + + def _initialize(self): + """Initialize the plug-in module + + @rtype: boolean + """ + self.valid = False + try: + mod_name = ".".join([self._namepath, self.name]) + self._module = __import__(mod_name, [],[], ["not empty"]) + self.valid = True + except ImportError as e: + print("MODULE; failed import", mod_name, " error was:",e) + return False + self.module_spec = self._module.module_spec + for submodule in self.module_spec['provides']: + kid = self.module_spec['provides'][submodule] + kidname = kid['name'] + kid['module_name'] = '.'.join([mod_name, self.name]) + kid['is_imported'] = False + self.kids[kidname] = kid + self.kids_names.append(kidname) + return True + + def get_class(self, name): + if not name or name not in self.kids_names: + raise InvalidModuleName("Module name '%s' was invalid or not" + %name + "part of the module '%s'" %self.name) + kid = self.kids[name] + if kid['is_imported']: + module = kid['instance'] + else: + try: + module = __import__(kid['module_name'], [],[], ["not empty"]) + kid['instance'] = module + kid['is_imported'] = True + except ImportError: + raise + mod_class = getattr(module, kid['class']) + return mod_class + + +class Modules(object): + """Dynamic modules system for loading and retrieving any of the + installed emaint modules and/or provided class's + + @param path: Optional path to the "modules" directory or + defaults to the directory of this file + '/modules' + @param namepath: Optional python import path to the "modules" directory or + defaults to the directory name of this file + '.modules' + """ + + def __init__(self, path=None, namepath=None): + if path: + self._module_path = path + else: + self._module_path = os.path.join(( + os.path.dirname(os.path.realpath(__file__))), "modules") + if namepath: + self._namepath = namepath + else: + self._namepath = '.'.join(os.path.dirname( + os.path.realpath(__file__)), "modules") + self._modules = self._get_all_modules() + self.modules = ProtectedDict(self._modules) + self.module_names = sorted(self._modules) + #self.modules = {} + #for mod in self.module_names: + #self.module[mod] = LazyLoad( + + def _get_all_modules(self): + """scans the emaint modules dir for loadable modules + + @rtype: dictionary of module_plugins + """ + module_dir = self._module_path + importables = [] + names = os.listdir(module_dir) + for entry in names: + # skip any __init__ or __pycache__ files or directories + if entry.startswith('__'): + continue + try: + # test for statinfo to ensure it should a real module + # it will bail if it errors + os.lstat(os.path.join(module_dir, entry, '__init__.py')) + importables.append(entry) + except EnvironmentError: + pass + kids = {} + for entry in importables: + new_module = Module(entry, self._namepath) + for module_name in new_module.kids: + kid = new_module.kids[module_name] + kid['parent'] = new_module + kids[kid['name']] = kid + return kids + + def get_module_names(self): + """Convienence function to return the list of installed modules + available + + @rtype: list + @return: the installed module names available + """ + return self.module_names + + def get_class(self, modname): + """Retrieves a module class desired + + @type modname: string + @param modname: the module class name + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['parent'].get_class(modname) + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_description(self, modname): + """Retrieves the module class decription + + @type modname: string + @param modname: the module class name + @type string + @return: the modules class decription + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['description'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_functions(self, modname): + """Retrieves the module class exported function names + + @type modname: string + @param modname: the module class name + @type list + @return: the modules class exported function names + """ + if modname and modname in self.module_names: + mod = self._modules[modname]['functions'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return mod + + def get_func_descriptions(self, modname): + """Retrieves the module class exported functions descriptions + + @type modname: string + @param modname: the module class name + @type dictionary + @return: the modules class exported functions descriptions + """ + if modname and modname in self.module_names: + desc = self._modules[modname]['func_desc'] + else: + raise InvalidModuleName("Module name '%s' was invalid or not" + %modname + "found") + return desc diff --git a/portage_with_autodep/pym/portage/emaint/modules/__init__.py b/portage_with_autodep/pym/portage/emaint/modules/__init__.py new file mode 100644 index 0000000..f67197d --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/__init__.py @@ -0,0 +1,5 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""Plug-in modules for system health checks and maintenance. +""" diff --git a/portage_with_autodep/pym/portage/emaint/modules/binhost/__init__.py b/portage_with_autodep/pym/portage/emaint/modules/binhost/__init__.py new file mode 100644 index 0000000..c60e8bc --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/binhost/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""Scan and generate metadata indexes for binary packages. +""" + + +module_spec = { + 'name': 'binhost', + 'description': __doc__, + 'provides':{ + 'module1': { + 'name': "binhost", + 'class': "BinhostHandler", + 'description': __doc__, + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/portage_with_autodep/pym/portage/emaint/modules/binhost/binhost.py b/portage_with_autodep/pym/portage/emaint/modules/binhost/binhost.py new file mode 100644 index 0000000..c297545 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/binhost/binhost.py @@ -0,0 +1,163 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import stat + +import portage +from portage import os +from portage.util import writemsg + +import sys +if sys.hexversion >= 0x3000000: + long = int + +class BinhostHandler(object): + + short_desc = "Generate a metadata index for binary packages" + + def name(): + return "binhost" + name = staticmethod(name) + + def __init__(self): + eroot = portage.settings['EROOT'] + self._bintree = portage.db[eroot]["bintree"] + self._bintree.populate() + self._pkgindex_file = self._bintree._pkgindex_file + self._pkgindex = self._bintree._load_pkgindex() + + def _need_update(self, cpv, data): + + if "MD5" not in data: + return True + + size = data.get("SIZE") + if size is None: + return True + + mtime = data.get("MTIME") + if mtime is None: + return True + + pkg_path = self._bintree.getname(cpv) + try: + s = os.lstat(pkg_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + # We can't update the index for this one because + # it disappeared. + return False + + try: + if long(mtime) != s[stat.ST_MTIME]: + return True + if long(size) != long(s.st_size): + return True + except ValueError: + return True + + return False + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + missing = [] + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + if onProgress: + onProgress(maxval, i+1) + errors = ["'%s' is not in Packages" % cpv for cpv in missing] + stale = set(metadata).difference(cpv_all) + for cpv in stale: + errors.append("'%s' is not in the repository" % cpv) + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + bintree = self._bintree + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + missing = [] + maxval = 0 + if onProgress: + onProgress(maxval, 0) + pkgindex = self._pkgindex + missing = [] + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + + stale = set(metadata).difference(cpv_all) + if missing or stale: + from portage import locks + pkgindex_lock = locks.lockfile( + self._pkgindex_file, wantnewlockfile=1) + try: + # Repopulate with lock held. + bintree._populate() + cpv_all = self._bintree.dbapi.cpv_all() + cpv_all.sort() + + pkgindex = bintree._load_pkgindex() + self._pkgindex = pkgindex + + metadata = {} + for d in pkgindex.packages: + metadata[d["CPV"]] = d + + # Recount missing packages, with lock held. + del missing[:] + for i, cpv in enumerate(cpv_all): + d = metadata.get(cpv) + if not d or self._need_update(cpv, d): + missing.append(cpv) + + maxval = len(missing) + for i, cpv in enumerate(missing): + try: + metadata[cpv] = bintree._pkgindex_entry(cpv) + except portage.exception.InvalidDependString: + writemsg("!!! Invalid binary package: '%s'\n" % \ + bintree.getname(cpv), noiselevel=-1) + + if onProgress: + onProgress(maxval, i+1) + + for cpv in set(metadata).difference( + self._bintree.dbapi.cpv_all()): + del metadata[cpv] + + # We've updated the pkgindex, so set it to + # repopulate when necessary. + bintree.populated = False + + del pkgindex.packages[:] + pkgindex.packages.extend(metadata.values()) + bintree._pkgindex_write(self._pkgindex) + + finally: + locks.unlockfile(pkgindex_lock) + + if onProgress: + if maxval == 0: + maxval = 1 + onProgress(maxval, maxval) + return None diff --git a/portage_with_autodep/pym/portage/emaint/modules/config/__init__.py b/portage_with_autodep/pym/portage/emaint/modules/config/__init__.py new file mode 100644 index 0000000..f0585b3 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/config/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""Check and clean the config tracker list for uninstalled packages. +""" + + +module_spec = { + 'name': 'config', + 'description': __doc__, + 'provides':{ + 'module1': { + 'name': "cleanconfmem", + 'class': "CleanConfig", + 'description': __doc__, + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/portage_with_autodep/pym/portage/emaint/modules/config/config.py b/portage_with_autodep/pym/portage/emaint/modules/config/config.py new file mode 100644 index 0000000..dad024b --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/config/config.py @@ -0,0 +1,79 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.const import PRIVATE_PATH +from portage.util import grabdict, writedict + +class CleanConfig(object): + + short_desc = "Discard any no longer installed configs from emerge's tracker list" + + def __init__(self): + self._root = portage.settings["ROOT"] + self.target = os.path.join(portage.settings["EROOT"], PRIVATE_PATH, 'config') + + def name(): + return "cleanconfmem" + name = staticmethod(name) + + def load_configlist(self): + return grabdict(self.target) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + configs = self.load_configlist() + messages = [] + maxval = len(configs) + if onProgress: + onProgress(maxval, 0) + i = 0 + keys = sorted(configs) + for config in keys: + if not os.path.exists(config): + messages.append(" %s" % config) + if onProgress: + onProgress(maxval, i+1) + i += 1 + return self._format_output(messages) + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + configs = self.load_configlist() + messages = [] + maxval = len(configs) + if onProgress: + onProgress(maxval, 0) + i = 0 + + root = self._root + if root == "/": + root = None + modified = False + for config in sorted(configs): + if root is None: + full_path = config + else: + full_path = os.path.join(root, config.lstrip(os.sep)) + if not os.path.exists(full_path): + modified = True + configs.pop(config) + messages.append(" %s" % config) + if onProgress: + onProgress(maxval, i+1) + i += 1 + if modified: + writedict(configs, self.target) + return self._format_output(messages, True) + + def _format_output(self, messages=[], cleaned=False): + output = [] + if messages: + output.append('Not Installed:') + output += messages + tot = '------------------------------------\n Total %i Not installed' + if cleaned: + tot += ' ...Cleaned' + output.append(tot % len(messages)) + return output diff --git a/portage_with_autodep/pym/portage/emaint/modules/logs/__init__.py b/portage_with_autodep/pym/portage/emaint/modules/logs/__init__.py new file mode 100644 index 0000000..0407efe --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/logs/__init__.py @@ -0,0 +1,45 @@ +# Copyright 2005-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""Check and clean old logs in the PORT_LOGDIR. +""" + + +module_spec = { + 'name': 'logs', + 'description': __doc__, + 'provides':{ + 'module1': { + 'name': "logs", + 'class': "CleanLogs", + 'description': __doc__, + 'functions': ['check','clean'], + 'func_desc': { + 'clean': { + "short": "-C", "long": "--clean", + "help": "Cleans out logs more than 7 days old (cleanlogs only)" + \ + " module-options: -t, -p", + 'status': "Cleaning %s", + 'action': 'store_true', + 'func': 'clean', + }, + 'time': { + "short": "-t", "long": "--time", + "help": "(cleanlogs only): -t, --time Delete logs older than NUM of days", + 'status': "", + 'type': int, + 'dest': 'NUM', + 'func': 'clean' + }, + 'pretend': { + "short": "-p", "long": "--pretend", + "help": "(cleanlogs only): -p, --pretend Output logs that would be deleted", + 'status': "", + 'action': 'store_true', + 'dest': 'pretend', + 'func': 'clean' + } + } + } + } + } diff --git a/portage_with_autodep/pym/portage/emaint/modules/logs/logs.py b/portage_with_autodep/pym/portage/emaint/modules/logs/logs.py new file mode 100644 index 0000000..fe65cf5 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/logs/logs.py @@ -0,0 +1,103 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.util import shlex_split, varexpand + +## default clean command from make.globals +## PORT_LOGDIR_CLEAN = 'find "${PORT_LOGDIR}" -type f ! -name "summary.log*" -mtime +7 -delete' + +class CleanLogs(object): + + short_desc = "Clean PORT_LOGDIR logs" + + def name(): + return "logs" + name = staticmethod(name) + + + def can_progressbar(self, func): + return False + + + def check(self, **kwargs): + if kwargs: + options = kwargs.get('options', None) + if options: + options['pretend'] = True + return self.clean(**kwargs) + + + def clean(self, **kwargs): + """Log directory cleaning function + + @param **kwargs: optional dictionary of values used in this function are: + settings: portage settings instance: defaults to portage.settings + "PORT_LOGDIR": directory to clean + "PORT_LOGDIR_CLEAN": command for cleaning the logs. + options: dict: + 'NUM': int: number of days + 'pretend': boolean + """ + messages = [] + num_of_days = None + pretend = False + if kwargs: + # convuluted, I know, but portage.settings does not exist in + # kwargs.get() when called from _emerge.main.clean_logs() + settings = kwargs.get('settings', None) + if not settings: + settings = portage.settings + options = kwargs.get('options', None) + if options: + num_of_days = options.get('NUM', None) + pretend = options.get('pretend', False) + + clean_cmd = settings.get("PORT_LOGDIR_CLEAN") + if clean_cmd: + clean_cmd = shlex_split(clean_cmd) + if '-mtime' in clean_cmd and num_of_days is not None: + if num_of_days == 0: + i = clean_cmd.index('-mtime') + clean_cmd.remove('-mtime') + clean_cmd.pop(i) + else: + clean_cmd[clean_cmd.index('-mtime') +1] = \ + '+%s' % str(num_of_days) + if pretend: + if "-delete" in clean_cmd: + clean_cmd.remove("-delete") + + if not clean_cmd: + return [] + rval = self._clean_logs(clean_cmd, settings) + messages += self._convert_errors(rval) + return messages + + + @staticmethod + def _clean_logs(clean_cmd, settings): + logdir = settings.get("PORT_LOGDIR") + if logdir is None or not os.path.isdir(logdir): + return + + variables = {"PORT_LOGDIR" : logdir} + cmd = [varexpand(x, mydict=variables) for x in clean_cmd] + + try: + rval = portage.process.spawn(cmd, env=os.environ) + except portage.exception.CommandNotFound: + rval = 127 + return rval + + + @staticmethod + def _convert_errors(rval): + msg = [] + if rval != os.EX_OK: + msg.append("PORT_LOGDIR_CLEAN command returned %s" + % ("%d" % rval if rval else "None")) + msg.append("See the make.conf(5) man page for " + "PORT_LOGDIR_CLEAN usage instructions.") + return msg diff --git a/portage_with_autodep/pym/portage/emaint/modules/move/__init__.py b/portage_with_autodep/pym/portage/emaint/modules/move/__init__.py new file mode 100644 index 0000000..d31d7b3 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/move/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""Perform package move updates for installed and binary packages. +""" + + +module_spec = { + 'name': 'move', + 'description': __doc__, + 'provides':{ + 'module1': { + 'name': "moveinst", + 'class': "MoveInstalled", + 'description': __doc__, + 'options': ['check', 'fix'], + 'functions': ['check', 'fix'], + 'func_desc': { + } + }, + 'module2':{ + 'name': "movebin", + 'class': "MoveBinary", + 'description': "Perform package move updates for binary packages", + 'functions': ['check', 'fix'], + 'func_desc': { + } + } + } + } diff --git a/portage_with_autodep/pym/portage/emaint/modules/move/move.py b/portage_with_autodep/pym/portage/emaint/modules/move/move.py new file mode 100644 index 0000000..ef674d4 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/move/move.py @@ -0,0 +1,180 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.exception import InvalidData +from _emerge.Package import Package +from portage.versions import _pkg_str + +class MoveHandler(object): + + def __init__(self, tree, porttree): + self._tree = tree + self._portdb = porttree.dbapi + self._update_keys = Package._dep_keys + ("PROVIDE",) + self._master_repo = \ + self._portdb.getRepositoryName(self._portdb.porttree_root) + + def _grab_global_updates(self): + from portage.update import grab_updates, parse_updates + retupdates = {} + errors = [] + + for repo_name in self._portdb.getRepositories(): + repo = self._portdb.getRepositoryPath(repo_name) + updpath = os.path.join(repo, "profiles", "updates") + if not os.path.isdir(updpath): + continue + + try: + rawupdates = grab_updates(updpath) + except portage.exception.DirectoryNotFound: + rawupdates = [] + upd_commands = [] + for mykey, mystat, mycontent in rawupdates: + commands, errors = parse_updates(mycontent) + upd_commands.extend(commands) + errors.extend(errors) + retupdates[repo_name] = upd_commands + + if self._master_repo in retupdates: + retupdates['DEFAULT'] = retupdates[self._master_repo] + + return retupdates, errors + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + allupdates, errors = self._grab_global_updates() + # Matching packages and moving them is relatively fast, so the + # progress bar is updated in indeterminate mode. + match = self._tree.dbapi.match + aux_get = self._tree.dbapi.aux_get + pkg_str = self._tree.dbapi._pkg_str + settings = self._tree.dbapi.settings + if onProgress: + onProgress(0, 0) + for repo, updates in allupdates.items(): + if repo == 'DEFAULT': + continue + if not updates: + continue + + def repo_match(repository): + return repository == repo or \ + (repo == self._master_repo and \ + repository not in allupdates) + + for i, update_cmd in enumerate(updates): + if update_cmd[0] == "move": + origcp, newcp = update_cmd[1:] + for cpv in match(origcp): + try: + cpv = pkg_str(cpv, origcp.repo) + except (KeyError, InvalidData): + continue + if repo_match(cpv.repo): + errors.append("'%s' moved to '%s'" % (cpv, newcp)) + elif update_cmd[0] == "slotmove": + pkg, origslot, newslot = update_cmd[1:] + atom = pkg.with_slot(origslot) + for cpv in match(atom): + try: + cpv = pkg_str(cpv, atom.repo) + except (KeyError, InvalidData): + continue + if repo_match(cpv.repo): + errors.append("'%s' slot moved from '%s' to '%s'" % \ + (cpv, origslot, newslot)) + if onProgress: + onProgress(0, 0) + + # Searching for updates in all the metadata is relatively slow, so this + # is where the progress bar comes out of indeterminate mode. + cpv_all = self._tree.dbapi.cpv_all() + cpv_all.sort() + maxval = len(cpv_all) + meta_keys = self._update_keys + self._portdb._pkg_str_aux_keys + if onProgress: + onProgress(maxval, 0) + for i, cpv in enumerate(cpv_all): + try: + metadata = dict(zip(meta_keys, aux_get(cpv, meta_keys))) + except KeyError: + continue + try: + pkg = _pkg_str(cpv, metadata=metadata, settings=settings) + except InvalidData: + continue + metadata = dict((k, metadata[k]) for k in self._update_keys) + try: + updates = allupdates[pkg.repo] + except KeyError: + try: + updates = allupdates['DEFAULT'] + except KeyError: + continue + if not updates: + continue + metadata_updates = \ + portage.update_dbentries(updates, metadata, parent=pkg) + if metadata_updates: + errors.append("'%s' has outdated metadata" % cpv) + if onProgress: + onProgress(maxval, i+1) + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + allupdates, errors = self._grab_global_updates() + # Matching packages and moving them is relatively fast, so the + # progress bar is updated in indeterminate mode. + move = self._tree.dbapi.move_ent + slotmove = self._tree.dbapi.move_slot_ent + if onProgress: + onProgress(0, 0) + for repo, updates in allupdates.items(): + if repo == 'DEFAULT': + continue + if not updates: + continue + + def repo_match(repository): + return repository == repo or \ + (repo == self._master_repo and \ + repository not in allupdates) + + for i, update_cmd in enumerate(updates): + if update_cmd[0] == "move": + move(update_cmd, repo_match=repo_match) + elif update_cmd[0] == "slotmove": + slotmove(update_cmd, repo_match=repo_match) + if onProgress: + onProgress(0, 0) + + # Searching for updates in all the metadata is relatively slow, so this + # is where the progress bar comes out of indeterminate mode. + self._tree.dbapi.update_ents(allupdates, onProgress=onProgress) + return errors + +class MoveInstalled(MoveHandler): + + short_desc = "Perform package move updates for installed packages" + + def name(): + return "moveinst" + name = staticmethod(name) + def __init__(self): + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["vartree"], portage.db[eroot]["porttree"]) + +class MoveBinary(MoveHandler): + + short_desc = "Perform package move updates for binary packages" + + def name(): + return "movebin" + name = staticmethod(name) + def __init__(self): + eroot = portage.settings['EROOT'] + MoveHandler.__init__(self, portage.db[eroot]["bintree"], portage.db[eroot]['porttree']) diff --git a/portage_with_autodep/pym/portage/emaint/modules/resume/__init__.py b/portage_with_autodep/pym/portage/emaint/modules/resume/__init__.py new file mode 100644 index 0000000..965e8f9 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/resume/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""Check and fix problems in the resume and/or resume_backup files. +""" + + +module_spec = { + 'name': 'resume', + 'description': __doc__, + 'provides':{ + 'module1': { + 'name': "cleanresume", + 'class': "CleanResume", + 'description': "Discard emerge --resume merge lists", + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/portage_with_autodep/pym/portage/emaint/modules/resume/resume.py b/portage_with_autodep/pym/portage/emaint/modules/resume/resume.py new file mode 100644 index 0000000..1bada52 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/resume/resume.py @@ -0,0 +1,58 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage + + +class CleanResume(object): + + short_desc = "Discard emerge --resume merge lists" + + def name(): + return "cleanresume" + name = staticmethod(name) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + messages = [] + mtimedb = portage.mtimedb + resume_keys = ("resume", "resume_backup") + maxval = len(resume_keys) + if onProgress: + onProgress(maxval, 0) + for i, k in enumerate(resume_keys): + try: + d = mtimedb.get(k) + if d is None: + continue + if not isinstance(d, dict): + messages.append("unrecognized resume list: '%s'" % k) + continue + mergelist = d.get("mergelist") + if mergelist is None or not hasattr(mergelist, "__len__"): + messages.append("unrecognized resume list: '%s'" % k) + continue + messages.append("resume list '%s' contains %d packages" % \ + (k, len(mergelist))) + finally: + if onProgress: + onProgress(maxval, i+1) + return messages + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + delete_count = 0 + mtimedb = portage.mtimedb + resume_keys = ("resume", "resume_backup") + maxval = len(resume_keys) + if onProgress: + onProgress(maxval, 0) + for i, k in enumerate(resume_keys): + try: + if mtimedb.pop(k, None) is not None: + delete_count += 1 + finally: + if onProgress: + onProgress(maxval, i+1) + if delete_count: + mtimedb.commit() diff --git a/portage_with_autodep/pym/portage/emaint/modules/world/__init__.py b/portage_with_autodep/pym/portage/emaint/modules/world/__init__.py new file mode 100644 index 0000000..3f62270 --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/world/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +"""Check and fix problems in the world file. +""" + + +module_spec = { + 'name': 'world', + 'description': __doc__, + 'provides':{ + 'module1':{ + 'name': "world", + 'class': "WorldHandler", + 'description': __doc__, + 'functions': ['check', 'fix'], + 'func_desc': {} + } + } + } diff --git a/portage_with_autodep/pym/portage/emaint/modules/world/world.py b/portage_with_autodep/pym/portage/emaint/modules/world/world.py new file mode 100644 index 0000000..2c9dbff --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/modules/world/world.py @@ -0,0 +1,89 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os + + +class WorldHandler(object): + + short_desc = "Fix problems in the world file" + + def name(): + return "world" + name = staticmethod(name) + + def __init__(self): + self.invalid = [] + self.not_installed = [] + self.okay = [] + from portage._sets import load_default_config + setconfig = load_default_config(portage.settings, + portage.db[portage.settings['EROOT']]) + self._sets = setconfig.getSets() + + def _check_world(self, onProgress): + eroot = portage.settings['EROOT'] + self.world_file = os.path.join(eroot, portage.const.WORLD_FILE) + self.found = os.access(self.world_file, os.R_OK) + vardb = portage.db[eroot]["vartree"].dbapi + + from portage._sets import SETPREFIX + sets = self._sets + world_atoms = list(sets["selected"]) + maxval = len(world_atoms) + if onProgress: + onProgress(maxval, 0) + for i, atom in enumerate(world_atoms): + if not isinstance(atom, portage.dep.Atom): + if atom.startswith(SETPREFIX): + s = atom[len(SETPREFIX):] + if s in sets: + self.okay.append(atom) + else: + self.not_installed.append(atom) + else: + self.invalid.append(atom) + if onProgress: + onProgress(maxval, i+1) + continue + okay = True + if not vardb.match(atom): + self.not_installed.append(atom) + okay = False + if okay: + self.okay.append(atom) + if onProgress: + onProgress(maxval, i+1) + + def check(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + self._check_world(onProgress) + errors = [] + if self.found: + errors += ["'%s' is not a valid atom" % x for x in self.invalid] + errors += ["'%s' is not installed" % x for x in self.not_installed] + else: + errors.append(self.world_file + " could not be opened for reading") + return errors + + def fix(self, **kwargs): + onProgress = kwargs.get('onProgress', None) + world_set = self._sets["selected"] + world_set.lock() + try: + world_set.load() # maybe it's changed on disk + before = set(world_set) + self._check_world(onProgress) + after = set(self.okay) + errors = [] + if before != after: + try: + world_set.replace(self.okay) + except portage.exception.PortageException: + errors.append("%s could not be opened for writing" % \ + self.world_file) + return errors + finally: + world_set.unlock() + diff --git a/portage_with_autodep/pym/portage/emaint/progress.py b/portage_with_autodep/pym/portage/emaint/progress.py new file mode 100644 index 0000000..e43c2af --- /dev/null +++ b/portage_with_autodep/pym/portage/emaint/progress.py @@ -0,0 +1,61 @@ +# Copyright 2005-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import time +import signal + +import portage + + +class ProgressHandler(object): + def __init__(self): + self.reset() + + def reset(self): + self.curval = 0 + self.maxval = 0 + self.last_update = 0 + self.min_display_latency = 0.2 + + def onProgress(self, maxval, curval): + self.maxval = maxval + self.curval = curval + cur_time = time.time() + if cur_time - self.last_update >= self.min_display_latency: + self.last_update = cur_time + self.display() + + def display(self): + raise NotImplementedError(self) + + +class ProgressBar(ProgressHandler): + """Class to set up and return a Progress Bar""" + + def __init__(self, isatty, **kwargs): + self.isatty = isatty + self.kwargs = kwargs + ProgressHandler.__init__(self) + self.progressBar = None + + def start(self): + if self.isatty: + self.progressBar = portage.output.TermProgressBar(**self.kwargs) + signal.signal(signal.SIGWINCH, self.sigwinch_handler) + else: + self.onProgress = None + return self.onProgress + + def set_label(self, _label): + self.kwargs['label'] = _label + + def display(self): + self.progressBar.set(self.curval, self.maxval) + + def sigwinch_handler(self, signum, frame): + lines, self.progressBar.term_columns = \ + portage.output.get_term_size() + + def stop(self): + signal.signal(signal.SIGWINCH, signal.SIG_DFL) + diff --git a/portage_with_autodep/pym/portage/env/__init__.pyo b/portage_with_autodep/pym/portage/env/__init__.pyo Binary files differnew file mode 100644 index 0000000..846aea3 --- /dev/null +++ b/portage_with_autodep/pym/portage/env/__init__.pyo diff --git a/portage_with_autodep/pym/portage/env/config.pyo b/portage_with_autodep/pym/portage/env/config.pyo Binary files differnew file mode 100644 index 0000000..13c2e86 --- /dev/null +++ b/portage_with_autodep/pym/portage/env/config.pyo diff --git a/portage_with_autodep/pym/portage/env/loaders.py b/portage_with_autodep/pym/portage/env/loaders.py index b540fbb..372bc12 100644 --- a/portage_with_autodep/pym/portage/env/loaders.py +++ b/portage_with_autodep/pym/portage/env/loaders.py @@ -40,7 +40,7 @@ def RecursiveFileLoader(filename): @param filename: name of a file/directory to traverse @rtype: list - @returns: List of files to process + @return: List of files to process """ try: @@ -139,7 +139,7 @@ class FileLoader(DataLoader): load all files in self.fname @type: Boolean @rtype: tuple - @returns: + @return: Returns (data,errors), both may be empty dicts or populated. """ data = {} diff --git a/portage_with_autodep/pym/portage/env/loaders.pyo b/portage_with_autodep/pym/portage/env/loaders.pyo Binary files differnew file mode 100644 index 0000000..2622a9f --- /dev/null +++ b/portage_with_autodep/pym/portage/env/loaders.pyo diff --git a/portage_with_autodep/pym/portage/env/validators.pyo b/portage_with_autodep/pym/portage/env/validators.pyo Binary files differnew file mode 100644 index 0000000..cd18adb --- /dev/null +++ b/portage_with_autodep/pym/portage/env/validators.pyo diff --git a/portage_with_autodep/pym/portage/exception.py b/portage_with_autodep/pym/portage/exception.py index 7891120..5ccd750 100644 --- a/portage_with_autodep/pym/portage/exception.py +++ b/portage_with_autodep/pym/portage/exception.py @@ -78,6 +78,10 @@ class OperationNotPermitted(PortageException): from errno import EPERM as errno """An operation was not permitted operating system""" +class OperationNotSupported(PortageException): + from errno import EOPNOTSUPP as errno + """Operation not supported""" + class PermissionDenied(PortageException): from errno import EACCES as errno """Permission denied""" diff --git a/portage_with_autodep/pym/portage/exception.pyo b/portage_with_autodep/pym/portage/exception.pyo Binary files differnew file mode 100644 index 0000000..3a60e7c --- /dev/null +++ b/portage_with_autodep/pym/portage/exception.pyo diff --git a/portage_with_autodep/pym/portage/getbinpkg.py b/portage_with_autodep/pym/portage/getbinpkg.py index a511f51..212f788 100644 --- a/portage_with_autodep/pym/portage/getbinpkg.py +++ b/portage_with_autodep/pym/portage/getbinpkg.py @@ -1,5 +1,5 @@ # getbinpkg.py -- Portage binary-package helper functions -# Copyright 2003-2011 Gentoo Foundation +# Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from portage.output import colorize @@ -8,7 +8,10 @@ from portage.localization import _ import portage from portage import os from portage import _encodings +from portage import _unicode_decode from portage import _unicode_encode +from portage.package.ebuild.fetch import _hide_url_passwd +from _emerge.Package import _all_metadata_keys import sys import socket @@ -65,8 +68,15 @@ def make_metadata_dict(data): myid,myglob = data mydict = {} - for x in portage.xpak.getindex_mem(myid): - mydict[x] = portage.xpak.getitem(data,x) + for k_bytes in portage.xpak.getindex_mem(myid): + k = _unicode_decode(k_bytes, + encoding=_encodings['repo.content'], errors='replace') + if k not in _all_metadata_keys and \ + k != "CATEGORY": + continue + v = _unicode_decode(portage.xpak.getitem(data, k_bytes), + encoding=_encodings['repo.content'], errors='replace') + mydict[k] = v return mydict @@ -149,11 +159,16 @@ def create_conn(baseurl,conn=None): http_headers = {} http_params = {} if username and password: + try: + encodebytes = base64.encodebytes + except AttributeError: + # Python 2 + encodebytes = base64.encodestring http_headers = { - "Authorization": "Basic %s" % - base64.encodestring("%s:%s" % (username, password)).replace( - "\012", - "" + b"Authorization": "Basic %s" % \ + encodebytes(_unicode_encode("%s:%s" % (username, password))).replace( + b"\012", + b"" ), } @@ -354,7 +369,7 @@ def dir_get_list(baseurl,conn=None): if page: parser = ParseLinks() - parser.feed(page) + parser.feed(_unicode_decode(page)) del page listing = parser.get_anchors() else: @@ -542,7 +557,7 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= out.write(_("Loaded metadata pickle.\n")) out.flush() metadatafile.close() - except (IOError, OSError, EOFError, ValueError, pickle.UnpicklingError): + except (AttributeError, EOFError, EnvironmentError, ValueError, pickle.UnpicklingError): metadata = {} if baseurl not in metadata: metadata[baseurl]={} @@ -564,7 +579,8 @@ def dir_get_metadata(baseurl, conn=None, chunk_size=3000, verbose=1, usingcache= try: filelist = dir_get_list(baseurl, conn) except portage.exception.PortageException as e: - sys.stderr.write(_("!!! Error connecting to '%s'.\n") % baseurl) + sys.stderr.write(_("!!! Error connecting to '%s'.\n") % + _hide_url_passwd(baseurl)) sys.stderr.write("!!! %s\n" % str(e)) del e return metadata[baseurl]["data"] diff --git a/portage_with_autodep/pym/portage/getbinpkg.pyo b/portage_with_autodep/pym/portage/getbinpkg.pyo Binary files differnew file mode 100644 index 0000000..53ec2e9 --- /dev/null +++ b/portage_with_autodep/pym/portage/getbinpkg.pyo diff --git a/portage_with_autodep/pym/portage/glsa.py b/portage_with_autodep/pym/portage/glsa.py index a784d14..1857695 100644 --- a/portage_with_autodep/pym/portage/glsa.py +++ b/portage_with_autodep/pym/portage/glsa.py @@ -1,4 +1,4 @@ -# Copyright 2003-2011 Gentoo Foundation +# Copyright 2003-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import absolute_import @@ -17,7 +17,7 @@ from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode -from portage.versions import pkgsplit, catpkgsplit, pkgcmp, best +from portage.versions import pkgsplit, vercmp, best from portage.util import grabfile from portage.const import CACHE_PATH from portage.localization import _ @@ -372,17 +372,14 @@ def getMinUpgrade(vulnerableList, unaffectedList, portdbapi, vardbapi, minimize= for u in unaffectedList: mylist = match(u, portdbapi, match_type="match-all") for c in mylist: - c_pv = catpkgsplit(c) - i_pv = catpkgsplit(best(v_installed)) - if pkgcmp(c_pv[1:], i_pv[1:]) > 0 \ + i = best(v_installed) + if vercmp(c.version, i.version) > 0 \ and (rValue == None \ or not match("="+rValue, portdbapi) \ - or (minimize ^ (pkgcmp(c_pv[1:], catpkgsplit(rValue)[1:]) > 0)) \ + or (minimize ^ (vercmp(c.version, rValue.version) > 0)) \ and match("="+c, portdbapi)) \ and portdbapi.aux_get(c, ["SLOT"]) == vardbapi.aux_get(best(v_installed), ["SLOT"]): - rValue = c_pv[0]+"/"+c_pv[1]+"-"+c_pv[2] - if c_pv[3] != "r0": # we don't like -r0 for display - rValue += "-"+c_pv[3] + rValue = c return rValue def format_date(datestr): @@ -488,7 +485,7 @@ class Glsa: @type myfile: String @param myfile: Filename to grab the XML data from @rtype: None - @returns: None + @return: None """ self.DOM = xml.dom.minidom.parse(myfile) if not self.DOM.doctype: @@ -634,7 +631,7 @@ class Glsa: architectures. @rtype: Boolean - @returns: True if the system is affected, False if not + @return: True if the system is affected, False if not """ rValue = False for k in self.packages: @@ -654,7 +651,7 @@ class Glsa: GLSA was already applied. @rtype: Boolean - @returns: True if the GLSA was applied, False if not + @return: True if the GLSA was applied, False if not """ return (self.nr in get_applied_glsas(self.config)) @@ -665,7 +662,7 @@ class Glsa: applied or on explicit user request. @rtype: None - @returns: None + @return: None """ if not self.isApplied(): checkfile = io.open( diff --git a/portage_with_autodep/pym/portage/glsa.pyo b/portage_with_autodep/pym/portage/glsa.pyo Binary files differnew file mode 100644 index 0000000..65162f1 --- /dev/null +++ b/portage_with_autodep/pym/portage/glsa.pyo diff --git a/portage_with_autodep/pym/portage/localization.pyo b/portage_with_autodep/pym/portage/localization.pyo Binary files differnew file mode 100644 index 0000000..e992e3a --- /dev/null +++ b/portage_with_autodep/pym/portage/localization.pyo diff --git a/portage_with_autodep/pym/portage/locks.py b/portage_with_autodep/pym/portage/locks.py index 9ed1d6a..59fbc6e 100644 --- a/portage_with_autodep/pym/portage/locks.py +++ b/portage_with_autodep/pym/portage/locks.py @@ -1,5 +1,5 @@ # portage: Lock management code -# Copyright 2004-2010 Gentoo Foundation +# Copyright 2004-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \ @@ -8,13 +8,13 @@ __all__ = ["lockdir", "unlockdir", "lockfile", "unlockfile", \ import errno import fcntl -import stat +import platform import sys import time +import warnings import portage -from portage import os -from portage.const import PORTAGE_BIN_PATH +from portage import os, _encodings, _unicode_decode from portage.exception import DirectoryNotFound, FileNotFound, \ InvalidData, TryAgain, OperationNotPermitted, PermissionDenied from portage.data import portage_gid @@ -25,12 +25,30 @@ if sys.hexversion >= 0x3000000: basestring = str HARDLINK_FD = -2 +_HARDLINK_POLL_LATENCY = 3 # seconds _default_lock_fn = fcntl.lockf +if platform.python_implementation() == 'PyPy': + # workaround for https://bugs.pypy.org/issue747 + _default_lock_fn = fcntl.flock + # Used by emerge in order to disable the "waiting for lock" message # so that it doesn't interfere with the status display. _quiet = False + +_open_fds = set() + +def _close_fds(): + """ + This is intended to be called after a fork, in order to close file + descriptors for locks held by the parent process. This can be called + safely after a fork without exec, unlike the _setup_pipes close_fds + behavior. + """ + while _open_fds: + os.close(_open_fds.pop()) + def lockdir(mydir, flags=0): return lockfile(mydir, wantnewlockfile=1, flags=flags) def unlockdir(mylock): @@ -46,19 +64,31 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, if not mypath: raise InvalidData(_("Empty path given")) + # Support for file object or integer file descriptor parameters is + # deprecated due to ambiguity in whether or not it's safe to close + # the file descriptor, making it prone to "Bad file descriptor" errors + # or file descriptor leaks. if isinstance(mypath, basestring) and mypath[-1] == '/': mypath = mypath[:-1] + lockfilename_path = mypath if hasattr(mypath, 'fileno'): + warnings.warn("portage.locks.lockfile() support for " + "file object parameters is deprecated. Use a file path instead.", + DeprecationWarning, stacklevel=2) + lockfilename_path = getattr(mypath, 'name', None) mypath = mypath.fileno() if isinstance(mypath, int): + warnings.warn("portage.locks.lockfile() support for integer file " + "descriptor parameters is deprecated. Use a file path instead.", + DeprecationWarning, stacklevel=2) lockfilename = mypath wantnewlockfile = 0 unlinkfile = 0 elif wantnewlockfile: base, tail = os.path.split(mypath) lockfilename = os.path.join(base, "." + tail + ".portage_lockfile") - del base, tail + lockfilename_path = lockfilename unlinkfile = 1 else: lockfilename = mypath @@ -112,6 +142,8 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, # we're waiting on lockfile and use a blocking attempt. locking_method = _default_lock_fn try: + if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: + raise IOError(errno.ENOSYS, "Function not implemented") locking_method(myfd, fcntl.LOCK_EX|fcntl.LOCK_NB) except IOError as e: if not hasattr(e, "errno"): @@ -143,20 +175,22 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, raise if out is not None: out.eend(os.EX_OK) - elif e.errno == errno.ENOLCK: + elif e.errno in (errno.ENOSYS, errno.ENOLCK): # We're not allowed to lock on this FS. - os.close(myfd) - link_success = False - if lockfilename == str(lockfilename): - if wantnewlockfile: - try: - if os.stat(lockfilename)[stat.ST_NLINK] == 1: - os.unlink(lockfilename) - except OSError: - pass - link_success = hardlink_lockfile(lockfilename) + if not isinstance(lockfilename, int): + # If a file object was passed in, it's not safe + # to close the file descriptor because it may + # still be in use. + os.close(myfd) + lockfilename_path = _unicode_decode(lockfilename_path, + encoding=_encodings['fs'], errors='strict') + if not isinstance(lockfilename_path, basestring): + raise + link_success = hardlink_lockfile(lockfilename_path, + waiting_msg=waiting_msg, flags=flags) if not link_success: raise + lockfilename = lockfilename_path locking_method = None myfd = HARDLINK_FD else: @@ -172,6 +206,9 @@ def lockfile(mypath, wantnewlockfile=0, unlinkfile=0, mypath, wantnewlockfile=wantnewlockfile, unlinkfile=unlinkfile, waiting_msg=waiting_msg, flags=flags) + if myfd != HARDLINK_FD: + _open_fds.add(myfd) + writemsg(str((lockfilename,myfd,unlinkfile))+"\n",1) return (lockfilename,myfd,unlinkfile,locking_method) @@ -203,7 +240,7 @@ def unlockfile(mytuple): raise InvalidData if(myfd == HARDLINK_FD): - unhardlink_lockfile(lockfilename) + unhardlink_lockfile(lockfilename, unlinkfile=unlinkfile) return True # myfd may be None here due to myfd = mypath in lockfile() @@ -212,6 +249,7 @@ def unlockfile(mytuple): writemsg(_("lockfile does not exist '%s'\n") % lockfilename,1) if myfd is not None: os.close(myfd) + _open_fds.remove(myfd) return False try: @@ -222,6 +260,7 @@ def unlockfile(mytuple): except OSError: if isinstance(lockfilename, basestring): os.close(myfd) + _open_fds.remove(myfd) raise IOError(_("Failed to unlock file '%s'\n") % lockfilename) try: @@ -243,6 +282,7 @@ def unlockfile(mytuple): else: writemsg(_("lockfile does not exist '%s'\n") % lockfilename, 1) os.close(myfd) + _open_fds.remove(myfd) return False except SystemExit: raise @@ -255,6 +295,7 @@ def unlockfile(mytuple): # open fd closed automatically on them. if isinstance(lockfilename, basestring): os.close(myfd) + _open_fds.remove(myfd) return True @@ -262,65 +303,148 @@ def unlockfile(mytuple): def hardlock_name(path): - return path+".hardlock-"+os.uname()[1]+"-"+str(os.getpid()) + base, tail = os.path.split(path) + return os.path.join(base, ".%s.hardlock-%s-%s" % + (tail, os.uname()[1], os.getpid())) def hardlink_is_mine(link,lock): try: - return os.stat(link).st_nlink == 2 + lock_st = os.stat(lock) + if lock_st.st_nlink == 2: + link_st = os.stat(link) + return lock_st.st_ino == link_st.st_ino and \ + lock_st.st_dev == link_st.st_dev except OSError: - return False + pass + return False -def hardlink_lockfile(lockfilename, max_wait=14400): +def hardlink_lockfile(lockfilename, max_wait=DeprecationWarning, + waiting_msg=None, flags=0): """Does the NFS, hardlink shuffle to ensure locking on the disk. - We create a PRIVATE lockfile, that is just a placeholder on the disk. - Then we HARDLINK the real lockfile to that private file. + We create a PRIVATE hardlink to the real lockfile, that is just a + placeholder on the disk. If our file can 2 references, then we have the lock. :) Otherwise we lather, rise, and repeat. - We default to a 4 hour timeout. """ - start_time = time.time() + if max_wait is not DeprecationWarning: + warnings.warn("The 'max_wait' parameter of " + "portage.locks.hardlink_lockfile() is now unused. Use " + "flags=os.O_NONBLOCK instead.", + DeprecationWarning, stacklevel=2) + + global _quiet + out = None + displayed_waiting_msg = False + preexisting = os.path.exists(lockfilename) myhardlock = hardlock_name(lockfilename) - reported_waiting = False - - while(time.time() < (start_time + max_wait)): - # We only need it to exist. - myfd = os.open(myhardlock, os.O_CREAT|os.O_RDWR,0o660) - os.close(myfd) - - if not os.path.exists(myhardlock): - raise FileNotFound( - _("Created lockfile is missing: %(filename)s") % \ - {"filename" : myhardlock}) - try: - res = os.link(myhardlock, lockfilename) - except OSError: + # myhardlock must not exist prior to our link() call, and we can + # safely unlink it since its file name is unique to our PID + try: + os.unlink(myhardlock) + except OSError as e: + if e.errno in (errno.ENOENT, errno.ESTALE): pass + else: + func_call = "unlink('%s')" % myhardlock + if e.errno == OperationNotPermitted.errno: + raise OperationNotPermitted(func_call) + elif e.errno == PermissionDenied.errno: + raise PermissionDenied(func_call) + else: + raise - if hardlink_is_mine(myhardlock, lockfilename): - # We have the lock. - if reported_waiting: - writemsg("\n", noiselevel=-1) - return True - - if reported_waiting: - writemsg(".", noiselevel=-1) + while True: + # create lockfilename if it doesn't exist yet + try: + myfd = os.open(lockfilename, os.O_CREAT|os.O_RDWR, 0o660) + except OSError as e: + func_call = "open('%s')" % lockfilename + if e.errno == OperationNotPermitted.errno: + raise OperationNotPermitted(func_call) + elif e.errno == PermissionDenied.errno: + raise PermissionDenied(func_call) + else: + raise else: - reported_waiting = True - msg = _("\nWaiting on (hardlink) lockfile: (one '.' per 3 seconds)\n" - "%(bin_path)s/clean_locks can fix stuck locks.\n" - "Lockfile: %(lockfilename)s\n") % \ - {"bin_path": PORTAGE_BIN_PATH, "lockfilename": lockfilename} - writemsg(msg, noiselevel=-1) - time.sleep(3) - - os.unlink(myhardlock) - return False + myfd_st = None + try: + myfd_st = os.fstat(myfd) + if not preexisting: + # Don't chown the file if it is preexisting, since we + # want to preserve existing permissions in that case. + if myfd_st.st_gid != portage_gid: + os.fchown(myfd, -1, portage_gid) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + writemsg("%s: fchown('%s', -1, %d)\n" % \ + (e, lockfilename, portage_gid), noiselevel=-1) + writemsg(_("Cannot chown a lockfile: '%s'\n") % \ + lockfilename, noiselevel=-1) + writemsg(_("Group IDs of current user: %s\n") % \ + " ".join(str(n) for n in os.getgroups()), + noiselevel=-1) + else: + # another process has removed the file, so we'll have + # to create it again + continue + finally: + os.close(myfd) + + # If fstat shows more than one hardlink, then it's extremely + # unlikely that the following link call will result in a lock, + # so optimize away the wasteful link call and sleep or raise + # TryAgain. + if myfd_st is not None and myfd_st.st_nlink < 2: + try: + os.link(lockfilename, myhardlock) + except OSError as e: + func_call = "link('%s', '%s')" % (lockfilename, myhardlock) + if e.errno == OperationNotPermitted.errno: + raise OperationNotPermitted(func_call) + elif e.errno == PermissionDenied.errno: + raise PermissionDenied(func_call) + elif e.errno in (errno.ESTALE, errno.ENOENT): + # another process has removed the file, so we'll have + # to create it again + continue + else: + raise + else: + if hardlink_is_mine(myhardlock, lockfilename): + if out is not None: + out.eend(os.EX_OK) + break + + try: + os.unlink(myhardlock) + except OSError as e: + # This should not happen, since the file name of + # myhardlock is unique to our host and PID, + # and the above link() call succeeded. + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + raise FileNotFound(myhardlock) + + if flags & os.O_NONBLOCK: + raise TryAgain(lockfilename) + + if out is None and not _quiet: + out = portage.output.EOutput() + if out is not None and not displayed_waiting_msg: + displayed_waiting_msg = True + if waiting_msg is None: + waiting_msg = _("waiting for lock on %s\n") % lockfilename + out.ebegin(waiting_msg) + + time.sleep(_HARDLINK_POLL_LATENCY) + + return True -def unhardlink_lockfile(lockfilename): +def unhardlink_lockfile(lockfilename, unlinkfile=True): myhardlock = hardlock_name(lockfilename) - if hardlink_is_mine(myhardlock, lockfilename): + if unlinkfile and hardlink_is_mine(myhardlock, lockfilename): # Make sure not to touch lockfilename unless we really have a lock. try: os.unlink(lockfilename) @@ -344,7 +468,7 @@ def hardlock_cleanup(path, remove_all_locks=False): if os.path.isfile(path+"/"+x): parts = x.split(".hardlock-") if len(parts) == 2: - filename = parts[0] + filename = parts[0][1:] hostpid = parts[1].split("-") host = "-".join(hostpid[:-1]) pid = hostpid[-1] @@ -368,7 +492,7 @@ def hardlock_cleanup(path, remove_all_locks=False): remove_all_locks: for y in mylist[x]: for z in mylist[x][y]: - filename = path+"/"+x+".hardlock-"+y+"-"+z + filename = path+"/."+x+".hardlock-"+y+"-"+z if filename == mylockname: continue try: diff --git a/portage_with_autodep/pym/portage/locks.pyo b/portage_with_autodep/pym/portage/locks.pyo Binary files differnew file mode 100644 index 0000000..9c90a2f --- /dev/null +++ b/portage_with_autodep/pym/portage/locks.pyo diff --git a/portage_with_autodep/pym/portage/mail.py b/portage_with_autodep/pym/portage/mail.py index 17dfcaf..3fcadd2 100644 --- a/portage_with_autodep/pym/portage/mail.py +++ b/portage_with_autodep/pym/portage/mail.py @@ -40,8 +40,7 @@ else: def TextMessage(_text): from email.mime.text import MIMEText mimetext = MIMEText(_text) - if sys.hexversion >= 0x3000000: - mimetext.set_charset("UTF-8") + mimetext.set_charset("UTF-8") return mimetext def create_message(sender, recipient, subject, body, attachments=None): diff --git a/portage_with_autodep/pym/portage/mail.pyo b/portage_with_autodep/pym/portage/mail.pyo Binary files differnew file mode 100644 index 0000000..bc3a76d --- /dev/null +++ b/portage_with_autodep/pym/portage/mail.pyo diff --git a/portage_with_autodep/pym/portage/manifest.py b/portage_with_autodep/pym/portage/manifest.py index 13efab7..90324ee 100644 --- a/portage_with_autodep/pym/portage/manifest.py +++ b/portage_with_autodep/pym/portage/manifest.py @@ -1,8 +1,10 @@ -# Copyright 1999-2011 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno import io +import re +import warnings import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -17,8 +19,13 @@ from portage import _unicode_encode from portage.exception import DigestException, FileNotFound, \ InvalidDataType, MissingParameter, PermissionDenied, \ PortageException, PortagePackageException +from portage.const import (MANIFEST1_HASH_FUNCTIONS, MANIFEST2_HASH_DEFAULTS, + MANIFEST2_HASH_FUNCTIONS, MANIFEST2_IDENTIFIERS, MANIFEST2_REQUIRED_HASH) from portage.localization import _ +# Characters prohibited by repoman's file.name check. +_prohibited_filename_chars_re = re.compile(r'[^a-zA-Z0-9._\-+:]') + class FileNotInManifestException(PortageException): pass @@ -30,10 +37,14 @@ def manifest2AuxfileFilter(filename): for x in mysplit: if x[:1] == '.': return False + if _prohibited_filename_chars_re.search(x) is not None: + return False return not filename[:7] == 'digest-' def manifest2MiscfileFilter(filename): filename = filename.strip(os.sep) + if _prohibited_filename_chars_re.search(filename) is not None: + return False return not (filename in ["CVS", ".svn", "files", "Manifest"] or filename.endswith(".ebuild")) def guessManifestFileType(filename): @@ -49,9 +60,15 @@ def guessManifestFileType(filename): else: return "DIST" +def guessThinManifestFileType(filename): + type = guessManifestFileType(filename) + if type != "DIST": + return None + return "DIST" + def parseManifest2(mysplit): myentry = None - if len(mysplit) > 4 and mysplit[0] in portage.const.MANIFEST2_IDENTIFIERS: + if len(mysplit) > 4 and mysplit[0] in MANIFEST2_IDENTIFIERS: mytype = mysplit[0] myname = mysplit[1] try: @@ -93,25 +110,33 @@ class Manifest2Entry(ManifestEntry): class Manifest(object): parsers = (parseManifest2,) def __init__(self, pkgdir, distdir, fetchlist_dict=None, - manifest1_compat=False, from_scratch=False): - """ create new Manifest instance for package in pkgdir - and add compability entries for old portage versions if manifest1_compat == True. + manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, + allow_missing=False, allow_create=True, hashes=None): + """ Create new Manifest instance for package in pkgdir. Do not parse Manifest file if from_scratch == True (only for internal use) The fetchlist_dict parameter is required only for generation of - a Manifest (not needed for parsing and checking sums).""" + a Manifest (not needed for parsing and checking sums). + If thin is specified, then the manifest carries only info for + distfiles.""" + + if manifest1_compat is not DeprecationWarning: + warnings.warn("The manifest1_compat parameter of the " + "portage.manifest.Manifest constructor is deprecated.", + DeprecationWarning, stacklevel=2) + self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep self.fhashdict = {} self.hashes = set() - self.hashes.update(portage.const.MANIFEST2_HASH_FUNCTIONS) - if manifest1_compat: - raise NotImplementedError("manifest1 support has been removed") + + if hashes is None: + hashes = MANIFEST2_HASH_DEFAULTS + + self.hashes.update(hashes.intersection(MANIFEST2_HASH_FUNCTIONS)) self.hashes.difference_update(hashname for hashname in \ list(self.hashes) if hashname not in hashfunc_map) self.hashes.add("size") - if manifest1_compat: - raise NotImplementedError("manifest1 support has been removed") - self.hashes.add(portage.const.MANIFEST2_REQUIRED_HASH) - for t in portage.const.MANIFEST2_IDENTIFIERS: + self.hashes.add(MANIFEST2_REQUIRED_HASH) + for t in MANIFEST2_IDENTIFIERS: self.fhashdict[t] = {} if not from_scratch: self._read() @@ -120,7 +145,13 @@ class Manifest(object): else: self.fetchlist_dict = {} self.distdir = distdir - self.guessType = guessManifestFileType + self.thin = thin + if thin: + self.guessType = guessThinManifestFileType + else: + self.guessType = guessManifestFileType + self.allow_missing = allow_missing + self.allow_create = allow_create def getFullname(self): """ Returns the absolute path to the Manifest file for this instance """ @@ -129,7 +160,7 @@ class Manifest(object): def getDigests(self): """ Compability function for old digest/manifest code, returns dict of filename:{hashfunction:hashvalue} """ rval = {} - for t in portage.const.MANIFEST2_IDENTIFIERS: + for t in MANIFEST2_IDENTIFIERS: rval.update(self.fhashdict[t]) return rval @@ -200,7 +231,7 @@ class Manifest(object): return myhashdict def _createManifestEntries(self): - valid_hashes = set(portage.const.MANIFEST2_HASH_FUNCTIONS) + valid_hashes = set(MANIFEST2_HASH_FUNCTIONS) valid_hashes.add('size') mytypes = list(self.fhashdict) mytypes.sort() @@ -218,16 +249,19 @@ class Manifest(object): def checkIntegrity(self): for t in self.fhashdict: for f in self.fhashdict[t]: - if portage.const.MANIFEST2_REQUIRED_HASH not in self.fhashdict[t][f]: - raise MissingParameter(_("Missing %s checksum: %s %s") % (portage.const.MANIFEST2_REQUIRED_HASH, t, f)) + if MANIFEST2_REQUIRED_HASH not in self.fhashdict[t][f]: + raise MissingParameter(_("Missing %s checksum: %s %s") % + (MANIFEST2_REQUIRED_HASH, t, f)) def write(self, sign=False, force=False): """ Write Manifest instance to disk, optionally signing it """ + if not self.allow_create: + return self.checkIntegrity() try: myentries = list(self._createManifestEntries()) update_manifest = True - if not force: + if myentries and not force: try: f = io.open(_unicode_encode(self.getFullname(), encoding=_encodings['fs'], errors='strict'), @@ -246,9 +280,24 @@ class Manifest(object): pass else: raise + if update_manifest: - write_atomic(self.getFullname(), - "".join("%s\n" % str(myentry) for myentry in myentries)) + if myentries or not (self.thin or self.allow_missing): + # If myentries is empty, don't write an empty manifest + # when thin or allow_missing is enabled. Except for + # thin manifests with no DIST entries, myentries is + # non-empty for all currently known use cases. + write_atomic(self.getFullname(), "".join("%s\n" % + str(myentry) for myentry in myentries)) + else: + # With thin manifest, there's no need to have + # a Manifest file if there are no DIST entries. + try: + os.unlink(self.getFullname()) + except OSError as e: + if e.errno != errno.ENOENT: + raise + if sign: self.sign() except (IOError, OSError) as e: @@ -270,14 +319,14 @@ class Manifest(object): fname = os.path.join("files", fname) if not os.path.exists(self.pkgdir+fname) and not ignoreMissing: raise FileNotFound(fname) - if not ftype in portage.const.MANIFEST2_IDENTIFIERS: + if not ftype in MANIFEST2_IDENTIFIERS: raise InvalidDataType(ftype) if ftype == "AUX" and fname.startswith("files"): fname = fname[6:] self.fhashdict[ftype][fname] = {} if hashdict != None: self.fhashdict[ftype][fname].update(hashdict) - if not portage.const.MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname]: + if not MANIFEST2_REQUIRED_HASH in self.fhashdict[ftype][fname]: self.updateFileHashes(ftype, fname, checkExisting=False, ignoreMissing=ignoreMissing) def removeFile(self, ftype, fname): @@ -290,7 +339,7 @@ class Manifest(object): def findFile(self, fname): """ Return entrytype of the given file if present in Manifest or None if not present """ - for t in portage.const.MANIFEST2_IDENTIFIERS: + for t in MANIFEST2_IDENTIFIERS: if fname in self.fhashdict[t]: return t return None @@ -305,6 +354,8 @@ class Manifest(object): distfiles to raise a FileNotFound exception for (if no file or existing checksums are available), and defaults to all distfiles when not specified.""" + if not self.allow_create: + return if checkExisting: self.checkAllHashes() if assumeDistHashesSometimes or assumeDistHashesAlways: @@ -313,13 +364,88 @@ class Manifest(object): distfilehashes = {} self.__init__(self.pkgdir, self.distdir, fetchlist_dict=self.fetchlist_dict, from_scratch=True, - manifest1_compat=False) - cpvlist = [] + thin=self.thin, allow_missing=self.allow_missing, + allow_create=self.allow_create, hashes=self.hashes) pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) cat = self._pkgdir_category() pkgdir = self.pkgdir + if self.thin: + cpvlist = self._update_thin_pkgdir(cat, pn, pkgdir) + else: + cpvlist = self._update_thick_pkgdir(cat, pn, pkgdir) + + distlist = set() + for cpv in cpvlist: + distlist.update(self._getCpvDistfiles(cpv)) + + if requiredDistfiles is None: + # This allows us to force removal of stale digests for the + # ebuild --force digest option (no distfiles are required). + requiredDistfiles = set() + elif len(requiredDistfiles) == 0: + # repoman passes in an empty list, which implies that all distfiles + # are required. + requiredDistfiles = distlist.copy() + required_hash_types = set() + required_hash_types.add("size") + required_hash_types.add(MANIFEST2_REQUIRED_HASH) + for f in distlist: + fname = os.path.join(self.distdir, f) + mystat = None + try: + mystat = os.stat(fname) + except OSError: + pass + if f in distfilehashes and \ + not required_hash_types.difference(distfilehashes[f]) and \ + ((assumeDistHashesSometimes and mystat is None) or \ + (assumeDistHashesAlways and mystat is None) or \ + (assumeDistHashesAlways and mystat is not None and \ + set(distfilehashes[f]) == set(self.hashes) and \ + distfilehashes[f]["size"] == mystat.st_size)): + self.fhashdict["DIST"][f] = distfilehashes[f] + else: + try: + self.fhashdict["DIST"][f] = perform_multiple_checksums(fname, self.hashes) + except FileNotFound: + if f in requiredDistfiles: + raise + def _is_cpv(self, cat, pn, filename): + if not filename.endswith(".ebuild"): + return None + pf = filename[:-7] + ps = portage.versions._pkgsplit(pf) + cpv = "%s/%s" % (cat, pf) + if not ps: + raise PortagePackageException( + _("Invalid package name: '%s'") % cpv) + if ps[0] != pn: + raise PortagePackageException( + _("Package name does not " + "match directory name: '%s'") % cpv) + return cpv + + def _update_thin_pkgdir(self, cat, pn, pkgdir): + for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir): + break + cpvlist = [] + for f in pkgdir_files: + try: + f = _unicode_decode(f, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + continue + if f[:1] == '.': + continue + pf = self._is_cpv(cat, pn, f) + if pf is not None: + cpvlist.append(pf) + return cpvlist + + def _update_thick_pkgdir(self, cat, pn, pkgdir): + cpvlist = [] for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir): break for f in pkgdir_files: @@ -330,21 +456,10 @@ class Manifest(object): continue if f[:1] == ".": continue - pf = None - if f[-7:] == '.ebuild': - pf = f[:-7] + pf = self._is_cpv(cat, pn, f) if pf is not None: mytype = "EBUILD" - ps = portage.versions._pkgsplit(pf) - cpv = "%s/%s" % (cat, pf) - if not ps: - raise PortagePackageException( - _("Invalid package name: '%s'") % cpv) - if ps[0] != pn: - raise PortagePackageException( - _("Package name does not " - "match directory name: '%s'") % cpv) - cpvlist.append(cpv) + cpvlist.append(pf) elif manifest2MiscfileFilter(f): mytype = "MISC" else: @@ -368,41 +483,7 @@ class Manifest(object): continue self.fhashdict["AUX"][f] = perform_multiple_checksums( os.path.join(self.pkgdir, "files", f.lstrip(os.sep)), self.hashes) - distlist = set() - for cpv in cpvlist: - distlist.update(self._getCpvDistfiles(cpv)) - if requiredDistfiles is None: - # This allows us to force removal of stale digests for the - # ebuild --force digest option (no distfiles are required). - requiredDistfiles = set() - elif len(requiredDistfiles) == 0: - # repoman passes in an empty list, which implies that all distfiles - # are required. - requiredDistfiles = distlist.copy() - required_hash_types = set() - required_hash_types.add("size") - required_hash_types.add(portage.const.MANIFEST2_REQUIRED_HASH) - for f in distlist: - fname = os.path.join(self.distdir, f) - mystat = None - try: - mystat = os.stat(fname) - except OSError: - pass - if f in distfilehashes and \ - not required_hash_types.difference(distfilehashes[f]) and \ - ((assumeDistHashesSometimes and mystat is None) or \ - (assumeDistHashesAlways and mystat is None) or \ - (assumeDistHashesAlways and mystat is not None and \ - len(distfilehashes[f]) == len(self.hashes) and \ - distfilehashes[f]["size"] == mystat.st_size)): - self.fhashdict["DIST"][f] = distfilehashes[f] - else: - try: - self.fhashdict["DIST"][f] = perform_multiple_checksums(fname, self.hashes) - except FileNotFound: - if f in requiredDistfiles: - raise + return cpvlist def _pkgdir_category(self): return self.pkgdir.rstrip(os.sep).split(os.sep)[-2] @@ -417,7 +498,7 @@ class Manifest(object): return absname def checkAllHashes(self, ignoreMissingFiles=False): - for t in portage.const.MANIFEST2_IDENTIFIERS: + for t in MANIFEST2_IDENTIFIERS: self.checkTypeHashes(t, ignoreMissingFiles=ignoreMissingFiles) def checkTypeHashes(self, idtype, ignoreMissingFiles=False): @@ -481,7 +562,7 @@ class Manifest(object): def updateAllHashes(self, checkExisting=False, ignoreMissingFiles=True): """ Regenerate all hashes for all files in this Manifest. """ - for idtype in portage.const.MANIFEST2_IDENTIFIERS: + for idtype in MANIFEST2_IDENTIFIERS: self.updateTypeHashes(idtype, checkExisting=checkExisting, ignoreMissingFiles=ignoreMissingFiles) @@ -526,9 +607,11 @@ class Manifest(object): myfile.close() for l in lines: mysplit = l.split() - if len(mysplit) == 4 and mysplit[0] in portage.const.MANIFEST1_HASH_FUNCTIONS and not 1 in rVal: + if len(mysplit) == 4 and mysplit[0] in MANIFEST1_HASH_FUNCTIONS \ + and 1 not in rVal: rVal.append(1) - elif len(mysplit) > 4 and mysplit[0] in portage.const.MANIFEST2_IDENTIFIERS and ((len(mysplit) - 3) % 2) == 0 and not 2 in rVal: + elif len(mysplit) > 4 and mysplit[0] in MANIFEST2_IDENTIFIERS \ + and ((len(mysplit) - 3) % 2) == 0 and not 2 in rVal: rVal.append(2) return rVal diff --git a/portage_with_autodep/pym/portage/manifest.pyo b/portage_with_autodep/pym/portage/manifest.pyo Binary files differnew file mode 100644 index 0000000..d482bbd --- /dev/null +++ b/portage_with_autodep/pym/portage/manifest.pyo diff --git a/portage_with_autodep/pym/portage/news.py b/portage_with_autodep/pym/portage/news.py index 866e5b0..bbd9325 100644 --- a/portage_with_autodep/pym/portage/news.py +++ b/portage_with_autodep/pym/portage/news.py @@ -2,24 +2,30 @@ # Copyright 2006-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import print_function + __all__ = ["NewsManager", "NewsItem", "DisplayRestriction", "DisplayProfileRestriction", "DisplayKeywordRestriction", - "DisplayInstalledRestriction"] + "DisplayInstalledRestriction", + "count_unread_news", "display_news_notifications"] import io import logging import os as _os import re +from portage import OrderedDict from portage import os from portage import _encodings from portage import _unicode_decode from portage import _unicode_encode +from portage.const import NEWS_LIB_PATH from portage.util import apply_secpass_permissions, ensure_dirs, \ grabfile, normalize_path, write_atomic, writemsg_level from portage.data import portage_gid from portage.dep import isvalidatom from portage.localization import _ from portage.locks import lockfile, unlockfile +from portage.output import colorize from portage.exception import InvalidLocation, OperationNotPermitted, \ PermissionDenied @@ -39,7 +45,6 @@ class NewsManager(object): def __init__(self, portdb, vardb, news_path, unread_path, language_id='en'): self.news_path = news_path self.unread_path = unread_path - self.target_root = vardb.root self.language_id = language_id self.config = vardb.settings self.vdb = vardb @@ -114,7 +119,6 @@ class NewsManager(object): except PermissionDenied: return - updates = [] for itemid in news: try: itemid = _unicode_decode(itemid, @@ -250,10 +254,11 @@ class NewsItem(object): return self._valid def parse(self): - lines = io.open(_unicode_encode(self.path, + f = io.open(_unicode_encode(self.path, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace' - ).readlines() + mode='r', encoding=_encodings['content'], errors='replace') + lines = f.readlines() + f.close() self.restrictions = {} invalids = [] for i, line in enumerate(lines): @@ -349,3 +354,67 @@ class DisplayInstalledRestriction(DisplayRestriction): if vdb.match(self.atom): return True return False + +def count_unread_news(portdb, vardb, repos=None, update=True): + """ + Returns a dictionary mapping repos to integer counts of unread news items. + By default, this will scan all repos and check for new items that have + appeared since the last scan. + + @param portdb: a portage tree database + @type portdb: pordbapi + @param vardb: an installed package database + @type vardb: vardbapi + @param repos: names of repos to scan (None means to scan all available repos) + @type repos: list or None + @param update: check for new items (default is True) + @type update: boolean + @rtype: dict + @return: dictionary mapping repos to integer counts of unread news items + """ + + NEWS_PATH = os.path.join("metadata", "news") + UNREAD_PATH = os.path.join(vardb.settings['EROOT'], NEWS_LIB_PATH, "news") + news_counts = OrderedDict() + if repos is None: + repos = portdb.getRepositories() + + permission_msgs = set() + for repo in repos: + try: + manager = NewsManager(portdb, vardb, NEWS_PATH, UNREAD_PATH) + count = manager.getUnreadItems(repo, update=True) + except PermissionDenied as e: + # NOTE: The NewsManager typically handles permission errors by + # returning silently, so PermissionDenied won't necessarily be + # raised even if we do trigger a permission error above. + msg = _unicode_decode("Permission denied: '%s'\n") % (e,) + if msg in permission_msgs: + pass + else: + permission_msgs.add(msg) + writemsg_level(msg, level=logging.ERROR, noiselevel=-1) + news_counts[repo] = 0 + else: + news_counts[repo] = count + + return news_counts + +def display_news_notifications(news_counts): + """ + Display a notification for unread news items, using a dictionary mapping + repos to integer counts, like that returned from count_unread_news(). + """ + newsReaderDisplay = False + for repo, count in news_counts.items(): + if count > 0: + if not newsReaderDisplay: + newsReaderDisplay = True + print() + print(colorize("WARN", " * IMPORTANT:"), end=' ') + print("%s news items need reading for repository '%s'." % (count, repo)) + + if newsReaderDisplay: + print(colorize("WARN", " *"), end=' ') + print("Use " + colorize("GOOD", "eselect news") + " to read news items.") + print() diff --git a/portage_with_autodep/pym/portage/news.pyo b/portage_with_autodep/pym/portage/news.pyo Binary files differnew file mode 100644 index 0000000..bbd247c --- /dev/null +++ b/portage_with_autodep/pym/portage/news.pyo diff --git a/portage_with_autodep/pym/portage/output.py b/portage_with_autodep/pym/portage/output.py index 0e8245f..98bec81 100644 --- a/portage_with_autodep/pym/portage/output.py +++ b/portage_with_autodep/pym/portage/output.py @@ -162,11 +162,14 @@ def _parse_color_map(config_root='/', onerror=None): if token[0] in quotes and token[0] == token[-1]: token = token[1:-1] return token + + f = None try: - lineno=0 - for line in io.open(_unicode_encode(myfile, + f = io.open(_unicode_encode(myfile, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace'): + mode='r', encoding=_encodings['content'], errors='replace') + lineno = 0 + for line in f: lineno += 1 commenter_pos = line.find("#") @@ -226,6 +229,9 @@ def _parse_color_map(config_root='/', onerror=None): elif e.errno == errno.EACCES: raise PermissionDenied(myfile) raise + finally: + if f is not None: + f.close() def nc_len(mystr): tmp = re.sub(esc_seq + "^m]+m", "", mystr); @@ -319,6 +325,12 @@ def style_to_ansi_code(style): ret += codes.get(attr_name, attr_name) return ret +def colormap(): + mycolors = [] + for c in ("GOOD", "WARN", "BAD", "HILITE", "BRACKET", "NORMAL"): + mycolors.append("%s=$'%s'" % (c, style_to_ansi_code(c))) + return "\n".join(mycolors) + def colorize(color_key, text): global havecolor if havecolor: @@ -335,12 +347,12 @@ compat_functions_colors = ["bold","white","teal","turquoise","darkteal", "fuchsia","purple","blue","darkblue","green","darkgreen","yellow", "brown","darkyellow","red","darkred"] -def create_color_func(color_key): - def derived_func(*args): - newargs = list(args) - newargs.insert(0, color_key) - return colorize(*newargs) - return derived_func +class create_color_func(object): + __slots__ = ("_color_key",) + def __init__(self, color_key): + self._color_key = color_key + def __call__(self, text): + return colorize(self._color_key, text) for c in compat_functions_colors: globals()[c] = create_color_func(c) @@ -416,12 +428,14 @@ class StyleWriter(formatter.DumbWriter): def get_term_size(): """ Get the number of lines and columns of the tty that is connected to - stdout. Returns a tuple of (lines, columns) or (-1, -1) if an error + stdout. Returns a tuple of (lines, columns) or (0, 0) if an error occurs. The curses module is used if available, otherwise the output of - `stty size` is parsed. + `stty size` is parsed. The lines and columns values are guaranteed to be + greater than or equal to zero, since a negative COLUMNS variable is + known to prevent some commands from working (see bug #394091). """ if not sys.stdout.isatty(): - return -1, -1 + return (0, 0) try: import curses try: @@ -436,10 +450,13 @@ def get_term_size(): out = out.split() if len(out) == 2: try: - return int(out[0]), int(out[1]) + val = (int(out[0]), int(out[1])) except ValueError: pass - return -1, -1 + else: + if val[0] >= 0 and val[1] >= 0: + return val + return (0, 0) def set_term_size(lines, columns, fd): """ diff --git a/portage_with_autodep/pym/portage/output.pyo b/portage_with_autodep/pym/portage/output.pyo Binary files differnew file mode 100644 index 0000000..993a2de --- /dev/null +++ b/portage_with_autodep/pym/portage/output.pyo diff --git a/portage_with_autodep/pym/portage/package/__init__.pyo b/portage_with_autodep/pym/portage/package/__init__.pyo Binary files differnew file mode 100644 index 0000000..9d8f30c --- /dev/null +++ b/portage_with_autodep/pym/portage/package/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo Binary files differnew file mode 100644 index 0000000..927b4bc --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py index cd22554..0c613ce 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -11,7 +11,7 @@ from portage.dep import ExtendedAtomDict, _repo_separator, _slot_separator from portage.localization import _ from portage.package.ebuild._config.helper import ordered_by_atom_specificity from portage.util import grabdict_package, stack_lists, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str class KeywordsManager(object): """Manager class to handle keywords processing and validation""" @@ -20,7 +20,8 @@ class KeywordsManager(object): global_accept_keywords=""): self._pkeywords_list = [] rawpkeywords = [grabdict_package( - os.path.join(x, "package.keywords"), recursive=1, + os.path.join(x.location, "package.keywords"), + recursive=x.portage1_directories, verify_eapi=True) \ for x in profiles] for pkeyworddict in rawpkeywords: @@ -35,7 +36,8 @@ class KeywordsManager(object): self._p_accept_keywords = [] raw_p_accept_keywords = [grabdict_package( - os.path.join(x, "package.accept_keywords"), recursive=1, + os.path.join(x.location, "package.accept_keywords"), + recursive=x.portage1_directories, verify_eapi=True) \ for x in profiles] for d in raw_p_accept_keywords: @@ -75,10 +77,11 @@ class KeywordsManager(object): def getKeywords(self, cpv, slot, keywords, repo): - cp = cpv_getkey(cpv) - pkg = "".join((cpv, _slot_separator, slot)) - if repo and repo != Package.UNKNOWN_REPO: - pkg = "".join((pkg, _repo_separator, repo)) + if not hasattr(cpv, 'slot'): + pkg = _pkg_str(cpv, slot=slot, repo=repo) + else: + pkg = cpv + cp = pkg.cp keywords = [[x for x in keywords.split() if x != "-*"]] for pkeywords_dict in self._pkeywords_list: cpdict = pkeywords_dict.get(cp) @@ -206,12 +209,16 @@ class KeywordsManager(object): hasstable = False hastesting = False for gp in mygroups: - if gp == "*" or (gp == "-*" and len(mygroups) == 1): - writemsg(_("--- WARNING: Package '%(cpv)s' uses" - " '%(keyword)s' keyword.\n") % {"cpv": cpv, "keyword": gp}, - noiselevel=-1) - if gp == "*": - match = True + if gp == "*": + match = True + break + elif gp == "~*": + hastesting = True + for x in pgroups: + if x[:1] == "~": + match = True + break + if match: break elif gp in pgroups: match = True @@ -254,18 +261,19 @@ class KeywordsManager(object): """ pgroups = global_accept_keywords.split() + if not hasattr(cpv, 'slot'): + cpv = _pkg_str(cpv, slot=slot, repo=repo) cp = cpv_getkey(cpv) unmaskgroups = [] if self._p_accept_keywords: - cpv_slot = "%s:%s" % (cpv, slot) accept_keywords_defaults = tuple('~' + keyword for keyword in \ pgroups if keyword[:1] not in "~-") for d in self._p_accept_keywords: cpdict = d.get(cp) if cpdict: pkg_accept_keywords = \ - ordered_by_atom_specificity(cpdict, cpv_slot) + ordered_by_atom_specificity(cpdict, cpv) if pkg_accept_keywords: for x in pkg_accept_keywords: if not x: @@ -274,9 +282,8 @@ class KeywordsManager(object): pkgdict = self.pkeywordsdict.get(cp) if pkgdict: - cpv_slot = "%s:%s" % (cpv, slot) pkg_accept_keywords = \ - ordered_by_atom_specificity(pkgdict, cpv_slot, repo=repo) + ordered_by_atom_specificity(pkgdict, cpv) if pkg_accept_keywords: for x in pkg_accept_keywords: unmaskgroups.extend(x) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo Binary files differnew file mode 100644 index 0000000..15043f0 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/KeywordsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.py index effd55b..f76e7e2 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 201-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -10,7 +10,7 @@ from portage.dep import ExtendedAtomDict, use_reduce from portage.exception import InvalidDependString from portage.localization import _ from portage.util import grabdict, grabdict_package, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity @@ -119,8 +119,9 @@ class LicenseManager(object): cp = cpv_getkey(cpv) cpdict = self._plicensedict.get(cp) if cpdict: - cpv_slot = "%s:%s" % (cpv, slot) - plicence_list = ordered_by_atom_specificity(cpdict, cpv_slot, repo) + if not hasattr(cpv, slot): + cpv = _pkg_str(cpv, slot=slot, repo=repo) + plicence_list = ordered_by_atom_specificity(cpdict, cpv) if plicence_list: accept_license = list(self._accept_license) for x in plicence_list: diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo Binary files differnew file mode 100644 index 0000000..4a38298 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LicenseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py index c2b115b..f7a1177 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.py @@ -5,7 +5,11 @@ __all__ = ( 'LocationsManager', ) +import collections import io +import warnings + +import portage from portage import os, eapi_is_supported, _encodings, _unicode_encode from portage.const import CUSTOM_PROFILE_PATH, GLOBAL_CONFIG_PATH, \ PROFILE_PATH, USER_CONFIG_PATH @@ -13,7 +17,20 @@ from portage.exception import DirectoryNotFound, ParseError from portage.localization import _ from portage.util import ensure_dirs, grabfile, \ normalize_path, shlex_split, writemsg +from portage.repository.config import parse_layout_conf, \ + _portage1_profiles_allow_directories + + +_PORTAGE1_DIRECTORIES = frozenset([ + 'package.mask', 'package.provided', + 'package.use', 'package.use.mask', 'package.use.force', + 'use.mask', 'use.force']) + +_profile_node = collections.namedtuple('_profile_node', + 'location portage1_directories') +_allow_parent_colon = frozenset( + ["portage-2"]) class LocationsManager(object): @@ -25,9 +42,9 @@ class LocationsManager(object): self.config_root = config_root self.target_root = target_root self._user_config = local_config - + if self.eprefix is None: - self.eprefix = "" + self.eprefix = portage.const.EPREFIX if self.config_root is None: self.config_root = self.eprefix + os.sep @@ -37,17 +54,33 @@ class LocationsManager(object): self._check_var_directory("PORTAGE_CONFIGROOT", self.config_root) self.abs_user_config = os.path.join(self.config_root, USER_CONFIG_PATH) + self.config_profile_path = config_profile_path + + def load_profiles(self, repositories, known_repository_paths): + known_repository_paths = set(os.path.realpath(x) + for x in known_repository_paths) - if config_profile_path is None: - config_profile_path = \ + known_repos = [] + for x in known_repository_paths: + try: + layout_data = {"profile-formats": + repositories.get_repo_for_location(x).profile_formats} + except KeyError: + layout_data = parse_layout_conf(x)[0] + # force a trailing '/' for ease of doing startswith checks + known_repos.append((x + '/', layout_data)) + known_repos = tuple(known_repos) + + if self.config_profile_path is None: + self.config_profile_path = \ os.path.join(self.config_root, PROFILE_PATH) - if os.path.isdir(config_profile_path): - self.profile_path = config_profile_path + if os.path.isdir(self.config_profile_path): + self.profile_path = self.config_profile_path else: - config_profile_path = \ + self.config_profile_path = \ os.path.join(self.abs_user_config, 'make.profile') - if os.path.isdir(config_profile_path): - self.profile_path = config_profile_path + if os.path.isdir(self.config_profile_path): + self.profile_path = self.config_profile_path else: self.profile_path = None else: @@ -55,19 +88,22 @@ class LocationsManager(object): # here, in order to create an empty profile # for checking dependencies of packages with # empty KEYWORDS. - self.profile_path = config_profile_path + self.profile_path = self.config_profile_path # The symlink might not exist or might not be a symlink. self.profiles = [] + self.profiles_complex = [] if self.profile_path: try: - self._addProfile(os.path.realpath(self.profile_path)) + self._addProfile(os.path.realpath(self.profile_path), + repositories, known_repos) except ParseError as e: writemsg(_("!!! Unable to parse profile: '%s'\n") % \ self.profile_path, noiselevel=-1) writemsg("!!! ParseError: %s\n" % str(e), noiselevel=-1) self.profiles = [] + self.profiles_complex = [] if self._user_config and self.profiles: custom_prof = os.path.join( @@ -75,9 +111,11 @@ class LocationsManager(object): if os.path.exists(custom_prof): self.user_profile_dir = custom_prof self.profiles.append(custom_prof) + self.profiles_complex.append(_profile_node(custom_prof, True)) del custom_prof self.profiles = tuple(self.profiles) + self.profiles_complex = tuple(self.profiles_complex) def _check_var_directory(self, varname, var): if not os.path.isdir(var): @@ -86,14 +124,45 @@ class LocationsManager(object): noiselevel=-1) raise DirectoryNotFound(var) - def _addProfile(self, currentPath): + def _addProfile(self, currentPath, repositories, known_repos): + current_abs_path = os.path.abspath(currentPath) + allow_directories = True + allow_parent_colon = True + repo_loc = None + compat_mode = False + intersecting_repos = [x for x in known_repos if current_abs_path.startswith(x[0])] + if intersecting_repos: + # protect against nested repositories. Insane configuration, but the longest + # path will be the correct one. + repo_loc, layout_data = max(intersecting_repos, key=lambda x:len(x[0])) + allow_directories = any(x in _portage1_profiles_allow_directories + for x in layout_data['profile-formats']) + compat_mode = layout_data['profile-formats'] == ('portage-1-compat',) + allow_parent_colon = any(x in _allow_parent_colon + for x in layout_data['profile-formats']) + + if compat_mode: + offenders = _PORTAGE1_DIRECTORIES.intersection(os.listdir(currentPath)) + offenders = sorted(x for x in offenders + if os.path.isdir(os.path.join(currentPath, x))) + if offenders: + warnings.warn(_("Profile '%(profile_path)s' in repository " + "'%(repo_name)s' is implicitly using 'portage-1' profile format, but " + "the repository profiles are not marked as that format. This will break " + "in the future. Please either convert the following paths " + "to files, or add\nprofile-formats = portage-1\nto the " + "repositories layout.conf. Files: '%(files)s'\n") + % dict(profile_path=currentPath, repo_name=repo_loc, + files=', '.join(offenders))) + parentsFile = os.path.join(currentPath, "parent") eapi_file = os.path.join(currentPath, "eapi") + f = None try: - eapi = io.open(_unicode_encode(eapi_file, + f = io.open(_unicode_encode(eapi_file, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace' - ).readline().strip() + mode='r', encoding=_encodings['content'], errors='replace') + eapi = f.readline().strip() except IOError: pass else: @@ -102,21 +171,69 @@ class LocationsManager(object): "Profile contains unsupported " "EAPI '%s': '%s'") % \ (eapi, os.path.realpath(eapi_file),)) + finally: + if f is not None: + f.close() if os.path.exists(parentsFile): parents = grabfile(parentsFile) if not parents: raise ParseError( _("Empty parent file: '%s'") % parentsFile) for parentPath in parents: + abs_parent = parentPath[:1] == os.sep + if not abs_parent and allow_parent_colon: + parentPath = self._expand_parent_colon(parentsFile, + parentPath, repo_loc, repositories) + + # NOTE: This os.path.join() call is intended to ignore + # currentPath if parentPath is already absolute. parentPath = normalize_path(os.path.join( currentPath, parentPath)) + + if abs_parent or repo_loc is None or \ + not parentPath.startswith(repo_loc): + # It seems that this parent may point outside + # of the current repo, so realpath it. + parentPath = os.path.realpath(parentPath) + if os.path.exists(parentPath): - self._addProfile(parentPath) + self._addProfile(parentPath, repositories, known_repos) else: raise ParseError( _("Parent '%s' not found: '%s'") % \ (parentPath, parentsFile)) + self.profiles.append(currentPath) + self.profiles_complex.append( + _profile_node(currentPath, allow_directories)) + + def _expand_parent_colon(self, parentsFile, parentPath, + repo_loc, repositories): + colon = parentPath.find(":") + if colon == -1: + return parentPath + + if colon == 0: + if repo_loc is None: + raise ParseError( + _("Parent '%s' not found: '%s'") % \ + (parentPath, parentsFile)) + else: + parentPath = normalize_path(os.path.join( + repo_loc, 'profiles', parentPath[colon+1:])) + else: + p_repo_name = parentPath[:colon] + try: + p_repo_loc = repositories.get_location_for_name(p_repo_name) + except KeyError: + raise ParseError( + _("Parent '%s' not found: '%s'") % \ + (parentPath, parentsFile)) + else: + parentPath = normalize_path(os.path.join( + p_repo_loc, 'profiles', parentPath[colon+1:])) + + return parentPath def set_root_override(self, root_overwrite=None): # Allow ROOT setting to come from make.conf if it's not overridden diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo Binary files differnew file mode 100644 index 0000000..c64d313 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/LocationsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py index df93e10..bce1152 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.py @@ -5,9 +5,12 @@ __all__ = ( 'MaskManager', ) +import warnings + from portage import os from portage.dep import ExtendedAtomDict, match_from_list, _repo_separator, _slot_separator -from portage.util import append_repo, grabfile_package, stack_lists +from portage.localization import _ +from portage.util import append_repo, grabfile_package, stack_lists, writemsg from portage.versions import cpv_getkey from _emerge.Package import Package @@ -32,30 +35,76 @@ class MaskManager(object): # repo may be often referenced by others as the master. pmask_cache = {} - def grab_pmask(loc): + def grab_pmask(loc, repo_config): if loc not in pmask_cache: - pmask_cache[loc] = grabfile_package( - os.path.join(loc, "profiles", "package.mask"), - recursive=1, remember_source_file=True, verify_eapi=True) + path = os.path.join(loc, 'profiles', 'package.mask') + pmask_cache[loc] = grabfile_package(path, + recursive=repo_config.portage1_profiles, + remember_source_file=True, verify_eapi=True) + if repo_config.portage1_profiles_compat and os.path.isdir(path): + warnings.warn(_("Repository '%(repo_name)s' is implicitly using " + "'portage-1' profile format in its profiles/package.mask, but " + "the repository profiles are not marked as that format. This will break " + "in the future. Please either convert the following paths " + "to files, or add\nprofile-formats = portage-1\nto the " + "repositories layout.conf.\n") + % dict(repo_name=repo_config.name)) + return pmask_cache[loc] repo_pkgmasklines = [] for repo in repositories.repos_with_profiles(): lines = [] - repo_lines = grab_pmask(repo.location) + repo_lines = grab_pmask(repo.location, repo) + removals = frozenset(line[0][1:] for line in repo_lines + if line[0][:1] == "-") + matched_removals = set() for master in repo.masters: - master_lines = grab_pmask(master.location) + master_lines = grab_pmask(master.location, master) + for line in master_lines: + if line[0] in removals: + matched_removals.add(line[0]) + # Since we don't stack masters recursively, there aren't any + # atoms earlier in the stack to be matched by negative atoms in + # master_lines. Also, repo_lines may contain negative atoms + # that are intended to negate atoms from a different master + # than the one with which we are currently stacking. Therefore, + # we disable warn_for_unmatched_removal here (see bug #386569). lines.append(stack_lists([master_lines, repo_lines], incremental=1, - remember_source_file=True, warn_for_unmatched_removal=True, - strict_warn_for_unmatched_removal=strict_umatched_removal)) - if not repo.masters: + remember_source_file=True, warn_for_unmatched_removal=False)) + + # It's safe to warn for unmatched removal if masters have not + # been overridden by the user, which is guaranteed when + # user_config is false (when called by repoman). + if repo.masters: + unmatched_removals = removals.difference(matched_removals) + if unmatched_removals and not user_config: + source_file = os.path.join(repo.location, + "profiles", "package.mask") + unmatched_removals = list(unmatched_removals) + if len(unmatched_removals) > 3: + writemsg( + _("--- Unmatched removal atoms in %s: %s and %s more\n") % + (source_file, + ", ".join("-" + x for x in unmatched_removals[:3]), + len(unmatched_removals) - 3), noiselevel=-1) + else: + writemsg( + _("--- Unmatched removal atom(s) in %s: %s\n") % + (source_file, + ", ".join("-" + x for x in unmatched_removals)), + noiselevel=-1) + + else: lines.append(stack_lists([repo_lines], incremental=1, - remember_source_file=True, warn_for_unmatched_removal=True, + remember_source_file=True, warn_for_unmatched_removal=not user_config, strict_warn_for_unmatched_removal=strict_umatched_removal)) repo_pkgmasklines.extend(append_repo(stack_lists(lines), repo.name, remember_source_file=True)) repo_pkgunmasklines = [] for repo in repositories.repos_with_profiles(): + if not repo.portage1_profiles: + continue repo_lines = grabfile_package(os.path.join(repo.location, "profiles", "package.unmask"), \ recursive=1, remember_source_file=True, verify_eapi=True) lines = stack_lists([repo_lines], incremental=1, \ @@ -69,9 +118,14 @@ class MaskManager(object): profile_pkgunmasklines = [] for x in profiles: profile_pkgmasklines.append(grabfile_package( - os.path.join(x, "package.mask"), recursive=1, remember_source_file=True, verify_eapi=True)) - profile_pkgunmasklines.append(grabfile_package( - os.path.join(x, "package.unmask"), recursive=1, remember_source_file=True, verify_eapi=True)) + os.path.join(x.location, "package.mask"), + recursive=x.portage1_directories, + remember_source_file=True, verify_eapi=True)) + if x.portage1_directories: + profile_pkgunmasklines.append(grabfile_package( + os.path.join(x.location, "package.unmask"), + recursive=x.portage1_directories, + remember_source_file=True, verify_eapi=True)) profile_pkgmasklines = stack_lists(profile_pkgmasklines, incremental=1, \ remember_source_file=True, warn_for_unmatched_removal=True, strict_warn_for_unmatched_removal=strict_umatched_removal) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo Binary files differnew file mode 100644 index 0000000..f48eb47 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/MaskManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py index d7ef0f6..e1ec7f4 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -7,10 +7,10 @@ __all__ = ( from _emerge.Package import Package from portage import os -from portage.dep import ExtendedAtomDict, remove_slot, _get_useflag_re +from portage.dep import dep_getrepo, dep_getslot, ExtendedAtomDict, remove_slot, _get_useflag_re from portage.localization import _ from portage.util import grabfile, grabdict_package, read_corresponding_eapi_file, stack_lists, writemsg -from portage.versions import cpv_getkey +from portage.versions import cpv_getkey, _pkg_str from portage.package.ebuild._config.helper import ordered_by_atom_specificity @@ -65,9 +65,9 @@ class UseManager(object): self.repositories = repositories - def _parse_file_to_tuple(self, file_name): + def _parse_file_to_tuple(self, file_name, recursive=True): ret = [] - lines = grabfile(file_name, recursive=1) + lines = grabfile(file_name, recursive=recursive) eapi = read_corresponding_eapi_file(file_name) useflag_re = _get_useflag_re(eapi) for prefixed_useflag in lines: @@ -82,10 +82,10 @@ class UseManager(object): ret.append(prefixed_useflag) return tuple(ret) - def _parse_file_to_dict(self, file_name, juststrings=False): + def _parse_file_to_dict(self, file_name, juststrings=False, recursive=True): ret = {} location_dict = {} - file_dict = grabdict_package(file_name, recursive=1, verify_eapi=True) + file_dict = grabdict_package(file_name, recursive=recursive, verify_eapi=True) eapi = read_corresponding_eapi_file(file_name) useflag_re = _get_useflag_re(eapi) for k, v in file_dict.items(): @@ -132,19 +132,29 @@ class UseManager(object): return ret def _parse_profile_files_to_tuple_of_tuples(self, file_name, locations): - return tuple(self._parse_file_to_tuple(os.path.join(profile, file_name)) for profile in locations) + return tuple(self._parse_file_to_tuple( + os.path.join(profile.location, file_name), + recursive=profile.portage1_directories) + for profile in locations) def _parse_profile_files_to_tuple_of_dicts(self, file_name, locations, juststrings=False): - return tuple(self._parse_file_to_dict(os.path.join(profile, file_name), juststrings) for profile in locations) + return tuple(self._parse_file_to_dict( + os.path.join(profile.location, file_name), juststrings, + recursive=profile.portage1_directories) + for profile in locations) def getUseMask(self, pkg=None): if pkg is None: return frozenset(stack_lists( self._usemask_list, incremental=True)) + slot = None cp = getattr(pkg, "cp", None) if cp is None: - cp = cpv_getkey(remove_slot(pkg)) + slot = dep_getslot(pkg) + repo = dep_getrepo(pkg) + pkg = _pkg_str(remove_slot(pkg), slot=slot, repo=repo) + cp = pkg.cp usemask = [] if hasattr(pkg, "repo") and pkg.repo != Package.UNKNOWN_REPO: repos = [] diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo Binary files differnew file mode 100644 index 0000000..2c9a609 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/UseManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo Binary files differnew file mode 100644 index 0000000..b2ebd21 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/VirtualsManager.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo Binary files differnew file mode 100644 index 0000000..b03cc29 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo Binary files differnew file mode 100644 index 0000000..aeee789 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/env_var_validation.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo Binary files differnew file mode 100644 index 0000000..9854444 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/features_set.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.py b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.py index 4f46781..ee0c090 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -24,7 +24,7 @@ def ordered_by_atom_specificity(cpdict, pkg, repo=None): order to achieve desired results (and thus corrupting the ChangeLog like ordering of the file). """ - if repo and repo != Package.UNKNOWN_REPO: + if not hasattr(pkg, 'repo') and repo and repo != Package.UNKNOWN_REPO: pkg = pkg + _repo_separator + repo results = [] diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo Binary files differnew file mode 100644 index 0000000..f2b9261 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/helper.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py index 6d42809..1a75de9 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ( @@ -15,14 +15,14 @@ env_blacklist = frozenset(( "A", "AA", "CATEGORY", "DEPEND", "DESCRIPTION", "EAPI", "EBUILD_FORCE_TEST", "EBUILD_PHASE", "EBUILD_SKIP_MANIFEST", "ED", "EMERGE_FROM", "EPREFIX", "EROOT", - "HOMEPAGE", "INHERITED", "IUSE", + "GREP_OPTIONS", "HOMEPAGE", "INHERITED", "IUSE", "KEYWORDS", "LICENSE", "MERGE_TYPE", "PDEPEND", "PF", "PKGUSE", "PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE", "PORTAGE_BUILDIR_LOCKED", "PORTAGE_BUILT_USE", "PORTAGE_CONFIGROOT", "PORTAGE_IUSE", - "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME", "PORTAGE_SANDBOX_COMPAT_LEVEL", - "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "RESTRICT", - "ROOT", "SLOT", "SRC_URI" + "PORTAGE_NONFATAL", "PORTAGE_REPO_NAME", + "PORTAGE_USE", "PROPERTIES", "PROVIDE", "RDEPEND", "REPOSITORY", + "RESTRICT", "ROOT", "SLOT", "SRC_URI" )) environ_whitelist = [] @@ -36,7 +36,7 @@ environ_whitelist = [] # environment in order to prevent sandbox from sourcing /etc/profile # in it's bashrc (causing major leakage). environ_whitelist += [ - "ACCEPT_LICENSE", "BASH_ENV", "BUILD_PREFIX", "D", + "ACCEPT_LICENSE", "BASH_ENV", "BUILD_PREFIX", "COLUMNS", "D", "DISTDIR", "DOC_SYMLINKS_DIR", "EAPI", "EBUILD", "EBUILD_FORCE_TEST", "EBUILD_PHASE", "ECLASSDIR", "ECLASS_DEPTH", "ED", @@ -50,23 +50,23 @@ environ_whitelist += [ "PORTAGE_BINPKG_TMPFILE", "PORTAGE_BIN_PATH", "PORTAGE_BUILDDIR", "PORTAGE_BUNZIP2_COMMAND", "PORTAGE_BZIP2_COMMAND", - "PORTAGE_COLORMAP", + "PORTAGE_COLORMAP", "PORTAGE_COMPRESS_EXCLUDE_SUFFIXES", "PORTAGE_CONFIGROOT", "PORTAGE_DEBUG", "PORTAGE_DEPCACHEDIR", "PORTAGE_EBUILD_EXIT_FILE", "PORTAGE_FEATURES", "PORTAGE_GID", "PORTAGE_GRPNAME", "PORTAGE_INST_GID", "PORTAGE_INST_UID", "PORTAGE_IPC_DAEMON", "PORTAGE_IUSE", - "PORTAGE_LOG_FILE", + "PORTAGE_LOG_FILE", "PORTAGE_OVERRIDE_EPREFIX", "PORTAGE_PYM_PATH", "PORTAGE_PYTHON", "PORTAGE_QUIET", "PORTAGE_REPO_NAME", "PORTAGE_RESTRICT", - "PORTAGE_SANDBOX_COMPAT_LEVEL", "PORTAGE_SIGPIPE_STATUS", + "PORTAGE_SIGPIPE_STATUS", "PORTAGE_TMPDIR", "PORTAGE_UPDATE_ENV", "PORTAGE_USERNAME", "PORTAGE_VERBOSE", "PORTAGE_WORKDIR_MODE", "PORTDIR", "PORTDIR_OVERLAY", "PREROOTPATH", "PROFILE_PATHS", "REPLACING_VERSIONS", "REPLACED_BY_VERSION", "ROOT", "ROOTPATH", "T", "TMP", "TMPDIR", "USE_EXPAND", "USE_ORDER", "WORKDIR", - "XARGS", + "XARGS", "__PORTAGE_TEST_HARDLINK_LOCKS", ] # user config variables @@ -134,8 +134,9 @@ environ_filter += [ # portage config variables and variables set directly by portage environ_filter += [ "ACCEPT_CHOSTS", "ACCEPT_KEYWORDS", "ACCEPT_PROPERTIES", "AUTOCLEAN", - "CLEAN_DELAY", "COLLISION_IGNORE", "CONFIG_PROTECT", - "CONFIG_PROTECT_MASK", "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", + "CLEAN_DELAY", "COLLISION_IGNORE", + "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", + "EGENCACHE_DEFAULT_OPTS", "EMERGE_DEFAULT_OPTS", "EMERGE_LOG_DIR", "EMERGE_WARNING_DELAY", "FETCHCOMMAND", "FETCHCOMMAND_FTP", @@ -143,7 +144,8 @@ environ_filter += [ "FETCHCOMMAND_RSYNC", "FETCHCOMMAND_SFTP", "GENTOO_MIRRORS", "NOCONFMEM", "O", "PORTAGE_BACKGROUND", "PORTAGE_BACKGROUND_UNMERGE", - "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED", "PORTAGE_CALLER", + "PORTAGE_BINHOST", + "PORTAGE_BINHOST_CHUNKSIZE", "PORTAGE_BUILDIR_LOCKED", "PORTAGE_ELOG_CLASSES", "PORTAGE_ELOG_MAILFROM", "PORTAGE_ELOG_MAILSUBJECT", "PORTAGE_ELOG_MAILURI", "PORTAGE_ELOG_SYSTEM", @@ -156,8 +158,8 @@ environ_filter += [ "PORTAGE_RO_DISTDIRS", "PORTAGE_RSYNC_EXTRA_OPTS", "PORTAGE_RSYNC_OPTS", "PORTAGE_RSYNC_RETRIES", "PORTAGE_SYNC_STALE", - "PORTAGE_USE", "PORT_LOGDIR", - "QUICKPKG_DEFAULT_OPTS", + "PORTAGE_USE", "PORT_LOGDIR", "PORT_LOGDIR_CLEAN", + "QUICKPKG_DEFAULT_OPTS", "REPOMAN_DEFAULT_OPTS", "RESUMECOMMAND", "RESUMECOMMAND_FTP", "RESUMECOMMAND_HTTP", "RESUMECOMMAND_HTTPS", "RESUMECOMMAND_RSYNC", "RESUMECOMMAND_SFTP", diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo Binary files differnew file mode 100644 index 0000000..06ea37e --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/special_env_vars.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_config/unpack_dependencies.py b/portage_with_autodep/pym/portage/package/ebuild/_config/unpack_dependencies.py new file mode 100644 index 0000000..1375189 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_config/unpack_dependencies.py @@ -0,0 +1,38 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os, _supported_eapis +from portage.dep import use_reduce +from portage.eapi import eapi_has_automatic_unpack_dependencies +from portage.exception import InvalidDependString +from portage.localization import _ +from portage.util import grabfile, writemsg + +def load_unpack_dependencies_configuration(repositories): + repo_dict = {} + for repo in repositories.repos_with_profiles(): + for eapi in _supported_eapis: + if eapi_has_automatic_unpack_dependencies(eapi): + file_name = os.path.join(repo.location, "profiles", "unpack_dependencies", eapi) + lines = grabfile(file_name, recursive=True) + for line in lines: + elements = line.split() + suffix = elements[0].lower() + if len(elements) == 1: + writemsg(_("--- Missing unpack dependencies for '%s' suffix in '%s'\n") % (suffix, file_name)) + depend = " ".join(elements[1:]) + try: + use_reduce(depend, eapi=eapi) + except InvalidDependString as e: + writemsg(_("--- Invalid unpack dependencies for '%s' suffix in '%s': '%s'\n" % (suffix, file_name, e))) + else: + repo_dict.setdefault(repo.name, {}).setdefault(eapi, {})[suffix] = depend + + ret = {} + for repo in repositories.repos_with_profiles(): + for repo_name in [x.name for x in repo.masters] + [repo.name]: + for eapi in repo_dict.get(repo_name, {}): + for suffix, depend in repo_dict.get(repo_name, {}).get(eapi, {}).items(): + ret.setdefault(repo.name, {}).setdefault(eapi, {})[suffix] = depend + + return ret diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py new file mode 100644 index 0000000..d23677d --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.py @@ -0,0 +1,54 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +import portage +from portage.dep import _repo_separator +from portage.elog import elog_process +from portage.elog.messages import eerror + +def eapi_invalid(self, cpv, repo_name, settings, + eapi_var, eapi_parsed, eapi_lineno): + + msg = [] + msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" + " conform with PMS section 7.3.1 (see bug #402167):") % + (cpv, _repo_separator, repo_name), 70)) + + if not eapi_parsed: + # None means the assignment was not found, while an + # empty string indicates an (invalid) empty assingment. + msg.append( + "\tvalid EAPI assignment must" + " occur on or before line: %s" % + eapi_lineno) + else: + msg.append(("\tbash returned EAPI '%s' which does not match " + "assignment on line: %s") % + (eapi_var, eapi_lineno)) + + if 'parse-eapi-ebuild-head' in settings.features: + msg.extend(textwrap.wrap(("NOTE: This error will soon" + " become unconditionally fatal in a future version of Portage," + " but at this time, it can by made non-fatal by setting" + " FEATURES=-parse-eapi-ebuild-head in" + " make.conf."), 70)) + else: + msg.extend(textwrap.wrap(("NOTE: This error will soon" + " become unconditionally fatal in a future version of Portage." + " At the earliest opportunity, please enable" + " FEATURES=parse-eapi-ebuild-head in make.conf in order to" + " make this error fatal."), 70)) + + if portage.data.secpass >= 2: + # TODO: improve elog permission error handling (bug #416231) + for line in msg: + eerror(line, phase="other", key=cpv) + elog_process(cpv, settings, + phasefilter=("other",)) + + else: + out = portage.output.EOutput() + for line in msg: + out.eerror(line) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo Binary files differnew file mode 100644 index 0000000..0181c03 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_eapi_invalid.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo Binary files differnew file mode 100644 index 0000000..315cb0f --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/ExitCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo Binary files differnew file mode 100644 index 0000000..9f75518 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/IpcCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py index fb6e61e..7bbb0e8 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io @@ -7,6 +7,7 @@ import portage from portage import os from portage import _unicode_decode from portage.dep import Atom +from portage.eapi import eapi_has_repo_deps from portage.elog import messages as elog_messages from portage.exception import InvalidAtom from portage.package.ebuild._ipc.IpcCommand import IpcCommand @@ -26,19 +27,21 @@ class QueryCommand(IpcCommand): def __call__(self, argv): """ - @returns: tuple of (stdout, stderr, returncode) + @return: tuple of (stdout, stderr, returncode) """ cmd, root, atom_str = argv + eapi = self.settings.get('EAPI') + allow_repo = eapi_has_repo_deps(eapi) try: - atom = Atom(atom_str) + atom = Atom(atom_str, allow_repo=allow_repo) except InvalidAtom: return ('', 'invalid atom: %s\n' % atom_str, 2) warnings = [] try: - atom = Atom(atom_str, eapi=self.settings.get('EAPI')) + atom = Atom(atom_str, allow_repo=allow_repo, eapi=eapi) except InvalidAtom as e: warnings.append(_unicode_decode("QA Notice: %s: %s") % (cmd, e)) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo Binary files differnew file mode 100644 index 0000000..0e9ee96 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/QueryCommand.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo Binary files differnew file mode 100644 index 0000000..d9f8d25 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_ipc/__init__.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/_metadata_invalid.py b/portage_with_autodep/pym/portage/package/ebuild/_metadata_invalid.py new file mode 100644 index 0000000..bcf1f7f --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_metadata_invalid.py @@ -0,0 +1,41 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import textwrap + +import portage +from portage.dep import _repo_separator +from portage.elog import elog_process +from portage.elog.messages import eerror + +def eapi_invalid(self, cpv, repo_name, settings, + eapi_var, eapi_parsed, eapi_lineno): + + msg = [] + msg.extend(textwrap.wrap(("EAPI assignment in ebuild '%s%s%s' does not" + " conform with PMS section 7.3.1 (see bug #402167):") % + (cpv, _repo_separator, repo_name), 70)) + + if not eapi_parsed: + # None means the assignment was not found, while an + # empty string indicates an (invalid) empty assingment. + msg.append( + "\tvalid EAPI assignment must" + " occur on or before line: %s" % + eapi_lineno) + else: + msg.append(("\tbash returned EAPI '%s' which does not match " + "assignment on line: %s") % + (eapi_var, eapi_lineno)) + + if portage.data.secpass >= 2: + # TODO: improve elog permission error handling (bug #416231) + for line in msg: + eerror(line, phase="other", key=cpv) + elog_process(cpv, settings, + phasefilter=("other",)) + + else: + out = portage.output.EOutput() + for line in msg: + out.eerror(line) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py new file mode 100644 index 0000000..44e2576 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py @@ -0,0 +1,43 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.exception import (FileNotFound, + PermissionDenied, PortagePackageException) +from portage.localization import _ +from portage.util._async.ForkProcess import ForkProcess + +class ManifestProcess(ForkProcess): + + __slots__ = ("cp", "distdir", "fetchlist_dict", "repo_config") + + MODIFIED = 16 + + def _run(self): + mf = self.repo_config.load_manifest( + os.path.join(self.repo_config.location, self.cp), + self.distdir, fetchlist_dict=self.fetchlist_dict) + + try: + mf.create(assumeDistHashesAlways=True) + except FileNotFound as e: + portage.writemsg(_("!!! File %s doesn't exist, can't update " + "Manifest\n") % e, noiselevel=-1) + return 1 + + except PortagePackageException as e: + portage.writemsg(("!!! %s\n") % (e,), noiselevel=-1) + return 1 + + try: + modified = mf.write(sign=False) + except PermissionDenied as e: + portage.writemsg("!!! %s: %s\n" % (_("Permission Denied"), e,), + noiselevel=-1) + return 1 + else: + if modified: + return self.MODIFIED + else: + return os.EX_OK diff --git a/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py new file mode 100644 index 0000000..38ac482 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py @@ -0,0 +1,93 @@ +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import portage +from portage import os +from portage.dep import _repo_separator +from portage.exception import InvalidDependString +from portage.localization import _ +from portage.util._async.AsyncScheduler import AsyncScheduler +from .ManifestTask import ManifestTask + +class ManifestScheduler(AsyncScheduler): + + def __init__(self, portdb, cp_iter=None, + gpg_cmd=None, gpg_vars=None, force_sign_key=None, **kwargs): + + AsyncScheduler.__init__(self, **kwargs) + + self._portdb = portdb + + if cp_iter is None: + cp_iter = self._iter_every_cp() + self._cp_iter = cp_iter + self._gpg_cmd = gpg_cmd + self._gpg_vars = gpg_vars + self._force_sign_key = force_sign_key + self._task_iter = self._iter_tasks() + + def _next_task(self): + return next(self._task_iter) + + def _iter_every_cp(self): + # List categories individually, in order to start yielding quicker, + # and in order to reduce latency in case of a signal interrupt. + cp_all = self._portdb.cp_all + for category in sorted(self._portdb.categories): + for cp in cp_all(categories=(category,)): + yield cp + + def _iter_tasks(self): + portdb = self._portdb + distdir = portdb.settings["DISTDIR"] + disabled_repos = set() + + for cp in self._cp_iter: + if self._terminated.is_set(): + break + # We iterate over portdb.porttrees, since it's common to + # tweak this attribute in order to adjust repo selection. + for mytree in portdb.porttrees: + if self._terminated.is_set(): + break + repo_config = portdb.repositories.get_repo_for_location(mytree) + if not repo_config.create_manifest: + if repo_config.name not in disabled_repos: + disabled_repos.add(repo_config.name) + portage.writemsg( + _(">>> Skipping creating Manifest for %s%s%s; " + "repository is configured to not use them\n") % + (cp, _repo_separator, repo_config.name), + noiselevel=-1) + continue + cpv_list = portdb.cp_list(cp, mytree=[repo_config.location]) + if not cpv_list: + continue + fetchlist_dict = {} + try: + for cpv in cpv_list: + fetchlist_dict[cpv] = \ + list(portdb.getFetchMap(cpv, mytree=mytree)) + except InvalidDependString as e: + portage.writemsg( + _("!!! %s%s%s: SRC_URI: %s\n") % + (cp, _repo_separator, repo_config.name, e), + noiselevel=-1) + self._error_count += 1 + continue + + yield ManifestTask(cp=cp, distdir=distdir, + fetchlist_dict=fetchlist_dict, repo_config=repo_config, + gpg_cmd=self._gpg_cmd, gpg_vars=self._gpg_vars, + force_sign_key=self._force_sign_key) + + def _task_exit(self, task): + + if task.returncode != os.EX_OK: + if not self._terminated_tasks: + portage.writemsg( + "Error processing %s%s%s, continuing...\n" % + (task.cp, _repo_separator, task.repo_config.name), + noiselevel=-1) + + AsyncScheduler._task_exit(self, task) diff --git a/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py new file mode 100644 index 0000000..0ee2b91 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py @@ -0,0 +1,186 @@ +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import re +import subprocess + +from portage import os +from portage import _unicode_encode, _encodings +from portage.const import MANIFEST2_IDENTIFIERS +from portage.util import (atomic_ofstream, grablines, + shlex_split, varexpand, writemsg) +from portage.util._async.PipeLogger import PipeLogger +from portage.util._async.PopenProcess import PopenProcess +from _emerge.CompositeTask import CompositeTask +from _emerge.PipeReader import PipeReader +from .ManifestProcess import ManifestProcess + +class ManifestTask(CompositeTask): + + __slots__ = ("cp", "distdir", "fetchlist_dict", "gpg_cmd", + "gpg_vars", "repo_config", "force_sign_key", "_manifest_path") + + _PGP_HEADER = b"BEGIN PGP SIGNED MESSAGE" + _manifest_line_re = re.compile(r'^(%s) ' % "|".join(MANIFEST2_IDENTIFIERS)) + _gpg_key_id_re = re.compile(r'^[0-9A-F]*$') + _gpg_key_id_lengths = (8, 16, 24, 32, 40) + + def _start(self): + self._manifest_path = os.path.join(self.repo_config.location, + self.cp, "Manifest") + manifest_proc = ManifestProcess(cp=self.cp, distdir=self.distdir, + fetchlist_dict=self.fetchlist_dict, repo_config=self.repo_config, + scheduler=self.scheduler) + self._start_task(manifest_proc, self._manifest_proc_exit) + + def _manifest_proc_exit(self, manifest_proc): + self._assert_current(manifest_proc) + if manifest_proc.returncode not in (os.EX_OK, manifest_proc.MODIFIED): + self.returncode = manifest_proc.returncode + self._current_task = None + self.wait() + return + + modified = manifest_proc.returncode == manifest_proc.MODIFIED + sign = self.gpg_cmd is not None + + if not modified and sign: + sign = self._need_signature() + if not sign and self.force_sign_key is not None \ + and os.path.exists(self._manifest_path): + self._check_sig_key() + return + + if not sign or not os.path.exists(self._manifest_path): + self.returncode = os.EX_OK + self._current_task = None + self.wait() + return + + self._start_gpg_proc() + + def _check_sig_key(self): + null_fd = os.open('/dev/null', os.O_RDONLY) + popen_proc = PopenProcess(proc=subprocess.Popen( + ["gpg", "--verify", self._manifest_path], + stdin=null_fd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT), + pipe_reader=PipeReader()) + os.close(null_fd) + popen_proc.pipe_reader.input_files = { + "producer" : popen_proc.proc.stdout} + self._start_task(popen_proc, self._check_sig_key_exit) + + @staticmethod + def _parse_gpg_key(output): + """ + Returns the first token which appears to represent a gpg key + id, or None if there is no such token. + """ + regex = ManifestTask._gpg_key_id_re + lengths = ManifestTask._gpg_key_id_lengths + for token in output.split(): + m = regex.match(token) + if m is not None and len(m.group(0)) in lengths: + return m.group(0) + return None + + @staticmethod + def _normalize_gpg_key(key_str): + """ + Strips leading "0x" and trailing "!", and converts to uppercase + (intended to be the same format as that in gpg --verify output). + """ + key_str = key_str.upper() + if key_str.startswith("0X"): + key_str = key_str[2:] + key_str = key_str.rstrip("!") + return key_str + + def _check_sig_key_exit(self, proc): + self._assert_current(proc) + + parsed_key = self._parse_gpg_key( + proc.pipe_reader.getvalue().decode('utf_8', 'replace')) + if parsed_key is not None and \ + self._normalize_gpg_key(parsed_key) == \ + self._normalize_gpg_key(self.force_sign_key): + self.returncode = os.EX_OK + self._current_task = None + self.wait() + return + + if self._was_cancelled(): + self.wait() + return + + self._strip_sig(self._manifest_path) + self._start_gpg_proc() + + @staticmethod + def _strip_sig(manifest_path): + """ + Strip an existing signature from a Manifest file. + """ + line_re = ManifestTask._manifest_line_re + lines = grablines(manifest_path) + f = None + try: + f = atomic_ofstream(manifest_path) + for line in lines: + if line_re.match(line) is not None: + f.write(line) + f.close() + f = None + finally: + if f is not None: + f.abort() + + def _start_gpg_proc(self): + gpg_vars = self.gpg_vars + if gpg_vars is None: + gpg_vars = {} + else: + gpg_vars = gpg_vars.copy() + gpg_vars["FILE"] = self._manifest_path + gpg_cmd = varexpand(self.gpg_cmd, mydict=gpg_vars) + gpg_cmd = shlex_split(gpg_cmd) + gpg_proc = PopenProcess(proc=subprocess.Popen(gpg_cmd, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT)) + # PipeLogger echos output and efficiently monitors for process + # exit by listening for the stdout EOF event. + gpg_proc.pipe_reader = PipeLogger(background=self.background, + input_fd=gpg_proc.proc.stdout, scheduler=self.scheduler) + self._start_task(gpg_proc, self._gpg_proc_exit) + + def _gpg_proc_exit(self, gpg_proc): + if self._default_exit(gpg_proc) != os.EX_OK: + self.wait() + return + + rename_args = (self._manifest_path + ".asc", self._manifest_path) + try: + os.rename(*rename_args) + except OSError as e: + writemsg("!!! rename('%s', '%s'): %s\n" % rename_args + (e,), + noiselevel=-1) + try: + os.unlink(self._manifest_path + ".asc") + except OSError: + pass + self.returncode = 1 + else: + self.returncode = os.EX_OK + + self._current_task = None + self.wait() + + def _need_signature(self): + try: + with open(_unicode_encode(self._manifest_path, + encoding=_encodings['fs'], errors='strict'), 'rb') as f: + return self._PGP_HEADER not in f.readline() + except IOError as e: + if e.errno in (errno.ENOENT, errno.ESTALE): + return False + raise diff --git a/portage_with_autodep/pym/portage/tests/dbapi/__init__.py b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/__init__.py index 532918b..418ad86 100644 --- a/portage_with_autodep/pym/portage/tests/dbapi/__init__.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_parallel_manifest/__init__.py @@ -1,2 +1,2 @@ -# Copyright 2011 Gentoo Foundation +# Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py index befdc89..94f8c79 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.py @@ -1,10 +1,10 @@ # Copyright 2010-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import shutil import tempfile from portage import os +from portage import shutil from portage.const import EBUILD_PHASES from portage.elog import elog_process from portage.package.ebuild.config import config @@ -20,7 +20,11 @@ def spawn_nofetch(portdb, ebuild_path, settings=None): to cache metadata. It will be cloned internally, in order to prevent any changes from interfering with the calling code. If settings is None then a suitable config instance will be - acquired from the given portdbapi instance. + acquired from the given portdbapi instance. Do not use the + settings parameter unless setcpv has been called on the given + instance, since otherwise it's possible to trigger issues like + bug #408817 due to fragile assumptions involving the config + state inside doebuild_environment(). A private PORTAGE_BUILDDIR will be created and cleaned up, in order to avoid any interference with any other processes. diff --git a/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo Binary files differnew file mode 100644 index 0000000..ac449ea --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/_spawn_nofetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.py b/portage_with_autodep/pym/portage/package/ebuild/config.py index a8c6ad6..97cbd99 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/config.py +++ b/portage_with_autodep/pym/portage/package/ebuild/config.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -7,7 +7,10 @@ __all__ = [ import copy from itertools import chain +import grp import logging +import platform +import pwd import re import sys import warnings @@ -21,10 +24,9 @@ from portage import bsd_chflags, \ load_mod, os, selinux, _unicode_decode from portage.const import CACHE_PATH, \ DEPCACHE_PATH, INCREMENTALS, MAKE_CONF_FILE, \ - MODULES_FILE_PATH, PORTAGE_BIN_PATH, PORTAGE_PYM_PATH, \ + MODULES_FILE_PATH, \ PRIVATE_PATH, PROFILE_PATH, USER_CONFIG_PATH, \ USER_VIRTUALS_FILE -from portage.const import _SANDBOX_COMPAT_LEVEL from portage.dbapi import dbapi from portage.dbapi.porttree import portdbapi from portage.dbapi.vartree import vartree @@ -41,7 +43,7 @@ from portage.util import ensure_dirs, getconfig, grabdict, \ grabdict_package, grabfile, grabfile_package, LazyItemsDict, \ normalize_path, shlex_split, stack_dictlist, stack_dicts, stack_lists, \ writemsg, writemsg_level -from portage.versions import catpkgsplit, catsplit, cpv_getkey +from portage.versions import catpkgsplit, catsplit, cpv_getkey, _pkg_str from portage.package.ebuild._config import special_env_vars from portage.package.ebuild._config.env_var_validation import validate_cmd_var @@ -120,11 +122,19 @@ class config(object): virtuals ...etc you look in here. """ + _constant_keys = frozenset(['PORTAGE_BIN_PATH', 'PORTAGE_GID', + 'PORTAGE_PYM_PATH']) + _setcpv_aux_keys = ('DEFINED_PHASES', 'DEPEND', 'EAPI', 'INHERITED', 'IUSE', 'REQUIRED_USE', 'KEYWORDS', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'SLOT', 'repository', 'RESTRICT', 'LICENSE',) + _module_aliases = { + "cache.metadata_overlay.database" : "portage.cache.flat_hash.database", + "portage.cache.metadata_overlay.database" : "portage.cache.flat_hash.database", + } + _case_insensitive_vars = special_env_vars.case_insensitive_vars _default_globals = special_env_vars.default_globals _env_blacklist = special_env_vars.env_blacklist @@ -135,7 +145,8 @@ class config(object): def __init__(self, clone=None, mycpv=None, config_profile_path=None, config_incrementals=None, config_root=None, target_root=None, - _eprefix=None, local_config=True, env=None, _unmatched_removal=False): + eprefix=None, local_config=True, env=None, + _unmatched_removal=False): """ @param clone: If provided, init will use deepcopy to copy by value the instance. @type clone: Instance of config class. @@ -151,8 +162,8 @@ class config(object): @type config_root: String @param target_root: __init__ override of $ROOT env variable. @type target_root: String - @param _eprefix: set the EPREFIX variable (private, used by internal tests) - @type _eprefix: String + @param eprefix: set the EPREFIX variable (default is portage.const.EPREFIX) + @type eprefix: String @param local_config: Enables loading of local config (/etc/portage); used most by repoman to ignore local config (keywording and unmasking) @type local_config: Boolean @@ -164,10 +175,6 @@ class config(object): @type _unmatched_removal: Boolean """ - # rename local _eprefix variable for convenience - eprefix = _eprefix - del _eprefix - # When initializing the global portage.settings instance, avoid # raising exceptions whenever possible since exceptions thrown # from 'import portage' or 'import portage.exceptions' statements @@ -208,6 +215,7 @@ class config(object): self.repositories = clone.repositories self._iuse_implicit_match = clone._iuse_implicit_match self._non_user_variables = clone._non_user_variables + self._env_d_blacklist = clone._env_d_blacklist self._repo_make_defaults = clone._repo_make_defaults self.usemask = clone.usemask self.useforce = clone.useforce @@ -272,9 +280,6 @@ class config(object): eprefix = locations_manager.eprefix config_root = locations_manager.config_root - self.profiles = locations_manager.profiles - self.profile_path = locations_manager.profile_path - self.user_profile_dir = locations_manager.user_profile_dir abs_user_config = locations_manager.abs_user_config make_conf = getconfig( @@ -293,6 +298,38 @@ class config(object): eroot = locations_manager.eroot self.global_config_path = locations_manager.global_config_path + # The expand_map is used for variable substitution + # in getconfig() calls, and the getconfig() calls + # update expand_map with the value of each variable + # assignment that occurs. Variable substitution occurs + # in the following order, which corresponds to the + # order of appearance in self.lookuplist: + # + # * env.d + # * make.globals + # * make.defaults + # * make.conf + # + # Notably absent is "env", since we want to avoid any + # interaction with the calling environment that might + # lead to unexpected results. + + env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), + expand=False) or {} + expand_map = env_d.copy() + self._expand_map = expand_map + + # Allow make.globals to set default paths relative to ${EPREFIX}. + expand_map["EPREFIX"] = eprefix + + make_globals = getconfig(os.path.join( + self.global_config_path, 'make.globals'), expand=expand_map) + if make_globals is None: + make_globals = {} + + for k, v in self._default_globals.items(): + make_globals.setdefault(k, v) + if config_incrementals is None: self.incrementals = INCREMENTALS else: @@ -302,14 +339,20 @@ class config(object): self.module_priority = ("user", "default") self.modules = {} - modules_loader = KeyValuePairFileLoader( - os.path.join(config_root, MODULES_FILE_PATH), None, None) + modules_file = os.path.join(config_root, MODULES_FILE_PATH) + modules_loader = KeyValuePairFileLoader(modules_file, None, None) modules_dict, modules_errors = modules_loader.load() self.modules["user"] = modules_dict if self.modules["user"] is None: self.modules["user"] = {} + user_auxdbmodule = \ + self.modules["user"].get("portdbapi.auxdbmodule") + if user_auxdbmodule is not None and \ + user_auxdbmodule in self._module_aliases: + warnings.warn("'%s' is deprecated: %s" % + (user_auxdbmodule, modules_file)) + self.modules["default"] = { - "portdbapi.metadbmodule": "portage.cache.metadata.database", "portdbapi.auxdbmodule": "portage.cache.flat_hash.database", } @@ -328,43 +371,9 @@ class config(object): self.configlist.append({}) self.configdict["pkginternal"] = self.configlist[-1] - self.packages_list = [grabfile_package(os.path.join(x, "packages"), verify_eapi=True) for x in self.profiles] - self.packages = tuple(stack_lists(self.packages_list, incremental=1)) - del self.packages_list - #self.packages = grab_stacked("packages", self.profiles, grabfile, incremental_lines=1) - - # revmaskdict - self.prevmaskdict={} - for x in self.packages: - # Negative atoms are filtered by the above stack_lists() call. - if not isinstance(x, Atom): - x = Atom(x.lstrip('*')) - self.prevmaskdict.setdefault(x.cp, []).append(x) - - # The expand_map is used for variable substitution - # in getconfig() calls, and the getconfig() calls - # update expand_map with the value of each variable - # assignment that occurs. Variable substitution occurs - # in the following order, which corresponds to the - # order of appearance in self.lookuplist: - # - # * env.d - # * make.globals - # * make.defaults - # * make.conf - # - # Notably absent is "env", since we want to avoid any - # interaction with the calling environment that might - # lead to unexpected results. - expand_map = {} - self._expand_map = expand_map - - env_d = getconfig(os.path.join(eroot, "etc", "profile.env"), - expand=expand_map) # env_d will be None if profile.env doesn't exist. if env_d: self.configdict["env.d"].update(env_d) - expand_map.update(env_d) # backupenv is used for calculating incremental variables. if env is None: @@ -390,40 +399,72 @@ class config(object): self.configdict["env"] = LazyItemsDict(self.backupenv) - for x in (self.global_config_path,): - self.mygcfg = getconfig(os.path.join(x, "make.globals"), - expand=expand_map) - if self.mygcfg: - break + self.configlist.append(make_globals) + self.configdict["globals"]=self.configlist[-1] - if self.mygcfg is None: - self.mygcfg = {} + self.make_defaults_use = [] - for k, v in self._default_globals.items(): - self.mygcfg.setdefault(k, v) + #Loading Repositories + self["PORTAGE_CONFIGROOT"] = config_root + self["ROOT"] = target_root + self["EPREFIX"] = eprefix + self["EROOT"] = eroot + known_repos = [] + portdir = "" + portdir_overlay = "" + for confs in [make_globals, make_conf, self.configdict["env"]]: + v = confs.get("PORTDIR") + if v is not None: + portdir = v + known_repos.append(v) + v = confs.get("PORTDIR_OVERLAY") + if v is not None: + portdir_overlay = v + known_repos.extend(shlex_split(v)) + known_repos = frozenset(known_repos) + self["PORTDIR"] = portdir + self["PORTDIR_OVERLAY"] = portdir_overlay + self.lookuplist = [self.configdict["env"]] + self.repositories = load_repository_config(self) - self.configlist.append(self.mygcfg) - self.configdict["globals"]=self.configlist[-1] + locations_manager.load_profiles(self.repositories, known_repos) - self.make_defaults_use = [] - self.mygcfg = {} + profiles_complex = locations_manager.profiles_complex + self.profiles = locations_manager.profiles + self.profile_path = locations_manager.profile_path + self.user_profile_dir = locations_manager.user_profile_dir + + packages_list = [grabfile_package(os.path.join(x, "packages"), + verify_eapi=True) for x in self.profiles] + self.packages = tuple(stack_lists(packages_list, incremental=1)) + + # revmaskdict + self.prevmaskdict={} + for x in self.packages: + # Negative atoms are filtered by the above stack_lists() call. + if not isinstance(x, Atom): + x = Atom(x.lstrip('*')) + self.prevmaskdict.setdefault(x.cp, []).append(x) + + + mygcfg = {} if self.profiles: mygcfg_dlists = [getconfig(os.path.join(x, "make.defaults"), expand=expand_map) for x in self.profiles] self._make_defaults = mygcfg_dlists - self.mygcfg = stack_dicts(mygcfg_dlists, + mygcfg = stack_dicts(mygcfg_dlists, incrementals=self.incrementals) - if self.mygcfg is None: - self.mygcfg = {} - self.configlist.append(self.mygcfg) + if mygcfg is None: + mygcfg = {} + self.configlist.append(mygcfg) self.configdict["defaults"]=self.configlist[-1] - self.mygcfg = getconfig( + mygcfg = getconfig( os.path.join(config_root, MAKE_CONF_FILE), tolerant=tolerant, allow_sourcing=True, expand=expand_map) or {} - self.mygcfg.update(getconfig( + mygcfg.update(getconfig( os.path.join(abs_user_config, 'make.conf'), tolerant=tolerant, allow_sourcing=True, expand=expand_map) or {}) @@ -439,10 +480,18 @@ class config(object): non_user_variables = frozenset(non_user_variables) self._non_user_variables = non_user_variables + self._env_d_blacklist = frozenset(chain( + profile_only_variables, + self._env_blacklist, + )) + env_d = self.configdict["env.d"] + for k in self._env_d_blacklist: + env_d.pop(k, None) + for k in profile_only_variables: - self.mygcfg.pop(k, None) + mygcfg.pop(k, None) - self.configlist.append(self.mygcfg) + self.configlist.append(mygcfg) self.configdict["conf"]=self.configlist[-1] self.configlist.append(LazyItemsDict()) @@ -472,25 +521,23 @@ class config(object): self["ROOT"] = target_root self.backup_changes("ROOT") + # The PORTAGE_OVERRIDE_EPREFIX variable propagates the EPREFIX + # of this config instance to any portage commands or API + # consumers running in subprocesses. self["EPREFIX"] = eprefix self.backup_changes("EPREFIX") + self["PORTAGE_OVERRIDE_EPREFIX"] = eprefix + self.backup_changes("PORTAGE_OVERRIDE_EPREFIX") self["EROOT"] = eroot self.backup_changes("EROOT") - self["PORTAGE_SANDBOX_COMPAT_LEVEL"] = _SANDBOX_COMPAT_LEVEL - self.backup_changes("PORTAGE_SANDBOX_COMPAT_LEVEL") - self._ppropertiesdict = portage.dep.ExtendedAtomDict(dict) self._penvdict = portage.dep.ExtendedAtomDict(dict) - #Loading Repositories - self.repositories = load_repository_config(self) - #filling PORTDIR and PORTDIR_OVERLAY variable for compatibility main_repo = self.repositories.mainRepo() if main_repo is not None: - main_repo = main_repo.user_location - self["PORTDIR"] = main_repo + self["PORTDIR"] = main_repo.user_location self.backup_changes("PORTDIR") # repoman controls PORTDIR_OVERLAY via the environment, so no @@ -501,11 +548,11 @@ class config(object): new_ov = [] if portdir_overlay: - whitespace_re = re.compile(r"\s") + shell_quote_re = re.compile(r"[\s\\\"'$`]") for ov in portdir_overlay: ov = normalize_path(ov) if os.path.isdir(ov): - if whitespace_re.search(ov) is not None: + if shell_quote_re.search(ov) is not None: ov = portage._shell_quote(ov) new_ov.append(ov) else: @@ -528,11 +575,11 @@ class config(object): self._repo_make_defaults[repo.name] = d #Read package.keywords and package.accept_keywords. - self._keywords_manager = KeywordsManager(self.profiles, abs_user_config, \ + self._keywords_manager = KeywordsManager(profiles_complex, abs_user_config, \ local_config, global_accept_keywords=self.configdict["defaults"].get("ACCEPT_KEYWORDS", "")) #Read all USE related files from profiles and optionally from user config. - self._use_manager = UseManager(self.repositories, self.profiles, abs_user_config, user_config=local_config) + self._use_manager = UseManager(self.repositories, profiles_complex, abs_user_config, user_config=local_config) #Initialize all USE related variables we track ourselves. self.usemask = self._use_manager.getUseMask() self.useforce = self._use_manager.getUseForce() @@ -549,7 +596,7 @@ class config(object): self.configdict["conf"].get("ACCEPT_LICENSE", "")) #Read package.mask and package.unmask from profiles and optionally from user config - self._mask_manager = MaskManager(self.repositories, self.profiles, + self._mask_manager = MaskManager(self.repositories, profiles_complex, abs_user_config, user_config=local_config, strict_umatched_removal=_unmatched_removal) @@ -597,16 +644,21 @@ class config(object): self.categories = [grabfile(os.path.join(x, "categories")) \ for x in locations_manager.profile_and_user_locations] category_re = dbapi._category_re - self.categories = tuple(sorted( + # categories used to be a tuple, but now we use a frozenset + # for hashed category validation in pordbapi.cp_list() + self.categories = frozenset( x for x in stack_lists(self.categories, incremental=1) - if category_re.match(x) is not None)) + if category_re.match(x) is not None) archlist = [grabfile(os.path.join(x, "arch.list")) \ for x in locations_manager.profile_and_user_locations] archlist = stack_lists(archlist, incremental=1) self.configdict["conf"]["PORTAGE_ARCHLIST"] = " ".join(archlist) - pkgprovidedlines = [grabfile(os.path.join(x, "package.provided"), recursive=1) for x in self.profiles] + pkgprovidedlines = [grabfile( + os.path.join(x.location, "package.provided"), + recursive=x.portage1_directories) + for x in profiles_complex] pkgprovidedlines = stack_lists(pkgprovidedlines, incremental=1) has_invalid_data = False for x in range(len(pkgprovidedlines)-1, -1, -1): @@ -649,9 +701,6 @@ class config(object): if "USE_ORDER" not in self: self.backupenv["USE_ORDER"] = "env:pkg:conf:defaults:pkginternal:repo:env.d" - self["PORTAGE_GID"] = str(portage_gid) - self.backup_changes("PORTAGE_GID") - self.depcachedir = DEPCACHE_PATH if eprefix: # See comments about make.globals and EPREFIX @@ -678,19 +727,60 @@ class config(object): self["CBUILD"] = self["CHOST"] self.backup_changes("CBUILD") - self["PORTAGE_BIN_PATH"] = PORTAGE_BIN_PATH - self.backup_changes("PORTAGE_BIN_PATH") - self["PORTAGE_PYM_PATH"] = PORTAGE_PYM_PATH - self.backup_changes("PORTAGE_PYM_PATH") + if "USERLAND" not in self: + # Set default USERLAND so that our test cases can assume that + # it's always set. This allows isolated-functions.sh to avoid + # calling uname -s when sourced. + system = platform.system() + if system is not None and \ + (system.endswith("BSD") or system == "DragonFly"): + self["USERLAND"] = "BSD" + else: + self["USERLAND"] = "GNU" + self.backup_changes("USERLAND") - for var in ("PORTAGE_INST_UID", "PORTAGE_INST_GID"): + default_inst_ids = { + "PORTAGE_INST_GID": "0", + "PORTAGE_INST_UID": "0", + } + + if eprefix: + # For prefix environments, default to the UID and GID of + # the top-level EROOT directory. try: - self[var] = str(int(self.get(var, "0"))) + eroot_st = os.stat(eroot) + except OSError: + pass + else: + default_inst_ids["PORTAGE_INST_GID"] = str(eroot_st.st_gid) + default_inst_ids["PORTAGE_INST_UID"] = str(eroot_st.st_uid) + + if "PORTAGE_USERNAME" not in self: + try: + pwd_struct = pwd.getpwuid(eroot_st.st_uid) + except KeyError: + pass + else: + self["PORTAGE_USERNAME"] = pwd_struct.pw_name + self.backup_changes("PORTAGE_USERNAME") + + if "PORTAGE_GRPNAME" not in self: + try: + grp_struct = grp.getgrgid(eroot_st.st_gid) + except KeyError: + pass + else: + self["PORTAGE_GRPNAME"] = grp_struct.gr_name + self.backup_changes("PORTAGE_GRPNAME") + + for var, default_val in default_inst_ids.items(): + try: + self[var] = str(int(self.get(var, default_val))) except ValueError: writemsg(_("!!! %s='%s' is not a valid integer. " - "Falling back to '0'.\n") % (var, self[var]), + "Falling back to %s.\n") % (var, self[var], default_val), noiselevel=-1) - self[var] = "0" + self[var] = default_val self.backup_changes(var) # initialize self.features @@ -699,21 +789,33 @@ class config(object): if bsd_chflags: self.features.add('chflags') - if 'parse-eapi-ebuild-head' in self.features: - portage._validate_cache_for_unsupported_eapis = False - self._iuse_implicit_match = _iuse_implicit_match_cache(self) self._validate_commands() - for k in self._case_insensitive_vars: - if k in self: - self[k] = self[k].lower() - self.backup_changes(k) + for k in self._case_insensitive_vars: + if k in self: + self[k] = self[k].lower() + self.backup_changes(k) + + if main_repo is not None and not main_repo.sync: + main_repo_sync = self.get("SYNC") + if main_repo_sync: + main_repo.sync = main_repo_sync + + # The first constructed config object initializes these modules, + # and subsequent calls to the _init() functions have no effect. + portage.output._init(config_root=self['PORTAGE_CONFIGROOT']) + portage.data._init(self) if mycpv: self.setcpv(mycpv) + @property + def mygcfg(self): + warnings.warn("portage.config.mygcfg is deprecated", stacklevel=3) + return {} + def _validate_commands(self): for k in special_env_vars.validate_commands: v = self.get(k) @@ -817,11 +919,26 @@ class config(object): writemsg(_("!!! INVALID ACCEPT_KEYWORDS: %s\n") % str(group), noiselevel=-1) - abs_profile_path = os.path.join(self["PORTAGE_CONFIGROOT"], - PROFILE_PATH) - if (not self.profile_path or \ - not os.path.exists(os.path.join(self.profile_path, "parent"))) and \ - os.path.exists(os.path.join(self["PORTDIR"], "profiles")): + profile_broken = not self.profile_path or \ + not os.path.exists(os.path.join(self.profile_path, "parent")) and \ + os.path.exists(os.path.join(self["PORTDIR"], "profiles")) + + if profile_broken: + abs_profile_path = None + for x in (PROFILE_PATH, 'etc/portage/make.profile'): + x = os.path.join(self["PORTAGE_CONFIGROOT"], x) + try: + os.lstat(x) + except OSError: + pass + else: + abs_profile_path = x + break + + if abs_profile_path is None: + abs_profile_path = os.path.join(self["PORTAGE_CONFIGROOT"], + PROFILE_PATH) + writemsg(_("\n\n!!! %s is not a symlink and will probably prevent most merges.\n") % abs_profile_path, noiselevel=-1) writemsg(_("!!! It should point into a profile within %s/profiles/\n") % self["PORTDIR"]) @@ -851,13 +968,32 @@ class config(object): writemsg(_("!!! FEATURES=fakeroot is enabled, but the " "fakeroot binary is not installed.\n"), noiselevel=-1) + if os.getuid() == 0 and not hasattr(os, "setgroups"): + warning_shown = False + + if "userpriv" in self.features: + writemsg(_("!!! FEATURES=userpriv is enabled, but " + "os.setgroups is not available.\n"), noiselevel=-1) + warning_shown = True + + if "userfetch" in self.features: + writemsg(_("!!! FEATURES=userfetch is enabled, but " + "os.setgroups is not available.\n"), noiselevel=-1) + warning_shown = True + + if warning_shown and platform.python_implementation() == 'PyPy': + writemsg(_("!!! See https://bugs.pypy.org/issue833 for details.\n"), + noiselevel=-1) + def load_best_module(self,property_string): best_mod = best_from_dict(property_string,self.modules,self.module_priority) mod = None try: mod = load_mod(best_mod) except ImportError: - if not best_mod.startswith("cache."): + if best_mod in self._module_aliases: + mod = load_mod(self._module_aliases[best_mod]) + elif not best_mod.startswith("cache."): raise else: best_mod = "portage." + best_mod @@ -1099,8 +1235,11 @@ class config(object): # packages since we want to save it PORTAGE_BUILT_USE for # evaluating conditional USE deps in atoms passed via IPC to # helpers like has_version and best_version. + aux_keys = set(aux_keys) + if hasattr(mydb, '_aux_cache_keys'): + aux_keys = aux_keys.intersection(mydb._aux_cache_keys) + aux_keys.add('USE') aux_keys = list(aux_keys) - aux_keys.append('USE') for k, v in zip(aux_keys, mydb.aux_get(self.mycpv, aux_keys)): pkg_configdict[k] = v built_use = frozenset(pkg_configdict.pop('USE').split()) @@ -1115,7 +1254,7 @@ class config(object): slot = pkg_configdict["SLOT"] iuse = pkg_configdict["IUSE"] if pkg is None: - cpv_slot = "%s:%s" % (self.mycpv, slot) + cpv_slot = _pkg_str(self.mycpv, slot=slot, repo=repository) else: cpv_slot = pkg pkginternaluse = [] @@ -1462,6 +1601,9 @@ class config(object): @return: A matching profile atom string or None if one is not found. """ + warnings.warn("The config._getProfileMaskAtom() method is deprecated.", + DeprecationWarning, stacklevel=2) + cp = cpv_getkey(cpv) profile_atoms = self.prevmaskdict.get(cp) if profile_atoms: @@ -1564,11 +1706,13 @@ class config(object): @return: A list of properties that have not been accepted. """ accept_properties = self._accept_properties + if not hasattr(cpv, 'slot'): + cpv = _pkg_str(cpv, slot=metadata["SLOT"], + repo=metadata.get("repository")) cp = cpv_getkey(cpv) cpdict = self._ppropertiesdict.get(cp) if cpdict: - cpv_slot = "%s:%s" % (cpv, metadata["SLOT"]) - pproperties_list = ordered_by_atom_specificity(cpdict, cpv_slot, repo=metadata.get('repository')) + pproperties_list = ordered_by_atom_specificity(cpdict, cpv) if pproperties_list: accept_properties = list(self._accept_properties) for x in pproperties_list: @@ -1699,6 +1843,8 @@ class config(object): env_d = getconfig(env_d_filename, expand=False) if env_d: # env_d will be None if profile.env doesn't exist. + for k in self._env_d_blacklist: + env_d.pop(k, None) self.configdict["env.d"].update(env_d) def regenerate(self, useonly=0, use_cache=None): @@ -2025,25 +2171,43 @@ class config(object): self._virtuals_manager._treeVirtuals = {} def __delitem__(self,mykey): - self.modifying() - for x in self.lookuplist: - if x != None: - if mykey in x: - del x[mykey] + self.pop(mykey) + + def __getitem__(self, key): + try: + return self._getitem(key) + except KeyError: + return '' # for backward compat, don't raise KeyError + + def _getitem(self, mykey): + + if mykey in self._constant_keys: + # These two point to temporary values when + # portage plans to update itself. + if mykey == "PORTAGE_BIN_PATH": + return portage._bin_path + elif mykey == "PORTAGE_PYM_PATH": + return portage._pym_path + + elif mykey == "PORTAGE_GID": + return _unicode_decode(str(portage_gid)) - def __getitem__(self,mykey): for d in self.lookuplist: - if mykey in d: + try: return d[mykey] - return '' # for backward compat, don't raise KeyError + except KeyError: + pass + + raise KeyError(mykey) def get(self, k, x=None): - for d in self.lookuplist: - if k in d: - return d[k] - return x + try: + return self._getitem(k) + except KeyError: + return x def pop(self, key, *args): + self.modifying() if len(args) > 1: raise TypeError( "pop expected at most 2 arguments, got " + \ @@ -2059,10 +2223,12 @@ class config(object): def __contains__(self, mykey): """Called to implement membership test operators (in and not in).""" - for d in self.lookuplist: - if mykey in d: - return True - return False + try: + self._getitem(mykey) + except KeyError: + return False + else: + return True def setdefault(self, k, x=None): v = self.get(k) @@ -2077,6 +2243,7 @@ class config(object): def __iter__(self): keys = set() + keys.update(self._constant_keys) for d in self.lookuplist: keys.update(d) return iter(keys) @@ -2086,7 +2253,7 @@ class config(object): def iteritems(self): for k in self: - yield (k, self[k]) + yield (k, self._getitem(k)) def items(self): return list(self.iteritems()) @@ -2168,8 +2335,14 @@ class config(object): if not eapi_exports_merge_type(eapi): mydict.pop("MERGE_TYPE", None) - # Prefix variables are supported starting with EAPI 3. - if phase == 'depend' or eapi is None or not eapi_supports_prefix(eapi): + # Prefix variables are supported beginning with EAPI 3, or when + # force-prefix is in FEATURES, since older EAPIs would otherwise be + # useless with prefix configurations. This brings compatibility with + # the prefix branch of portage, which also supports EPREFIX for all + # EAPIs (for obvious reasons). + if phase == 'depend' or \ + ('force-prefix' not in self.features and + eapi is not None and not eapi_supports_prefix(eapi)): mydict.pop("ED", None) mydict.pop("EPREFIX", None) mydict.pop("EROOT", None) diff --git a/portage_with_autodep/pym/portage/package/ebuild/config.pyo b/portage_with_autodep/pym/portage/package/ebuild/config.pyo Binary files differnew file mode 100644 index 0000000..742ee2b --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/config.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo Binary files differnew file mode 100644 index 0000000..2b9362b --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/deprecated_profile_check.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py index 1e34b14..8705639 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.py @@ -8,7 +8,6 @@ import warnings from portage import os, _encodings, _unicode_decode from portage.exception import DigestException, FileNotFound from portage.localization import _ -from portage.manifest import Manifest from portage.output import EOutput from portage.util import writemsg @@ -16,7 +15,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): """ Verifies checksums. Assumes all files have been downloaded. @rtype: int - @returns: 1 on success and 0 on failure + @return: 1 on success and 0 on failure """ if justmanifest is not None: @@ -28,49 +27,33 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): if mysettings.get("EBUILD_SKIP_MANIFEST") == "1": return 1 - allow_missing = "allow-missing-manifests" in mysettings.features pkgdir = mysettings["O"] - manifest_path = os.path.join(pkgdir, "Manifest") - if not os.path.exists(manifest_path): - if allow_missing: - return 1 - writemsg(_("!!! Manifest file not found: '%s'\n") % manifest_path, - noiselevel=-1) - if strict: - return 0 - else: - return 1 if mf is None: - mf = Manifest(pkgdir, mysettings["DISTDIR"]) - manifest_empty = True - for d in mf.fhashdict.values(): - if d: - manifest_empty = False - break - if manifest_empty: - writemsg(_("!!! Manifest is empty: '%s'\n") % manifest_path, - noiselevel=-1) - if strict: - return 0 - else: - return 1 + mf = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + mf = mf.load_manifest(pkgdir, mysettings["DISTDIR"]) eout = EOutput() eout.quiet = mysettings.get("PORTAGE_QUIET", None) == "1" try: - if strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings: - eout.ebegin(_("checking ebuild checksums ;-)")) - mf.checkTypeHashes("EBUILD") - eout.eend(0) - eout.ebegin(_("checking auxfile checksums ;-)")) - mf.checkTypeHashes("AUX") - eout.eend(0) - eout.ebegin(_("checking miscfile checksums ;-)")) - mf.checkTypeHashes("MISC", ignoreMissingFiles=True) - eout.eend(0) + if not mf.thin and strict and "PORTAGE_PARALLEL_FETCHONLY" not in mysettings: + if mf.fhashdict.get("EBUILD"): + eout.ebegin(_("checking ebuild checksums ;-)")) + mf.checkTypeHashes("EBUILD") + eout.eend(0) + if mf.fhashdict.get("AUX"): + eout.ebegin(_("checking auxfile checksums ;-)")) + mf.checkTypeHashes("AUX") + eout.eend(0) + if mf.fhashdict.get("MISC"): + eout.ebegin(_("checking miscfile checksums ;-)")) + mf.checkTypeHashes("MISC", ignoreMissingFiles=True) + eout.eend(0) for f in myfiles: eout.ebegin(_("checking %s ;-)") % f) ftype = mf.findFile(f) if ftype is None: + if mf.allow_missing: + continue eout.eend(1) writemsg(_("\n!!! Missing digest for '%s'\n") % (f,), noiselevel=-1) @@ -90,7 +73,7 @@ def digestcheck(myfiles, mysettings, strict=False, justmanifest=None, mf=None): writemsg(_("!!! Got: %s\n") % e.value[2], noiselevel=-1) writemsg(_("!!! Expected: %s\n") % e.value[3], noiselevel=-1) return 0 - if allow_missing: + if mf.thin or mf.allow_missing: # In this case we ignore any missing digests that # would otherwise be detected below. return 1 diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo Binary files differnew file mode 100644 index 0000000..66987a2 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/digestcheck.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py index eb7210e..6ad3397 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/digestgen.py +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.py @@ -17,7 +17,6 @@ from portage.dep import use_reduce from portage.exception import InvalidDependString, FileNotFound, \ PermissionDenied, PortagePackageException from portage.localization import _ -from portage.manifest import Manifest from portage.output import colorize from portage.package.ebuild.fetch import fetch from portage.util import writemsg, writemsg_stdout @@ -34,7 +33,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): @param myportdb: a portdbapi instance @type myportdb: portdbapi @rtype: int - @returns: 1 on success and 0 on failure + @return: 1 on success and 0 on failure """ if mysettings is None or myportdb is None: raise TypeError("portage.digestgen(): 'mysettings' and 'myportdb' parameter are required.") @@ -52,9 +51,21 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): del e return 0 mytree = os.path.dirname(os.path.dirname(mysettings["O"])) - manifest1_compat = False - mf = Manifest(mysettings["O"], mysettings["DISTDIR"], - fetchlist_dict=fetchlist_dict, manifest1_compat=manifest1_compat) + try: + mf = mysettings.repositories.get_repo_for_location(mytree) + except KeyError: + # backward compatibility + mytree = os.path.realpath(mytree) + mf = mysettings.repositories.get_repo_for_location(mytree) + + mf = mf.load_manifest(mysettings["O"], mysettings["DISTDIR"], + fetchlist_dict=fetchlist_dict) + + if not mf.allow_create: + writemsg_stdout(_(">>> Skipping creating Manifest for %s; " + "repository is configured to not use them\n") % mysettings["O"]) + return 1 + # Don't require all hashes since that can trigger excessive # fetches when sufficient digests already exist. To ease transition # while Manifest 1 is being removed, only require hashes that will @@ -102,8 +113,6 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): continue if missing_files: - mytree = os.path.realpath(os.path.dirname( - os.path.dirname(mysettings["O"]))) for myfile in missing_files: uris = set() all_restrict = set() @@ -139,8 +148,7 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): if not fetch({myfile : uris}, mysettings): myebuild = os.path.join(mysettings["O"], catsplit(cpv)[1] + ".ebuild") - spawn_nofetch(myportdb, myebuild, - settings=mysettings) + spawn_nofetch(myportdb, myebuild) writemsg(_("!!! Fetch failed for %s, can't update " "Manifest\n") % myfile, noiselevel=-1) if myfile in dist_hashes and \ @@ -183,8 +191,6 @@ def digestgen(myarchives=None, mysettings=None, myportdb=None): os.path.join(mysettings["DISTDIR"], filename)): auto_assumed.append(filename) if auto_assumed: - mytree = os.path.realpath( - os.path.dirname(os.path.dirname(mysettings["O"]))) cp = os.path.sep.join(mysettings["O"].split(os.path.sep)[-2:]) pkgs = myportdb.cp_list(cp, mytree=mytree) pkgs.sort() diff --git a/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo Binary files differnew file mode 100644 index 0000000..66876ec --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/digestgen.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py index c76c1ed..610172f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/doebuild.py +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['doebuild', 'doebuild_environment', 'spawn', 'spawnebuild'] @@ -10,7 +10,6 @@ from itertools import chain import logging import os as _os import re -import shutil import signal import stat import sys @@ -31,8 +30,8 @@ portage.proxy.lazyimport.lazyimport(globals(), ) from portage import auxdbkeys, bsd_chflags, \ - eapi_is_supported, merge, os, selinux, \ - unmerge, _encodings, _parse_eapi_ebuild_head, _os_merge, \ + eapi_is_supported, merge, os, selinux, shutil, \ + unmerge, _encodings, _os_merge, \ _shell_quote, _unicode_decode, _unicode_encode from portage.const import EBUILD_SH_ENV_FILE, EBUILD_SH_ENV_DIR, \ EBUILD_SH_BINARY, INVALID_ENV_FILE, MISC_SH_BINARY @@ -42,16 +41,16 @@ from portage.dbapi.porttree import _parse_uri_map from portage.dep import Atom, check_required_use, \ human_readable_required_use, paren_enclose, use_reduce from portage.eapi import eapi_exports_KV, eapi_exports_merge_type, \ - eapi_exports_replace_vars, eapi_has_required_use, \ - eapi_has_src_prepare_and_src_configure, eapi_has_pkg_pretend -from portage.elog import elog_process + eapi_exports_replace_vars, eapi_exports_REPOSITORY, \ + eapi_has_required_use, eapi_has_src_prepare_and_src_configure, \ + eapi_has_pkg_pretend +from portage.elog import elog_process, _preload_elog_modules from portage.elog.messages import eerror, eqawarn from portage.exception import DigestException, FileNotFound, \ IncorrectParameter, InvalidDependString, PermissionDenied, \ UnsupportedAPIException from portage.localization import _ -from portage.manifest import Manifest -from portage.output import style_to_ansi_code +from portage.output import colormap from portage.package.ebuild.prepare_build_dirs import prepare_build_dirs from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, noiselimit, normalize_path, \ @@ -116,6 +115,38 @@ def _spawn_phase(phase, settings, actionmap=None, **kwargs): ebuild_phase.wait() return ebuild_phase.returncode +def _doebuild_path(settings, eapi=None): + """ + Generate the PATH variable. + """ + + # Note: PORTAGE_BIN_PATH may differ from the global constant + # when portage is reinstalling itself. + portage_bin_path = settings["PORTAGE_BIN_PATH"] + eprefix = settings["EPREFIX"] + prerootpath = [x for x in settings.get("PREROOTPATH", "").split(":") if x] + rootpath = [x for x in settings.get("ROOTPATH", "").split(":") if x] + + prefixes = [] + if eprefix: + prefixes.append(eprefix) + prefixes.append("/") + + path = [] + + if eapi not in (None, "0", "1", "2", "3"): + path.append(os.path.join(portage_bin_path, "ebuild-helpers", "4")) + + path.append(os.path.join(portage_bin_path, "ebuild-helpers")) + path.extend(prerootpath) + + for prefix in prefixes: + for x in ("usr/local/sbin", "usr/local/bin", "usr/sbin", "usr/bin", "sbin", "bin"): + path.append(os.path.join(prefix, x)) + + path.extend(rootpath) + settings["PATH"] = ":".join(path) + def doebuild_environment(myebuild, mydo, myroot=None, settings=None, debug=False, use_cache=None, db=None): """ @@ -143,20 +174,32 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, ebuild_path = os.path.abspath(myebuild) pkg_dir = os.path.dirname(ebuild_path) mytree = os.path.dirname(os.path.dirname(pkg_dir)) - - if "CATEGORY" in mysettings.configdict["pkg"]: - cat = mysettings.configdict["pkg"]["CATEGORY"] - else: - cat = os.path.basename(normalize_path(os.path.join(pkg_dir, ".."))) - mypv = os.path.basename(ebuild_path)[:-7] - - mycpv = cat+"/"+mypv - mysplit = _pkgsplit(mypv) + mysplit = _pkgsplit(mypv, eapi=mysettings.configdict["pkg"].get("EAPI")) if mysplit is None: raise IncorrectParameter( _("Invalid ebuild path: '%s'") % myebuild) + if mysettings.mycpv is not None and \ + mysettings.configdict["pkg"].get("PF") == mypv and \ + "CATEGORY" in mysettings.configdict["pkg"]: + # Assume that PF is enough to assume that we've got + # the correct CATEGORY, though this is not really + # a solid assumption since it's possible (though + # unlikely) that two packages in different + # categories have the same PF. Callers should call + # setcpv or create a clean clone of a locked config + # instance in order to ensure that this assumption + # does not fail like in bug #408817. + cat = mysettings.configdict["pkg"]["CATEGORY"] + mycpv = mysettings.mycpv + elif os.path.basename(pkg_dir) in (mysplit[0], mypv): + # portdbapi or vardbapi + cat = os.path.basename(os.path.dirname(pkg_dir)) + mycpv = cat + "/" + mypv + else: + raise AssertionError("unable to determine CATEGORY") + # Make a backup of PORTAGE_TMPDIR prior to calling config.reset() # so that the caller can override it. tmpdir = mysettings["PORTAGE_TMPDIR"] @@ -208,11 +251,11 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["FILESDIR"] = pkg_dir+"/files" mysettings["PF"] = mypv - if hasattr(mydbapi, '_repo_info'): - repo_info = mydbapi._repo_info[mytree] - mysettings['PORTDIR'] = repo_info.portdir - mysettings['PORTDIR_OVERLAY'] = repo_info.portdir_overlay - mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo_info.name + if hasattr(mydbapi, 'repositories'): + repo = mydbapi.repositories.get_repo_for_location(mytree) + mysettings['PORTDIR'] = repo.eclass_db.porttrees[0] + mysettings['PORTDIR_OVERLAY'] = ' '.join(repo.eclass_db.porttrees[1:]) + mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] = repo.name mysettings["PORTDIR"] = os.path.realpath(mysettings["PORTDIR"]) mysettings["DISTDIR"] = os.path.realpath(mysettings["DISTDIR"]) @@ -235,16 +278,6 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, else: mysettings["PVR"]=mysplit[1]+"-"+mysplit[2] - if "PATH" in mysettings: - mysplit=mysettings["PATH"].split(":") - else: - mysplit=[] - # Note: PORTAGE_BIN_PATH may differ from the global constant - # when portage is reinstalling itself. - portage_bin_path = mysettings["PORTAGE_BIN_PATH"] - if portage_bin_path not in mysplit: - mysettings["PATH"] = portage_bin_path + ":" + mysettings["PATH"] - # All temporary directories should be subdirectories of # $PORTAGE_TMPDIR/portage, since it's common for /tmp and /var/tmp # to be mounted with the "noexec" option (see bug #346899). @@ -268,7 +301,9 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["T"] = os.path.join(mysettings["PORTAGE_BUILDDIR"], "temp") # Prefix forward compatability - mysettings["ED"] = mysettings["D"] + eprefix_lstrip = mysettings["EPREFIX"].lstrip(os.sep) + mysettings["ED"] = os.path.join( + mysettings["D"], eprefix_lstrip).rstrip(os.sep) + os.sep mysettings["PORTAGE_BASHRC"] = os.path.join( mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_FILE) @@ -276,37 +311,41 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, mysettings["PORTAGE_CONFIGROOT"], EBUILD_SH_ENV_DIR) # Allow color.map to control colors associated with einfo, ewarn, etc... - mycolors = [] - for c in ("GOOD", "WARN", "BAD", "HILITE", "BRACKET"): - mycolors.append("%s=$'%s'" % \ - (c, style_to_ansi_code(c))) - mysettings["PORTAGE_COLORMAP"] = "\n".join(mycolors) - - # All EAPI dependent code comes last, so that essential variables - # like PORTAGE_BUILDDIR are still initialized even in cases when + mysettings["PORTAGE_COLORMAP"] = colormap() + + if "COLUMNS" not in mysettings: + # Set COLUMNS, in order to prevent unnecessary stty calls + # inside the set_colors function of isolated-functions.sh. + # We cache the result in os.environ, in order to avoid + # multiple stty calls in cases when get_term_size() falls + # back to stty due to a missing or broken curses module. + columns = os.environ.get("COLUMNS") + if columns is None: + rows, columns = portage.output.get_term_size() + if columns < 1: + # Force a sane value for COLUMNS, so that tools + # like ls don't complain (see bug #394091). + columns = 80 + columns = str(columns) + os.environ["COLUMNS"] = columns + mysettings["COLUMNS"] = columns + + # EAPI is always known here, even for the "depend" phase, because + # EbuildMetadataPhase gets it from _parse_eapi_ebuild_head(). + eapi = mysettings.configdict['pkg']['EAPI'] + _doebuild_path(mysettings, eapi=eapi) + + # All EAPI dependent code comes last, so that essential variables like + # PATH and PORTAGE_BUILDDIR are still initialized even in cases when # UnsupportedAPIException needs to be raised, which can be useful # when uninstalling a package that has corrupt EAPI metadata. - eapi = None - if mydo == 'depend' and 'EAPI' not in mysettings.configdict['pkg']: - if eapi is None and 'parse-eapi-ebuild-head' in mysettings.features: - eapi = _parse_eapi_ebuild_head( - io.open(_unicode_encode(ebuild_path, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='replace')) - - if eapi is not None: - if not eapi_is_supported(eapi): - raise UnsupportedAPIException(mycpv, eapi) - mysettings.configdict['pkg']['EAPI'] = eapi + if not eapi_is_supported(eapi): + raise UnsupportedAPIException(mycpv, eapi) - if mydo != "depend": - # Metadata vars such as EAPI and RESTRICT are - # set by the above config.setcpv() call. - eapi = mysettings["EAPI"] - if not eapi_is_supported(eapi): - # can't do anything with this. - raise UnsupportedAPIException(mycpv, eapi) + if eapi_exports_REPOSITORY(eapi) and "PORTAGE_REPO_NAME" in mysettings.configdict["pkg"]: + mysettings.configdict["pkg"]["REPOSITORY"] = mysettings.configdict["pkg"]["PORTAGE_REPO_NAME"] + if mydo != "depend": if hasattr(mydbapi, "getFetchMap") and \ ("A" not in mysettings.configdict["pkg"] or \ "AA" not in mysettings.configdict["pkg"]): @@ -331,22 +370,41 @@ def doebuild_environment(myebuild, mydo, myroot=None, settings=None, else: mysettings.configdict["pkg"]["AA"] = " ".join(uri_map) - if not eapi_exports_KV(eapi): - # Discard KV for EAPIs that don't support it. Cache KV is restored - # from the backupenv whenever config.reset() is called. - mysettings.pop('KV', None) - elif mydo != 'depend' and 'KV' not in mysettings and \ - mydo in ('compile', 'config', 'configure', 'info', - 'install', 'nofetch', 'postinst', 'postrm', 'preinst', - 'prepare', 'prerm', 'setup', 'test', 'unpack'): - mykv, err1 = ExtractKernelVersion( - os.path.join(mysettings['EROOT'], "usr/src/linux")) - if mykv: - # Regular source tree - mysettings["KV"] = mykv - else: - mysettings["KV"] = "" - mysettings.backup_changes("KV") + ccache = "ccache" in mysettings.features + distcc = "distcc" in mysettings.features + if ccache or distcc: + # Use default ABI libdir in accordance with bug #355283. + libdir = None + default_abi = mysettings.get("DEFAULT_ABI") + if default_abi: + libdir = mysettings.get("LIBDIR_" + default_abi) + if not libdir: + libdir = "lib" + + if distcc: + mysettings["PATH"] = os.path.join(os.sep, eprefix_lstrip, + "usr", libdir, "distcc", "bin") + ":" + mysettings["PATH"] + + if ccache: + mysettings["PATH"] = os.path.join(os.sep, eprefix_lstrip, + "usr", libdir, "ccache", "bin") + ":" + mysettings["PATH"] + + if not eapi_exports_KV(eapi): + # Discard KV for EAPIs that don't support it. Cached KV is restored + # from the backupenv whenever config.reset() is called. + mysettings.pop('KV', None) + elif 'KV' not in mysettings and \ + mydo in ('compile', 'config', 'configure', 'info', + 'install', 'nofetch', 'postinst', 'postrm', 'preinst', + 'prepare', 'prerm', 'setup', 'test', 'unpack'): + mykv, err1 = ExtractKernelVersion( + os.path.join(mysettings['EROOT'], "usr/src/linux")) + if mykv: + # Regular source tree + mysettings["KV"] = mykv + else: + mysettings["KV"] = "" + mysettings.backup_changes("KV") _doebuild_manifest_cache = None _doebuild_broken_ebuilds = set() @@ -356,7 +414,7 @@ _doebuild_commands_without_builddir = ( 'fetch', 'fetchall', 'help', 'manifest' ) -def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, +def doebuild(myebuild, mydo, _unused=None, settings=None, debug=0, listonly=0, fetchonly=0, cleanup=0, dbkey=None, use_cache=1, fetchall=0, tree=None, mydbapi=None, vartree=None, prev_mtimes=None, fd_pipes=None, returnpid=False): @@ -368,10 +426,10 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, @type myebuild: String @param mydo: Phase to run @type mydo: String - @param myroot: $ROOT (usually '/', see man make.conf) - @type myroot: String - @param mysettings: Portage Configuration - @type mysettings: instance of portage.config + @param _unused: Deprecated (use settings["ROOT"] instead) + @type _unused: String + @param settings: Portage Configuration + @type settings: instance of portage.config @param debug: Turns on various debug information (eg, debug for spawn) @type debug: Boolean @param listonly: Used to wrap fetch(); passed such that fetch only lists files required. @@ -403,7 +461,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, caller clean up all returned PIDs. @type returnpid: Boolean @rtype: Boolean - @returns: + @return: 1. 0 for success 2. 1 for error @@ -414,7 +472,18 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, Other variables may not be strictly required, many have defaults that are set inside of doebuild. """ - + + if settings is None: + raise TypeError("settings parameter is required") + mysettings = settings + myroot = settings['EROOT'] + + if _unused is not None and _unused != mysettings['EROOT']: + warnings.warn("The third parameter of the " + "portage.doebuild() is now unused. Use " + "settings['ROOT'] instead.", + DeprecationWarning, stacklevel=2) + if not tree: writemsg("Warning: tree not specified to doebuild\n") tree = "porttree" @@ -432,6 +501,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, "install":["test"], "rpm": ["install"], "package":["install"], + "merge" :["install"], } if mydbapi is None: @@ -480,21 +550,28 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return 1 global _doebuild_manifest_cache + pkgdir = os.path.dirname(myebuild) + manifest_path = os.path.join(pkgdir, "Manifest") + if tree == "porttree": + repo_config = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))) + else: + repo_config = None + mf = None if "strict" in features and \ "digest" not in features and \ tree == "porttree" and \ + not repo_config.thin_manifest and \ mydo not in ("digest", "manifest", "help") and \ - not portage._doebuild_manifest_exempt_depend: + not portage._doebuild_manifest_exempt_depend and \ + not (repo_config.allow_missing_manifest and not os.path.exists(manifest_path)): # Always verify the ebuild checksums before executing it. global _doebuild_broken_ebuilds if myebuild in _doebuild_broken_ebuilds: return 1 - pkgdir = os.path.dirname(myebuild) - manifest_path = os.path.join(pkgdir, "Manifest") - # Avoid checking the same Manifest several times in a row during a # regen with an empty cache. if _doebuild_manifest_cache is None or \ @@ -505,7 +582,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, out.eerror(_("Manifest not found for '%s'") % (myebuild,)) _doebuild_broken_ebuilds.add(myebuild) return 1 - mf = Manifest(pkgdir, mysettings["DISTDIR"]) + mf = repo_config.load_manifest(pkgdir, mysettings["DISTDIR"]) else: mf = _doebuild_manifest_cache @@ -513,10 +590,12 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, try: mf.checkFileHashes("EBUILD", os.path.basename(myebuild)) except KeyError: - out = portage.output.EOutput() - out.eerror(_("Missing digest for '%s'") % (myebuild,)) - _doebuild_broken_ebuilds.add(myebuild) - return 1 + if not (mf.allow_missing and + os.path.basename(myebuild) not in mf.fhashdict["EBUILD"]): + out = portage.output.EOutput() + out.eerror(_("Missing digest for '%s'") % (myebuild,)) + _doebuild_broken_ebuilds.add(myebuild) + return 1 except FileNotFound: out = portage.output.EOutput() out.eerror(_("A file listed in the Manifest " @@ -536,7 +615,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if mf.getFullname() in _doebuild_broken_manifests: return 1 - if mf is not _doebuild_manifest_cache: + if mf is not _doebuild_manifest_cache and not mf.allow_missing: # Make sure that all of the ebuilds are # actually listed in the Manifest. @@ -553,8 +632,8 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, _doebuild_broken_manifests.add(manifest_path) return 1 - # Only cache it if the above stray files test succeeds. - _doebuild_manifest_cache = mf + # We cache it only after all above checks succeed. + _doebuild_manifest_cache = mf logfile=None builddir_lock = None @@ -594,7 +673,6 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if builddir_lock is not None: builddir_lock.unlock() - restrict = set(mysettings.get('PORTAGE_RESTRICT', '').split()) # get possible slot information from the deps file if mydo == "depend": writemsg("!!! DEBUG: dbkey: %s\n" % str(dbkey), 2) @@ -654,6 +732,13 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if rval != os.EX_OK: return rval + else: + # FEATURES=noauto only makes sense for porttree, and we don't want + # it to trigger redundant sourcing of the ebuild for API consumers + # that are using binary packages + if "noauto" in mysettings.features: + mysettings.features.discard("noauto") + # The info phase is special because it uses mkdtemp so and # user (not necessarily in the portage group) can run it. if mydo not in ('info',) and \ @@ -666,6 +751,73 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return unmerge(mysettings["CATEGORY"], mysettings["PF"], myroot, mysettings, vartree=vartree) + phases_to_run = set() + if "noauto" in mysettings.features or \ + mydo not in actionmap_deps: + phases_to_run.add(mydo) + else: + phase_stack = [mydo] + while phase_stack: + x = phase_stack.pop() + if x in phases_to_run: + continue + phases_to_run.add(x) + phase_stack.extend(actionmap_deps.get(x, [])) + del phase_stack + + alist = set(mysettings.configdict["pkg"].get("A", "").split()) + + unpacked = False + if tree != "porttree": + pass + elif "unpack" not in phases_to_run: + unpacked = os.path.exists(os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".unpacked")) + else: + try: + workdir_st = os.stat(mysettings["WORKDIR"]) + except OSError: + pass + else: + newstuff = False + if not os.path.exists(os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".unpacked")): + writemsg_stdout(_( + ">>> Not marked as unpacked; recreating WORKDIR...\n")) + newstuff = True + else: + for x in alist: + writemsg_stdout(">>> Checking %s's mtime...\n" % x) + try: + x_st = os.stat(os.path.join( + mysettings["DISTDIR"], x)) + except OSError: + # file not fetched yet + x_st = None + + if x_st is None or x_st.st_mtime > workdir_st.st_mtime: + writemsg_stdout(_(">>> Timestamp of " + "%s has changed; recreating WORKDIR...\n") % x) + newstuff = True + break + + if newstuff: + if builddir_lock is None and \ + 'PORTAGE_BUILDIR_LOCKED' not in mysettings: + builddir_lock = EbuildBuildDir( + scheduler=PollScheduler().sched_iface, + settings=mysettings) + builddir_lock.lock() + try: + _spawn_phase("clean", mysettings) + finally: + if builddir_lock is not None: + builddir_lock.unlock() + builddir_lock = None + else: + writemsg_stdout(_(">>> WORKDIR is up-to-date, keeping...\n")) + unpacked = True + # Build directory creation isn't required for any of these. # In the fetch phase, the directory is needed only for RESTRICT=fetch # in order to satisfy the sane $PWD requirement (from bug #239560) @@ -739,10 +891,9 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # Only try and fetch the files if we are going to need them ... # otherwise, if user has FEATURES=noauto and they run `ebuild clean # unpack compile install`, we will try and fetch 4 times :/ - need_distfiles = tree == "porttree" and \ + need_distfiles = tree == "porttree" and not unpacked and \ (mydo in ("fetch", "unpack") or \ mydo not in ("digest", "manifest") and "noauto" not in features) - alist = set(mysettings.configdict["pkg"].get("A", "").split()) if need_distfiles: src_uri, = mydbapi.aux_get(mysettings.mycpv, @@ -783,10 +934,14 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, return 0 return 1 - if mydo == "fetch": + if need_distfiles: # Files are already checked inside fetch(), # so do not check them again. checkme = [] + elif unpacked: + # The unpack phase is marked as complete, so it + # would be wasteful to check distfiles again. + checkme = [] else: checkme = alist @@ -845,7 +1000,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # this phase. This can raise PermissionDenied if # the current user doesn't have write access to $PKGDIR. if hasattr(portage, 'db'): - bintree = portage.db[mysettings["ROOT"]]["bintree"] + bintree = portage.db[mysettings['EROOT']]['bintree'] mysettings["PORTAGE_BINPKG_TMPFILE"] = \ bintree.getname(mysettings.mycpv) + \ ".%s" % (os.getpid(),) @@ -866,6 +1021,13 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, if mydo == "package" and bintree is not None: bintree.inject(mysettings.mycpv, filename=mysettings["PORTAGE_BINPKG_TMPFILE"]) + else: + if "PORTAGE_BINPKG_TMPFILE" in mysettings: + try: + os.unlink(mysettings["PORTAGE_BINPKG_TMPFILE"]) + except OSError: + pass + elif mydo=="qmerge": # check to ensure install was run. this *only* pops up when users # forget it and are using ebuild @@ -877,6 +1039,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # qmerge is a special phase that implies noclean. if "noclean" not in mysettings.features: mysettings.features.add("noclean") + _handle_self_update(mysettings, vartree.dbapi) #qmerge is specifically not supposed to do a runtime dep check retval = merge( mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], @@ -893,6 +1056,7 @@ def doebuild(myebuild, mydo, myroot, mysettings, debug=0, listonly=0, # so that it's only called once. elog_process(mysettings.mycpv, mysettings) if retval == os.EX_OK: + _handle_self_update(mysettings, vartree.dbapi) retval = merge(mysettings["CATEGORY"], mysettings["PF"], mysettings["D"], os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info"), myroot, mysettings, @@ -944,10 +1108,31 @@ def _check_temp_dir(settings): # as some people use a separate PORTAGE_TMPDIR mount # we prefer that as the checks below would otherwise be pointless # for those people. - if os.path.exists(os.path.join(settings["PORTAGE_TMPDIR"], "portage")): - checkdir = os.path.join(settings["PORTAGE_TMPDIR"], "portage") + tmpdir = os.path.realpath(settings["PORTAGE_TMPDIR"]) + if os.path.exists(os.path.join(tmpdir, "portage")): + checkdir = os.path.realpath(os.path.join(tmpdir, "portage")) + if ("sandbox" in settings.features or + "usersandox" in settings.features) and \ + not checkdir.startswith(tmpdir + os.sep): + msg = _("The 'portage' subdirectory of the directory " + "referenced by the PORTAGE_TMPDIR variable appears to be " + "a symlink. In order to avoid sandbox violations (see bug " + "#378379), you must adjust PORTAGE_TMPDIR instead of using " + "the symlink located at '%s'. A suitable PORTAGE_TMPDIR " + "setting would be '%s'.") % \ + (os.path.join(tmpdir, "portage"), checkdir) + lines = [] + lines.append("") + lines.append("") + lines.extend(wrap(msg, 72)) + lines.append("") + for line in lines: + if line: + line = "!!! %s" % (line,) + writemsg("%s\n" % (line,), noiselevel=-1) + return 1 else: - checkdir = settings["PORTAGE_TMPDIR"] + checkdir = tmpdir if not os.access(checkdir, os.W_OK): writemsg(_("%s is not writable.\n" @@ -955,8 +1140,7 @@ def _check_temp_dir(settings): noiselevel=-1) return 1 - else: - fd = tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir) + with tempfile.NamedTemporaryFile(prefix="exectest-", dir=checkdir) as fd: os.chmod(fd.name, 0o755) if not os.access(fd.name, os.X_OK): writemsg(_("Can not execute files in %s\n" @@ -1085,7 +1269,8 @@ def _validate_deps(mysettings, myroot, mydo, mydbapi): all_keys.add("SRC_URI") all_keys = tuple(all_keys) metadata = dict(zip(all_keys, - mydbapi.aux_get(mysettings.mycpv, all_keys))) + mydbapi.aux_get(mysettings.mycpv, all_keys, + myrepo=mysettings.get("PORTAGE_REPO_NAME")))) class FakeTree(object): def __init__(self, mydb): @@ -1173,7 +1358,7 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero @param keywords: Extra options encoded as a dict, to be passed to spawn @type keywords: Dictionary @rtype: Integer - @returns: + @return: 1. The return code of the spawned process. """ @@ -1201,7 +1386,8 @@ def spawn(mystring, mysettings, debug=0, free=0, droppriv=0, sesandbox=0, fakero # fake ownership/permissions will have to be converted to real # permissions in the merge phase. fakeroot = fakeroot and uid != 0 and portage.process.fakeroot_capable - if droppriv and not uid and portage_gid and portage_uid: + if droppriv and uid == 0 and portage_gid and portage_uid and \ + hasattr(os, "setgroups"): keywords.update({"uid":portage_uid,"gid":portage_gid, "groups":userpriv_groups,"umask":0o02}) if not free: @@ -1277,6 +1463,17 @@ def spawnebuild(mydo, actionmap, mysettings, debug, alwaysdep=0, if mydo == "pretend" and not eapi_has_pkg_pretend(eapi): return os.EX_OK + if not (mydo == "install" and "noauto" in mysettings.features): + check_file = os.path.join( + mysettings["PORTAGE_BUILDDIR"], ".%sed" % mydo.rstrip('e')) + if os.path.exists(check_file): + writemsg_stdout(_(">>> It appears that " + "'%(action)s' has already executed for '%(pkg)s'; skipping.\n") % + {"action":mydo, "pkg":mysettings["PF"]}) + writemsg_stdout(_(">>> Remove '%(file)s' to force %(action)s.\n") % + {"file":check_file, "action":mydo}) + return os.EX_OK + return _spawn_phase(mydo, mysettings, actionmap=actionmap, logfile=logfile, fd_pipes=fd_pipes, returnpid=returnpid) @@ -1285,13 +1482,14 @@ _post_phase_cmds = { "install" : [ "install_qa_check", - "install_symlink_html_docs"], + "install_symlink_html_docs", + "install_hooks"], "preinst" : [ "preinst_sfperms", "preinst_selinux_labels", "preinst_suid_scan", - "preinst_mask"] + ] } def _post_phase_userpriv_perms(mysettings): @@ -1320,7 +1518,9 @@ def _check_build_log(mysettings, out=None): except EnvironmentError: return + f_real = None if logfile.endswith('.gz'): + f_real = f f = gzip.GzipFile(filename='', mode='rb', fileobj=f) am_maintainer_mode = [] @@ -1425,19 +1625,32 @@ def _check_build_log(mysettings, out=None): msg.extend("\t" + line for line in make_jobserver) _eqawarn(msg) + f.close() + if f_real is not None: + f_real.close() + def _post_src_install_chost_fix(settings): """ It's possible that the ebuild has changed the CHOST variable, so revert it to the initial - setting. + setting. Also, revert IUSE in case it's corrupted + due to local environment settings like in bug #386829. """ - if settings.get('CATEGORY') == 'virtual': - return - chost = settings.get('CHOST') - if chost: - write_atomic(os.path.join(settings['PORTAGE_BUILDDIR'], - 'build-info', 'CHOST'), chost + '\n') + build_info_dir = os.path.join(settings['PORTAGE_BUILDDIR'], 'build-info') + + for k in ('IUSE',): + v = settings.get(k) + if v is not None: + write_atomic(os.path.join(build_info_dir, k), v + '\n') + + # The following variables are irrelevant for virtual packages. + if settings.get('CATEGORY') != 'virtual': + + for k in ('CHOST',): + v = settings.get(k) + if v is not None: + write_atomic(os.path.join(build_info_dir, k), v + '\n') _vdb_use_conditional_keys = ('DEPEND', 'LICENSE', 'PDEPEND', 'PROPERTIES', 'PROVIDE', 'RDEPEND', 'RESTRICT',) @@ -1481,6 +1694,7 @@ def _post_src_install_uid_fix(mysettings, out): _preinst_bsdflags(mysettings) destdir = mysettings["D"] + ed_len = len(mysettings["ED"]) unicode_errors = [] while True: @@ -1499,12 +1713,12 @@ def _post_src_install_uid_fix(mysettings, out): new_parent = _unicode_decode(parent, encoding=_encodings['merge'], errors='replace') new_parent = _unicode_encode(new_parent, - encoding=_encodings['merge'], errors='backslashreplace') + encoding='ascii', errors='backslashreplace') new_parent = _unicode_decode(new_parent, encoding=_encodings['merge'], errors='replace') os.rename(parent, new_parent) unicode_error = True - unicode_errors.append(new_parent[len(destdir):]) + unicode_errors.append(new_parent[ed_len:]) break for fname in chain(dirs, files): @@ -1517,13 +1731,13 @@ def _post_src_install_uid_fix(mysettings, out): new_fname = _unicode_decode(fname, encoding=_encodings['merge'], errors='replace') new_fname = _unicode_encode(new_fname, - encoding=_encodings['merge'], errors='backslashreplace') + encoding='ascii', errors='backslashreplace') new_fname = _unicode_decode(new_fname, encoding=_encodings['merge'], errors='replace') new_fpath = os.path.join(parent, new_fname) os.rename(fpath, new_fpath) unicode_error = True - unicode_errors.append(new_fpath[len(destdir):]) + unicode_errors.append(new_fpath[ed_len:]) fname = new_fname fpath = new_fpath else: @@ -1597,20 +1811,24 @@ def _post_src_install_uid_fix(mysettings, out): if unicode_errors: for l in _merge_unicode_error(unicode_errors): - eerror(l, phase='install', key=mysettings.mycpv, out=out) + eqawarn(l, phase='install', key=mysettings.mycpv, out=out) build_info_dir = os.path.join(mysettings['PORTAGE_BUILDDIR'], 'build-info') - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, 'SIZE'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode(str(size) + '\n')) + errors='strict') + f.write(_unicode_decode(str(size) + '\n')) + f.close() - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, 'BUILD_TIME'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode("%.0f\n" % (time.time(),))) + errors='strict') + f.write(_unicode_decode("%.0f\n" % (time.time(),))) + f.close() use = frozenset(mysettings['PORTAGE_USE'].split()) for k in _vdb_use_conditional_keys: @@ -1636,10 +1854,12 @@ def _post_src_install_uid_fix(mysettings, out): except OSError: pass continue - io.open(_unicode_encode(os.path.join(build_info_dir, + f = io.open(_unicode_encode(os.path.join(build_info_dir, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], - errors='strict').write(_unicode_decode(v + '\n')) + errors='strict') + f.write(_unicode_decode(v + '\n')) + f.close() _reapply_bsdflags_to_image(mysettings) @@ -1664,15 +1884,46 @@ def _post_src_install_soname_symlinks(mysettings, out): needed_filename = os.path.join(mysettings["PORTAGE_BUILDDIR"], "build-info", "NEEDED.ELF.2") + f = None try: - lines = io.open(_unicode_encode(needed_filename, + f = io.open(_unicode_encode(needed_filename, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace').readlines() + errors='replace') + lines = f.readlines() except IOError as e: if e.errno not in (errno.ENOENT, errno.ESTALE): raise return + finally: + if f is not None: + f.close() + + qa_no_symlink = "" + f = None + try: + f = io.open(_unicode_encode(os.path.join( + mysettings["PORTAGE_BUILDDIR"], + "build-info", "QA_SONAME_NO_SYMLINK"), + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + qa_no_symlink = f.read() + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + finally: + if f is not None: + f.close() + + qa_no_symlink = qa_no_symlink.split() + if qa_no_symlink: + if len(qa_no_symlink) > 1: + qa_no_symlink = "|".join("(%s)" % x for x in qa_no_symlink) + qa_no_symlink = "^(%s)$" % qa_no_symlink + else: + qa_no_symlink = "^%s$" % qa_no_symlink[0] + qa_no_symlink = re.compile(qa_no_symlink) libpaths = set(portage.util.getlibpaths( mysettings["ROOT"], env=mysettings)) @@ -1730,6 +1981,8 @@ def _post_src_install_soname_symlinks(mysettings, out): continue if not is_libdir(os.path.dirname(obj)): continue + if qa_no_symlink and qa_no_symlink.match(obj.strip(os.sep)) is not None: + continue obj_file_path = os.path.join(image_dir, obj.lstrip(os.sep)) sym_file_path = os.path.join(os.path.dirname(obj_file_path), soname) @@ -1746,8 +1999,7 @@ def _post_src_install_soname_symlinks(mysettings, out): if not missing_symlinks: return - qa_msg = ["QA Notice: Missing soname symlink(s) " + \ - "will be automatically created:"] + qa_msg = ["QA Notice: Missing soname symlink(s):"] qa_msg.append("") qa_msg.extend("\t%s -> %s" % (os.path.join( os.path.dirname(obj).lstrip(os.sep), soname), @@ -1757,20 +2009,11 @@ def _post_src_install_soname_symlinks(mysettings, out): for line in qa_msg: eqawarn(line, key=mysettings.mycpv, out=out) - _preinst_bsdflags(mysettings) - for obj, soname in missing_symlinks: - obj_file_path = os.path.join(image_dir, obj.lstrip(os.sep)) - sym_file_path = os.path.join(os.path.dirname(obj_file_path), soname) - os.symlink(os.path.basename(obj_file_path), sym_file_path) - _reapply_bsdflags_to_image(mysettings) - def _merge_unicode_error(errors): lines = [] - msg = _("This package installs one or more file names containing " - "characters that do not match your current locale " - "settings. The current setting for filesystem encoding is '%s'.") \ - % _encodings['merge'] + msg = _("QA Notice: This package installs one or more file names " + "containing characters that are not encoded with the UTF-8 encoding.") lines.extend(wrap(msg, 72)) lines.append("") @@ -1778,14 +2021,55 @@ def _merge_unicode_error(errors): lines.extend("\t" + x for x in errors) lines.append("") - if _encodings['merge'].lower().replace('_', '').replace('-', '') != 'utf8': - msg = _("For best results, UTF-8 encoding is recommended. See " - "the Gentoo Linux Localization Guide for instructions " - "about how to configure your locale for UTF-8 encoding:") - lines.extend(wrap(msg, 72)) - lines.append("") - lines.append("\t" + \ - "http://www.gentoo.org/doc/en/guide-localization.xml") - lines.append("") - return lines + +def _prepare_self_update(settings): + """ + Call this when portage is updating itself, in order to create + temporary copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH, since + the new versions may be incompatible. An atexit hook will + automatically clean up the temporary copies. + """ + + # sanity check: ensure that that this routine only runs once + if portage._bin_path != portage.const.PORTAGE_BIN_PATH: + return + + # Load lazily referenced portage submodules into memory, + # so imports won't fail during portage upgrade/downgrade. + _preload_elog_modules(settings) + portage.proxy.lazyimport._preload_portage_submodules() + + # Make the temp directory inside $PORTAGE_TMPDIR/portage, since + # it's common for /tmp and /var/tmp to be mounted with the + # "noexec" option (see bug #346899). + build_prefix = os.path.join(settings["PORTAGE_TMPDIR"], "portage") + portage.util.ensure_dirs(build_prefix) + base_path_tmp = tempfile.mkdtemp( + "", "._portage_reinstall_.", build_prefix) + portage.process.atexit_register(shutil.rmtree, base_path_tmp) + + orig_bin_path = portage._bin_path + portage._bin_path = os.path.join(base_path_tmp, "bin") + shutil.copytree(orig_bin_path, portage._bin_path, symlinks=True) + + orig_pym_path = portage._pym_path + portage._pym_path = os.path.join(base_path_tmp, "pym") + shutil.copytree(orig_pym_path, portage._pym_path, symlinks=True) + + for dir_path in (base_path_tmp, portage._bin_path, portage._pym_path): + os.chmod(dir_path, 0o755) + +def _handle_self_update(settings, vardb): + cpv = settings.mycpv + if settings["ROOT"] == "/" and \ + portage.dep.match_from_list( + portage.const.PORTAGE_PACKAGE_ATOM, [cpv]): + inherited = frozenset(settings.get('INHERITED', '').split()) + if not vardb.cpv_exists(cpv) or \ + '9999' in cpv or \ + 'git' in inherited or \ + 'git-2' in inherited: + _prepare_self_update(settings) + return True + return False diff --git a/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo Binary files differnew file mode 100644 index 0000000..a6ebb1d --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/doebuild.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.py b/portage_with_autodep/pym/portage/package/ebuild/fetch.py index 5cbbf87..b795b28 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/fetch.py +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 from __future__ import print_function @@ -10,7 +10,6 @@ import io import logging import random import re -import shutil import stat import sys import tempfile @@ -24,7 +23,7 @@ portage.proxy.lazyimport.lazyimport(globals(), 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', ) -from portage import OrderedDict, os, selinux, _encodings, \ +from portage import OrderedDict, os, selinux, shutil, _encodings, \ _shell_quote, _unicode_encode from portage.checksum import hashfunc_map, perform_md5, verify_all from portage.const import BASH_BINARY, CUSTOM_MIRRORS_FILE, \ @@ -34,7 +33,6 @@ from portage.exception import FileNotFound, OperationNotPermitted, \ PortageException, TryAgain from portage.localization import _ from portage.locks import lockfile, unlockfile -from portage.manifest import Manifest from portage.output import colorize, EOutput from portage.util import apply_recursive_permissions, \ apply_secpass_permissions, ensure_dirs, grabdict, shlex_split, \ @@ -48,6 +46,9 @@ _userpriv_spawn_kwargs = ( ("umask", 0o02), ) +def _hide_url_passwd(url): + return re.sub(r'//(.+):.+@(.+)', r'//\1:*password*@\2', url) + def _spawn_fetch(settings, args, **kwargs): """ Spawn a process with appropriate settings for fetching, including @@ -68,7 +69,8 @@ def _spawn_fetch(settings, args, **kwargs): } if "userfetch" in settings.features and \ - os.getuid() == 0 and portage_gid and portage_uid: + os.getuid() == 0 and portage_gid and portage_uid and \ + hasattr(os, "setgroups"): kwargs.update(_userpriv_spawn_kwargs) spawn_func = spawn @@ -356,7 +358,8 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, allow_missing_digests = True pkgdir = mysettings.get("O") if digests is None and not (pkgdir is None or skip_manifest): - mydigests = Manifest( + mydigests = mysettings.repositories.get_repo_for_location( + os.path.dirname(os.path.dirname(pkgdir))).load_manifest( pkgdir, mysettings["DISTDIR"]).getTypeDigests("DIST") elif digests is None or skip_manifest: # no digests because fetch was not called for a specific package @@ -612,18 +615,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, elif userfetch: has_space = False - if not has_space: - writemsg(_("!!! Insufficient space to store %s in %s\n") % \ - (myfile, mysettings["DISTDIR"]), noiselevel=-1) - - if has_space_superuser: - writemsg(_("!!! Insufficient privileges to use " - "remaining space.\n"), noiselevel=-1) - if userfetch: - writemsg(_("!!! You may set FEATURES=\"-userfetch\"" - " in /etc/make.conf in order to fetch with\n" - "!!! superuser privileges.\n"), noiselevel=-1) - if distdir_writable and use_locks: lock_kwargs = {} @@ -646,7 +637,10 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, match, mystat = _check_distfile( myfile_path, pruned_digests, eout) if match: - if distdir_writable: + # Skip permission adjustment for symlinks, since we don't + # want to modify anything outside of the primary DISTDIR, + # and symlinks typically point to PORTAGE_RO_DISTDIRS. + if distdir_writable and not os.path.islink(myfile_path): try: apply_secpass_permissions(myfile_path, gid=portage_gid, mode=0o664, mask=0o2, @@ -727,6 +721,20 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, os.symlink(readonly_file, myfile_path) continue + # this message is shown only after we know that + # the file is not already fetched + if not has_space: + writemsg(_("!!! Insufficient space to store %s in %s\n") % \ + (myfile, mysettings["DISTDIR"]), noiselevel=-1) + + if has_space_superuser: + writemsg(_("!!! Insufficient privileges to use " + "remaining space.\n"), noiselevel=-1) + if userfetch: + writemsg(_("!!! You may set FEATURES=\"-userfetch\"" + " in /etc/make.conf in order to fetch with\n" + "!!! superuser privileges.\n"), noiselevel=-1) + if fsmirrors and not os.path.exists(myfile_path) and has_space: for mydir in fsmirrors: mirror_file = os.path.join(mydir, myfile) @@ -746,14 +754,18 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, raise del e else: - try: - apply_secpass_permissions( - myfile_path, gid=portage_gid, mode=0o664, mask=0o2, - stat_cached=mystat) - except PortageException as e: - if not os.access(myfile_path, os.R_OK): - writemsg(_("!!! Failed to adjust permissions:" - " %s\n") % str(e), noiselevel=-1) + # Skip permission adjustment for symlinks, since we don't + # want to modify anything outside of the primary DISTDIR, + # and symlinks typically point to PORTAGE_RO_DISTDIRS. + if not os.path.islink(myfile_path): + try: + apply_secpass_permissions(myfile_path, + gid=portage_gid, mode=0o664, mask=0o2, + stat_cached=mystat) + except PortageException as e: + if not os.access(myfile_path, os.R_OK): + writemsg(_("!!! Failed to adjust permissions:" + " %s\n") % (e,), noiselevel=-1) # If the file is empty then it's obviously invalid. Remove # the empty file and try to download if possible. @@ -940,7 +952,7 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, locfetch=fetchcommand command_var = fetchcommand_var writemsg_stdout(_(">>> Downloading '%s'\n") % \ - re.sub(r'//(.+):.+@(.+)/',r'//\1:*password*@\2/', loc)) + _hide_url_passwd(loc)) variables = { "DISTDIR": mysettings["DISTDIR"], "URI": loc, @@ -1019,18 +1031,19 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # Fetch failed... Try the next one... Kill 404 files though. if (mystat[stat.ST_SIZE]<100000) and (len(myfile)>4) and not ((myfile[-5:]==".html") or (myfile[-4:]==".htm")): html404=re.compile("<title>.*(not found|404).*</title>",re.I|re.M) - if html404.search(io.open( + with io.open( _unicode_encode(myfile_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace' - ).read()): - try: - os.unlink(mysettings["DISTDIR"]+"/"+myfile) - writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n")) - fetched = 0 - continue - except (IOError, OSError): - pass + ) as f: + if html404.search(f.read()): + try: + os.unlink(mysettings["DISTDIR"]+"/"+myfile) + writemsg(_(">>> Deleting invalid distfile. (Improper 404 redirect from server.)\n")) + fetched = 0 + continue + except (IOError, OSError): + pass fetched = 1 continue if True: @@ -1040,7 +1053,6 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0, # from another mirror... verified_ok,reason = verify_all(mysettings["DISTDIR"]+"/"+myfile, mydigests[myfile]) if not verified_ok: - print(reason) writemsg(_("!!! Fetched file: %s VERIFY FAILED!\n") % myfile, noiselevel=-1) writemsg(_("!!! Reason: %s\n") % reason[0], diff --git a/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo Binary files differnew file mode 100644 index 0000000..3bd81df --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/fetch.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py index f2af638..8a88c2f 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.py @@ -83,7 +83,13 @@ def getmaskingreason(mycpv, metadata=None, settings=None, pmasklists = [] for profile in locations: pmask_filename = os.path.join(profile, "package.mask") - pmasklists.append((pmask_filename, grablines(pmask_filename, recursive=1))) + node = None + for l, recursive_filename in grablines(pmask_filename, + recursive=1, remember_source_file=True): + if node is None or node[0] != recursive_filename: + node = (recursive_filename, []) + pmasklists.append(node) + node[1].append(l) pmaskdict = settings._mask_manager._pmaskdict if mycp in pmaskdict: diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo Binary files differnew file mode 100644 index 0000000..1614244 --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingreason.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py index 4c65fcc..9bf605d 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['getmaskingstatus'] @@ -7,11 +7,9 @@ import sys import portage from portage import eapi_is_supported, _eapi_is_deprecated -from portage.dep import match_from_list, _slot_separator, _repo_separator from portage.localization import _ from portage.package.ebuild.config import config -from portage.versions import catpkgsplit, cpv_getkey -from _emerge.Package import Package +from portage.versions import catpkgsplit, _pkg_str if sys.hexversion >= 0x3000000: basestring = str @@ -53,9 +51,6 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): metadata = pkg.metadata installed = pkg.installed - mysplit = catpkgsplit(mycpv) - if not mysplit: - raise ValueError(_("invalid CPV: %s") % mycpv) if metadata is None: db_keys = list(portdb._aux_cache_keys) try: @@ -70,11 +65,14 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): else: metadata["USE"] = "" - rValue = [] + if not hasattr(mycpv, 'slot'): + try: + mycpv = _pkg_str(mycpv, slot=metadata['SLOT'], + repo=metadata.get('repository')) + except portage.exception.InvalidData: + raise ValueError(_("invalid CPV: %s") % mycpv) - # profile checking - if settings._getProfileMaskAtom(mycpv, metadata): - rValue.append(_MaskReason("profile", "profile")) + rValue = [] # package.mask checking if settings._getMaskAtom(mycpv, metadata): @@ -85,8 +83,6 @@ def _getmaskingstatus(mycpv, settings, portdb, myrepo=None): mygroups = settings._getKeywords(mycpv, metadata) licenses = metadata["LICENSE"] properties = metadata["PROPERTIES"] - if eapi.startswith("-"): - eapi = eapi[1:] if not eapi_is_supported(eapi): return [_MaskReason("EAPI", "EAPI %s" % eapi)] elif _eapi_is_deprecated(eapi) and not installed: diff --git a/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo Binary files differnew file mode 100644 index 0000000..9cf1d9d --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/getmaskingstatus.pyo diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py index 616dc2e..b8fbdc5 100644 --- a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.py @@ -5,11 +5,11 @@ __all__ = ['prepare_build_dirs'] import errno import gzip -import shutil import stat import time -from portage import os, _encodings, _unicode_encode, _unicode_decode +import portage +from portage import os, shutil, _encodings, _unicode_encode, _unicode_decode from portage.data import portage_gid, portage_uid, secpass from portage.exception import DirectoryNotFound, FileNotFound, \ OperationNotPermitted, PermissionDenied, PortageException @@ -118,11 +118,13 @@ def _adjust_perms_msg(settings, msg): background = settings.get("PORTAGE_BACKGROUND") == "1" log_path = settings.get("PORTAGE_LOG_FILE") log_file = None + log_file_real = None if background and log_path is not None: try: log_file = open(_unicode_encode(log_path, encoding=_encodings['fs'], errors='strict'), mode='ab') + log_file_real = log_file except IOError: def write(msg): pass @@ -139,6 +141,8 @@ def _adjust_perms_msg(settings, msg): finally: if log_file is not None: log_file.close() + if log_file_real is not log_file: + log_file_real.close() def _prepare_features_dirs(mysettings): @@ -311,7 +315,7 @@ def _prepare_workdir(mysettings): logdir = normalize_path(mysettings["PORT_LOGDIR"]) logid_path = os.path.join(mysettings["PORTAGE_BUILDDIR"], ".logid") if not os.path.exists(logid_path): - open(_unicode_encode(logid_path), 'w') + open(_unicode_encode(logid_path), 'w').close() logid_time = _unicode_decode(time.strftime("%Y%m%d-%H%M%S", time.gmtime(os.stat(logid_path).st_mtime)), encoding=_encodings['content'], errors='replace') @@ -342,13 +346,31 @@ def _prepare_workdir(mysettings): writemsg(_unicode_decode("!!! %s: %s\n") % (_("Permission Denied"), log_subdir), noiselevel=-1) + tmpdir_log_path = os.path.join( + mysettings["T"], "build.log%s" % compress_log_ext) if not logdir_subdir_ok: # NOTE: When sesandbox is enabled, the local SELinux security policies # may not allow output to be piped out of the sesandbox domain. The # current policy will allow it to work when a pty is available, but # not through a normal pipe. See bug #162404. - mysettings["PORTAGE_LOG_FILE"] = os.path.join( - mysettings["T"], "build.log%s" % compress_log_ext) + mysettings["PORTAGE_LOG_FILE"] = tmpdir_log_path + else: + # Create a symlink from tmpdir_log_path to PORTAGE_LOG_FILE, as + # requested in bug #412865. + make_new_symlink = False + try: + target = os.readlink(tmpdir_log_path) + except OSError: + make_new_symlink = True + else: + if target != mysettings["PORTAGE_LOG_FILE"]: + make_new_symlink = True + if make_new_symlink: + try: + os.unlink(tmpdir_log_path) + except OSError: + pass + os.symlink(mysettings["PORTAGE_LOG_FILE"], tmpdir_log_path) def _ensure_log_subdirs(logdir, subdir): """ @@ -358,13 +380,27 @@ def _ensure_log_subdirs(logdir, subdir): and subdir are assumed to be normalized absolute paths. """ st = os.stat(logdir) + uid = -1 gid = st.st_gid grp_mode = 0o2070 & st.st_mode + # If logdir is writable by the portage group but its uid + # is not portage_uid, then set the uid to portage_uid if + # we have privileges to do so, for compatibility with our + # default logrotate config (see bug 378451). With the + # "su portage portage" directive and logrotate-3.8.0, + # logrotate's chown call during the compression phase will + # only succeed if the log file's uid is portage_uid. + if grp_mode and gid == portage_gid and \ + portage.data.secpass >= 2: + uid = portage_uid + if st.st_uid != portage_uid: + ensure_dirs(logdir, uid=uid) + logdir_split_len = len(logdir.split(os.sep)) subdir_split = subdir.split(os.sep)[logdir_split_len:] subdir_split.reverse() current = logdir while subdir_split: current = os.path.join(current, subdir_split.pop()) - ensure_dirs(current, gid=gid, mode=grp_mode, mask=0) + ensure_dirs(current, uid=uid, gid=gid, mode=grp_mode, mask=0) diff --git a/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo Binary files differnew file mode 100644 index 0000000..2dcfaea --- /dev/null +++ b/portage_with_autodep/pym/portage/package/ebuild/prepare_build_dirs.pyo diff --git a/portage_with_autodep/pym/portage/process.py b/portage_with_autodep/pym/portage/process.py index 6866a2f..d7d1037 100644 --- a/portage_with_autodep/pym/portage/process.py +++ b/portage_with_autodep/pym/portage/process.py @@ -1,9 +1,11 @@ # portage.py -- core Portage functionality -# Copyright 1998-2010 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import atexit +import errno +import platform import signal import sys import traceback @@ -32,6 +34,18 @@ if os.path.isdir("/proc/%i/fd" % os.getpid()): def get_open_fds(): return (int(fd) for fd in os.listdir("/proc/%i/fd" % os.getpid()) \ if fd.isdigit()) + + if platform.python_implementation() == 'PyPy': + # EAGAIN observed with PyPy 1.8. + _get_open_fds = get_open_fds + def get_open_fds(): + try: + return _get_open_fds() + except OSError as e: + if e.errno != errno.EAGAIN: + raise + return range(max_fd_limit) + else: def get_open_fds(): return range(max_fd_limit) @@ -257,7 +271,7 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, pid = os.fork() - if not pid: + if pid == 0: try: _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, pre_exec) @@ -272,6 +286,9 @@ def spawn(mycommand, env={}, opt_name=None, fd_pipes=None, returnpid=False, sys.stderr.flush() os._exit(1) + if not isinstance(pid, int): + raise AssertionError("fork returned non-integer: %s" % (repr(pid),)) + # Add the pid to our local and the global pid lists. mypids.append(pid) spawned_pids.append(pid) @@ -350,13 +367,18 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, @param pre_exec: A function to be called with no arguments just prior to the exec call. @type pre_exec: callable @rtype: None - @returns: Never returns (calls os.execve) + @return: Never returns (calls os.execve) """ # If the process we're creating hasn't been given a name # assign it the name of the executable. if not opt_name: - opt_name = os.path.basename(binary) + if binary is portage._python_interpreter: + # NOTE: PyPy 1.7 will die due to "libary path not found" if argv[0] + # does not contain the full path of the binary. + opt_name = binary + else: + opt_name = os.path.basename(binary) # Set up the command's argument list. myargs = [opt_name] @@ -391,8 +413,20 @@ def _exec(binary, mycommand, opt_name, fd_pipes, env, gid, groups, uid, umask, # And switch to the new process. os.execve(binary, myargs, env) -def _setup_pipes(fd_pipes): - """Setup pipes for a forked process.""" +def _setup_pipes(fd_pipes, close_fds=True): + """Setup pipes for a forked process. + + WARNING: When not followed by exec, the close_fds behavior + can trigger interference from destructors that close file + descriptors. This interference happens when the garbage + collector intermittently executes such destructors after their + corresponding file descriptors have been re-used, leading + to intermittent "[Errno 9] Bad file descriptor" exceptions in + forked processes. This problem has been observed with PyPy 1.8, + and also with CPython under some circumstances (as triggered + by xmpppy in bug #374335). In order to close a safe subset of + file descriptors, see portage.locks._close_fds(). + """ my_fds = {} # To protect from cases where direct assignment could # clobber needed fds ({1:2, 2:1}) we first dupe the fds @@ -402,14 +436,16 @@ def _setup_pipes(fd_pipes): # Then assign them to what they should be. for fd in my_fds: os.dup2(my_fds[fd], fd) - # Then close _all_ fds that haven't been explicitly - # requested to be kept open. - for fd in get_open_fds(): - if fd not in my_fds: - try: - os.close(fd) - except OSError: - pass + + if close_fds: + # Then close _all_ fds that haven't been explicitly + # requested to be kept open. + for fd in get_open_fds(): + if fd not in my_fds: + try: + os.close(fd) + except OSError: + pass def find_binary(binary): """ @@ -418,7 +454,7 @@ def find_binary(binary): @param binary: Name of the binary to find @type string @rtype: None or string - @returns: full path to binary or None if the binary could not be located. + @return: full path to binary or None if the binary could not be located. """ for path in os.environ.get("PATH", "").split(":"): filename = "%s/%s" % (path, binary) diff --git a/portage_with_autodep/pym/portage/process.pyo b/portage_with_autodep/pym/portage/process.pyo Binary files differnew file mode 100644 index 0000000..c53af30 --- /dev/null +++ b/portage_with_autodep/pym/portage/process.pyo diff --git a/portage_with_autodep/pym/portage/proxy/__init__.pyo b/portage_with_autodep/pym/portage/proxy/__init__.pyo Binary files differnew file mode 100644 index 0000000..b3d096b --- /dev/null +++ b/portage_with_autodep/pym/portage/proxy/__init__.pyo diff --git a/portage_with_autodep/pym/portage/proxy/lazyimport.pyo b/portage_with_autodep/pym/portage/proxy/lazyimport.pyo Binary files differnew file mode 100644 index 0000000..9da8089 --- /dev/null +++ b/portage_with_autodep/pym/portage/proxy/lazyimport.pyo diff --git a/portage_with_autodep/pym/portage/proxy/objectproxy.pyo b/portage_with_autodep/pym/portage/proxy/objectproxy.pyo Binary files differnew file mode 100644 index 0000000..f0919ff --- /dev/null +++ b/portage_with_autodep/pym/portage/proxy/objectproxy.pyo diff --git a/portage_with_autodep/pym/portage/repository/__init__.pyo b/portage_with_autodep/pym/portage/repository/__init__.pyo Binary files differnew file mode 100644 index 0000000..ab526c0 --- /dev/null +++ b/portage_with_autodep/pym/portage/repository/__init__.pyo diff --git a/portage_with_autodep/pym/portage/repository/config.py b/portage_with_autodep/pym/portage/repository/config.py index 9f0bb99..20f1919 100644 --- a/portage_with_autodep/pym/portage/repository/config.py +++ b/portage_with_autodep/pym/portage/repository/config.py @@ -1,21 +1,37 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import io import logging +import warnings +import sys import re try: - from configparser import SafeConfigParser, ParsingError + from configparser import ParsingError + if sys.hexversion >= 0x3020000: + from configparser import ConfigParser as SafeConfigParser + else: + from configparser import SafeConfigParser except ImportError: from ConfigParser import SafeConfigParser, ParsingError -from portage import os -from portage.const import USER_CONFIG_PATH, REPO_NAME_LOC +from portage import eclass_cache, os +from portage.const import (MANIFEST2_HASH_FUNCTIONS, MANIFEST2_REQUIRED_HASH, + REPO_NAME_LOC, USER_CONFIG_PATH) from portage.env.loaders import KeyValuePairFileLoader -from portage.util import normalize_path, writemsg, writemsg_level, shlex_split +from portage.util import (normalize_path, read_corresponding_eapi_file, shlex_split, + stack_lists, writemsg, writemsg_level) from portage.localization import _ +from portage import _unicode_decode from portage import _unicode_encode from portage import _encodings +from portage import manifest + +_valid_profile_formats = frozenset( + ['pms', 'portage-1', 'portage-2']) + +_portage1_profiles_allow_directories = frozenset( + ["portage-1-compat", "portage-1", 'portage-2']) _repo_name_sub_re = re.compile(r'[^\w-]') @@ -35,8 +51,13 @@ def _gen_valid_repo(name): class RepoConfig(object): """Stores config of one repository""" - __slots__ = ['aliases', 'eclass_overrides', 'eclass_locations', 'location', 'user_location', 'masters', 'main_repo', - 'missing_repo_name', 'name', 'priority', 'sync', 'format'] + __slots__ = ('aliases', 'allow_missing_manifest', 'allow_provide_virtual', + 'cache_formats', 'create_manifest', 'disable_manifest', 'eapi', + 'eclass_db', 'eclass_locations', 'eclass_overrides', 'format', 'location', + 'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name', + 'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority', + 'profile_formats', 'sign_commit', 'sign_manifest', 'sync', + 'thin_manifest', 'update_changelog', 'user_location') def __init__(self, name, repo_opts): """Build a RepoConfig with options in repo_opts @@ -51,11 +72,15 @@ class RepoConfig(object): if eclass_overrides is not None: eclass_overrides = tuple(eclass_overrides.split()) self.eclass_overrides = eclass_overrides - #Locations are computed later. + # Eclass databases and locations are computed later. + self.eclass_db = None self.eclass_locations = None - #Masters are only read from layout.conf. - self.masters = None + # Masters from repos.conf override layout.conf. + masters = repo_opts.get('masters') + if masters is not None: + masters = tuple(masters.split()) + self.masters = masters #The main-repo key makes only sense for the 'DEFAULT' section. self.main_repo = repo_opts.get('main-repo') @@ -87,59 +112,153 @@ class RepoConfig(object): location = None self.location = location + eapi = None missing = True if self.location is not None: - name, missing = self._read_repo_name(self.location) - # We must ensure that the name conforms to PMS 3.1.5 - # in order to avoid InvalidAtom exceptions when we - # use it to generate atoms. - name = _gen_valid_repo(name) - if not name: - # name only contains invalid characters - name = "x-" + os.path.basename(self.location) - name = _gen_valid_repo(name) - # If basename only contains whitespace then the - # end result is name = 'x-'. - + eapi = read_corresponding_eapi_file(os.path.join(self.location, REPO_NAME_LOC)) + name, missing = self._read_valid_repo_name(self.location) elif name == "DEFAULT": missing = False + + self.eapi = eapi self.name = name self.missing_repo_name = missing + # sign_commit is disabled by default, since it requires Git >=1.7.9, + # and key_id configured by `git config user.signingkey key_id` + self.sign_commit = False + self.sign_manifest = True + self.thin_manifest = False + self.allow_missing_manifest = False + self.allow_provide_virtual = False + self.create_manifest = True + self.disable_manifest = False + self.manifest_hashes = None + self.update_changelog = False + self.cache_formats = None + self.portage1_profiles = True + self.portage1_profiles_compat = False + + # Parse layout.conf. + if self.location: + layout_filename = os.path.join(self.location, "metadata", "layout.conf") + layout_data = parse_layout_conf(self.location, self.name)[0] + + # layout.conf masters may be overridden here if we have a masters + # setting from the user's repos.conf + if self.masters is None: + self.masters = layout_data['masters'] + + if layout_data['aliases']: + aliases = self.aliases + if aliases is None: + aliases = () + # repos.conf aliases come after layout.conf aliases, giving + # them the ability to do incremental overrides + self.aliases = layout_data['aliases'] + tuple(aliases) + + for value in ('allow-missing-manifest', + 'allow-provide-virtual', 'cache-formats', + 'create-manifest', 'disable-manifest', 'manifest-hashes', + 'profile-formats', + 'sign-commit', 'sign-manifest', 'thin-manifest', 'update-changelog'): + setattr(self, value.lower().replace("-", "_"), layout_data[value]) + + self.portage1_profiles = any(x in _portage1_profiles_allow_directories + for x in layout_data['profile-formats']) + self.portage1_profiles_compat = layout_data['profile-formats'] == ('portage-1-compat',) + + def iter_pregenerated_caches(self, auxdbkeys, readonly=True, force=False): + """ + Reads layout.conf cache-formats from left to right and yields cache + instances for each supported type that's found. If no cache-formats + are specified in layout.conf, 'pms' type is assumed if the + metadata/cache directory exists or force is True. + """ + formats = self.cache_formats + if not formats: + if not force: + return + formats = ('pms',) + + for fmt in formats: + name = None + if fmt == 'pms': + from portage.cache.metadata import database + name = 'metadata/cache' + elif fmt == 'md5-dict': + from portage.cache.flat_hash import md5_database as database + name = 'metadata/md5-cache' + + if name is not None: + yield database(self.location, name, + auxdbkeys, readonly=readonly) + + def get_pregenerated_cache(self, auxdbkeys, readonly=True, force=False): + """ + Returns the first cache instance yielded from + iter_pregenerated_caches(), or None if no cache is available or none + of the available formats are supported. + """ + return next(self.iter_pregenerated_caches( + auxdbkeys, readonly=readonly, force=force), None) + + def load_manifest(self, *args, **kwds): + kwds['thin'] = self.thin_manifest + kwds['allow_missing'] = self.allow_missing_manifest + kwds['allow_create'] = self.create_manifest + kwds['hashes'] = self.manifest_hashes + if self.disable_manifest: + kwds['from_scratch'] = True + return manifest.Manifest(*args, **kwds) def update(self, new_repo): """Update repository with options in another RepoConfig""" - if new_repo.aliases is not None: - self.aliases = new_repo.aliases - if new_repo.eclass_overrides is not None: - self.eclass_overrides = new_repo.eclass_overrides - if new_repo.masters is not None: - self.masters = new_repo.masters + + keys = set(self.__slots__) + keys.discard("missing_repo_name") + for k in keys: + v = getattr(new_repo, k, None) + if v is not None: + setattr(self, k, v) + if new_repo.name is not None: - self.name = new_repo.name self.missing_repo_name = new_repo.missing_repo_name - if new_repo.user_location is not None: - self.user_location = new_repo.user_location - if new_repo.location is not None: - self.location = new_repo.location - if new_repo.priority is not None: - self.priority = new_repo.priority - if new_repo.sync is not None: - self.sync = new_repo.sync - - def _read_repo_name(self, repo_path): + + @staticmethod + def _read_valid_repo_name(repo_path): + name, missing = RepoConfig._read_repo_name(repo_path) + # We must ensure that the name conforms to PMS 3.1.5 + # in order to avoid InvalidAtom exceptions when we + # use it to generate atoms. + name = _gen_valid_repo(name) + if not name: + # name only contains invalid characters + name = "x-" + os.path.basename(repo_path) + name = _gen_valid_repo(name) + # If basename only contains whitespace then the + # end result is name = 'x-'. + return name, missing + + @staticmethod + def _read_repo_name(repo_path): """ Read repo_name from repo_path. Returns repo_name, missing. """ repo_name_path = os.path.join(repo_path, REPO_NAME_LOC) + f = None try: - return io.open( + f = io.open( _unicode_encode(repo_name_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace').readline().strip(), False + errors='replace') + return f.readline().strip(), False except EnvironmentError: return "x-" + os.path.basename(repo_path), True + finally: + if f is not None: + f.close() def info_string(self): """ @@ -167,109 +286,154 @@ class RepoConfig(object): repo_msg.append("") return "\n".join(repo_msg) + def __repr__(self): + return "<portage.repository.config.RepoConfig(name='%s', location='%s')>" % (self.name, _unicode_decode(self.location)) + + def __str__(self): + d = {} + for k in self.__slots__: + d[k] = getattr(self, k, None) + return _unicode_decode("%s") % (d,) + + if sys.hexversion < 0x3000000: + + __unicode__ = __str__ + + def __str__(self): + return _unicode_encode(self.__unicode__()) + class RepoConfigLoader(object): """Loads and store config of several repositories, loaded from PORTDIR_OVERLAY or repos.conf""" - def __init__(self, paths, settings): - """Load config from files in paths""" - def parse(paths, prepos, ignored_map, ignored_location_map): - """Parse files in paths to load config""" - parser = SafeConfigParser() - try: - parser.read(paths) - except ParsingError as e: - writemsg(_("!!! Error while reading repo config file: %s\n") % e, noiselevel=-1) - prepos['DEFAULT'] = RepoConfig("DEFAULT", parser.defaults()) - for sname in parser.sections(): - optdict = {} - for oname in parser.options(sname): - optdict[oname] = parser.get(sname, oname) - - repo = RepoConfig(sname, optdict) - if repo.location and not os.path.exists(repo.location): - writemsg(_("!!! Invalid repos.conf entry '%s'" - " (not a dir): '%s'\n") % (sname, repo.location), noiselevel=-1) - continue - if repo.name in prepos: - old_location = prepos[repo.name].location - if old_location is not None and repo.location is not None and old_location != repo.location: - ignored_map.setdefault(repo.name, []).append(old_location) - ignored_location_map[old_location] = repo.name - prepos[repo.name].update(repo) - else: + @staticmethod + def _add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map): + """Add overlays in PORTDIR_OVERLAY as repositories""" + overlays = [] + if portdir: + portdir = normalize_path(portdir) + overlays.append(portdir) + try: + port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)] + except ValueError as e: + #File "/usr/lib/python3.2/shlex.py", line 168, in read_token + # raise ValueError("No closing quotation") + writemsg(_("!!! Invalid PORTDIR_OVERLAY:" + " %s: %s\n") % (e, portdir_overlay), noiselevel=-1) + port_ov = [] + overlays.extend(port_ov) + default_repo_opts = {} + if prepos['DEFAULT'].aliases is not None: + default_repo_opts['aliases'] = \ + ' '.join(prepos['DEFAULT'].aliases) + if prepos['DEFAULT'].eclass_overrides is not None: + default_repo_opts['eclass-overrides'] = \ + ' '.join(prepos['DEFAULT'].eclass_overrides) + if prepos['DEFAULT'].masters is not None: + default_repo_opts['masters'] = \ + ' '.join(prepos['DEFAULT'].masters) + + if overlays: + # We need a copy of the original repos.conf data, since we're + # going to modify the prepos dict and some of the RepoConfig + # objects that we put in prepos may have to be discarded if + # they get overridden by a repository with the same name but + # a different location. This is common with repoman, for example, + # when temporarily overriding an rsync repo with another copy + # of the same repo from CVS. + repos_conf = prepos.copy() + #overlay priority is negative because we want them to be looked before any other repo + base_priority = 0 + for ov in overlays: + if os.path.isdir(ov): + repo_opts = default_repo_opts.copy() + repo_opts['location'] = ov + repo = RepoConfig(None, repo_opts) + # repos_conf_opts contains options from repos.conf + repos_conf_opts = repos_conf.get(repo.name) + if repos_conf_opts is not None: + # Selectively copy only the attributes which + # repos.conf is allowed to override. + for k in ('aliases', 'eclass_overrides', 'masters', 'priority'): + v = getattr(repos_conf_opts, k, None) + if v is not None: + setattr(repo, k, v) + + if repo.name in prepos: + old_location = prepos[repo.name].location + if old_location is not None and old_location != repo.location: + ignored_map.setdefault(repo.name, []).append(old_location) + ignored_location_map[old_location] = repo.name + if old_location == portdir: + portdir = repo.user_location + + if ov == portdir and portdir not in port_ov: + repo.priority = -1000 + elif repo.priority is None: + repo.priority = base_priority + base_priority += 1 + prepos[repo.name] = repo + else: + writemsg(_("!!! Invalid PORTDIR_OVERLAY" + " (not a dir): '%s'\n") % ov, noiselevel=-1) - def add_overlays(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map): - """Add overlays in PORTDIR_OVERLAY as repositories""" - overlays = [] - if portdir: - portdir = normalize_path(portdir) - overlays.append(portdir) - port_ov = [normalize_path(i) for i in shlex_split(portdir_overlay)] - overlays.extend(port_ov) - default_repo_opts = {} - if prepos['DEFAULT'].aliases is not None: - default_repo_opts['aliases'] = \ - ' '.join(prepos['DEFAULT'].aliases) - if prepos['DEFAULT'].eclass_overrides is not None: - default_repo_opts['eclass-overrides'] = \ - ' '.join(prepos['DEFAULT'].eclass_overrides) - if prepos['DEFAULT'].masters is not None: - default_repo_opts['masters'] = \ - ' '.join(prepos['DEFAULT'].masters) - if overlays: - #overlay priority is negative because we want them to be looked before any other repo - base_priority = 0 - for ov in overlays: - if os.path.isdir(ov): - repo_opts = default_repo_opts.copy() - repo_opts['location'] = ov - repo = RepoConfig(None, repo_opts) - repo_conf_opts = prepos.get(repo.name) - if repo_conf_opts is not None: - if repo_conf_opts.aliases is not None: - repo_opts['aliases'] = \ - ' '.join(repo_conf_opts.aliases) - if repo_conf_opts.eclass_overrides is not None: - repo_opts['eclass-overrides'] = \ - ' '.join(repo_conf_opts.eclass_overrides) - if repo_conf_opts.masters is not None: - repo_opts['masters'] = \ - ' '.join(repo_conf_opts.masters) - repo = RepoConfig(repo.name, repo_opts) - if repo.name in prepos: - old_location = prepos[repo.name].location - if old_location is not None and old_location != repo.location: - ignored_map.setdefault(repo.name, []).append(old_location) - ignored_location_map[old_location] = repo.name - if old_location == portdir: - portdir = repo.user_location - prepos[repo.name].update(repo) - repo = prepos[repo.name] - else: - prepos[repo.name] = repo - - if ov == portdir and portdir not in port_ov: - repo.priority = -1000 - else: - repo.priority = base_priority - base_priority += 1 + return portdir - else: - writemsg(_("!!! Invalid PORTDIR_OVERLAY" - " (not a dir): '%s'\n") % ov, noiselevel=-1) + @staticmethod + def _parse(paths, prepos, ignored_map, ignored_location_map): + """Parse files in paths to load config""" + parser = SafeConfigParser() - return portdir + # use read_file/readfp in order to control decoding of unicode + try: + # Python >=3.2 + read_file = parser.read_file + except AttributeError: + read_file = parser.readfp - def repo_priority(r): - """ - Key funtion for comparing repositories by priority. - None is equal priority zero. - """ - x = prepos[r].priority - if x is None: - return 0 - return x + for p in paths: + f = None + try: + f = io.open(_unicode_encode(p, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + except EnvironmentError: + pass + else: + try: + read_file(f) + except ParsingError as e: + writemsg(_unicode_decode( + _("!!! Error while reading repo config file: %s\n") + ) % e, noiselevel=-1) + finally: + if f is not None: + f.close() + + prepos['DEFAULT'] = RepoConfig("DEFAULT", parser.defaults()) + for sname in parser.sections(): + optdict = {} + for oname in parser.options(sname): + optdict[oname] = parser.get(sname, oname) + + repo = RepoConfig(sname, optdict) + if repo.location and not os.path.exists(repo.location): + writemsg(_("!!! Invalid repos.conf entry '%s'" + " (not a dir): '%s'\n") % (sname, repo.location), noiselevel=-1) + continue + + if repo.name in prepos: + old_location = prepos[repo.name].location + if old_location is not None and repo.location is not None and old_location != repo.location: + ignored_map.setdefault(repo.name, []).append(old_location) + ignored_location_map[old_location] = repo.name + prepos[repo.name].update(repo) + else: + prepos[repo.name] = repo + + def __init__(self, paths, settings): + """Load config from files in paths""" prepos = {} location_map = {} @@ -279,10 +443,12 @@ class RepoConfigLoader(object): portdir = settings.get('PORTDIR', '') portdir_overlay = settings.get('PORTDIR_OVERLAY', '') - parse(paths, prepos, ignored_map, ignored_location_map) + + self._parse(paths, prepos, ignored_map, ignored_location_map) + # If PORTDIR_OVERLAY contains a repo with the same repo_name as # PORTDIR, then PORTDIR is overridden. - portdir = add_overlays(portdir, portdir_overlay, prepos, + portdir = self._add_repositories(portdir, portdir_overlay, prepos, ignored_map, ignored_location_map) if portdir and portdir.strip(): portdir = os.path.realpath(portdir) @@ -294,38 +460,14 @@ class RepoConfigLoader(object): for repo in prepos.values() if repo.location is not None and repo.missing_repo_name) - #Parse layout.conf and read masters key. - for repo in prepos.values(): - if not repo.location: - continue - layout_filename = os.path.join(repo.location, "metadata", "layout.conf") - layout_file = KeyValuePairFileLoader(layout_filename, None, None) - layout_data, layout_errors = layout_file.load() - - masters = layout_data.get('masters') - if masters and masters.strip(): - masters = masters.split() - else: - masters = None - repo.masters = masters - - aliases = layout_data.get('aliases') - if aliases and aliases.strip(): - aliases = aliases.split() - else: - aliases = None - if aliases: - if repo.aliases: - aliases.extend(repo.aliases) - repo.aliases = tuple(sorted(set(aliases))) - #Take aliases into account. new_prepos = {} for repo_name, repo in prepos.items(): names = set() names.add(repo_name) if repo.aliases: - names.update(repo.aliases) + aliases = stack_lists([repo.aliases], incremental=True) + names.update(aliases) for name in names: if name in new_prepos: @@ -342,16 +484,11 @@ class RepoConfigLoader(object): # filter duplicates from aliases, by only including # items where repo.name == key - prepos_order = [repo.name for key, repo in prepos.items() \ - if repo.name == key and repo.location is not None] - prepos_order.sort(key=repo_priority) - if portdir in location_map: - portdir_repo = prepos[location_map[portdir]] - portdir_sync = settings.get('SYNC', '') - #if SYNC variable is set and not overwritten by repos.conf - if portdir_sync and not portdir_repo.sync: - portdir_repo.sync = portdir_sync + prepos_order = sorted(prepos.items(), key=lambda r:r[1].priority or 0) + + prepos_order = [repo.name for (key, repo) in prepos_order + if repo.name == key and repo.location is not None] if prepos['DEFAULT'].main_repo is None or \ prepos['DEFAULT'].main_repo not in prepos: @@ -406,7 +543,12 @@ class RepoConfigLoader(object): eclass_locations = [] eclass_locations.extend(master_repo.location for master_repo in repo.masters) - eclass_locations.append(repo.location) + # Only append the current repo to eclass_locations if it's not + # there already. This allows masters to have more control over + # eclass override order, which may be useful for scenarios in + # which there is a plan to migrate eclasses to a master repo. + if repo.location not in eclass_locations: + eclass_locations.append(repo.location) if repo.eclass_overrides: for other_repo_name in repo.eclass_overrides: @@ -419,6 +561,23 @@ class RepoConfigLoader(object): level=logging.ERROR, noiselevel=-1) repo.eclass_locations = tuple(eclass_locations) + eclass_dbs = {} + for repo_name, repo in prepos.items(): + if repo_name == "DEFAULT": + continue + + eclass_db = None + for eclass_location in repo.eclass_locations: + tree_db = eclass_dbs.get(eclass_location) + if tree_db is None: + tree_db = eclass_cache.cache(eclass_location) + eclass_dbs[eclass_location] = tree_db + if eclass_db is None: + eclass_db = tree_db.copy() + else: + eclass_db.append(tree_db) + repo.eclass_db = eclass_db + self._prepos_changed = True self._repo_location_list = [] @@ -488,6 +647,9 @@ class RepoConfigLoader(object): return None return self.treemap[repo_name] + def get_repo_for_location(self, location): + return self.prepos[self.get_name_for_location(location)] + def __getitem__(self, repo_name): return self.prepos[repo_name] @@ -502,3 +664,113 @@ def load_repository_config(settings): repoconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"], USER_CONFIG_PATH, "repos.conf")) return RepoConfigLoader(repoconfigpaths, settings) + +def _get_repo_name(repo_location, cached=None): + if cached is not None: + return cached + name, missing = RepoConfig._read_repo_name(repo_location) + if missing: + return None + return name + +def parse_layout_conf(repo_location, repo_name=None): + eapi = read_corresponding_eapi_file(os.path.join(repo_location, REPO_NAME_LOC)) + + layout_filename = os.path.join(repo_location, "metadata", "layout.conf") + layout_file = KeyValuePairFileLoader(layout_filename, None, None) + layout_data, layout_errors = layout_file.load() + + data = {} + + # None indicates abscence of a masters setting, which later code uses + # to trigger a backward compatibility fallback that sets an implicit + # master. In order to avoid this fallback behavior, layout.conf can + # explicitly set masters to an empty value, which will result in an + # empty tuple here instead of None. + masters = layout_data.get('masters') + if masters is not None: + masters = tuple(masters.split()) + data['masters'] = masters + data['aliases'] = tuple(layout_data.get('aliases', '').split()) + + data['allow-provide-virtual'] = \ + layout_data.get('allow-provide-virtuals', 'false').lower() == 'true' + + data['sign-commit'] = layout_data.get('sign-commits', 'false').lower() \ + == 'true' + + data['sign-manifest'] = layout_data.get('sign-manifests', 'true').lower() \ + == 'true' + + data['thin-manifest'] = layout_data.get('thin-manifests', 'false').lower() \ + == 'true' + + manifest_policy = layout_data.get('use-manifests', 'strict').lower() + data['allow-missing-manifest'] = manifest_policy != 'strict' + data['create-manifest'] = manifest_policy != 'false' + data['disable-manifest'] = manifest_policy == 'false' + + # for compatibility w/ PMS, fallback to pms; but also check if the + # cache exists or not. + cache_formats = layout_data.get('cache-formats', 'pms').lower().split() + if 'pms' in cache_formats and not os.path.isdir( + os.path.join(repo_location, 'metadata', 'cache')): + cache_formats.remove('pms') + data['cache-formats'] = tuple(cache_formats) + + manifest_hashes = layout_data.get('manifest-hashes') + if manifest_hashes is not None: + manifest_hashes = frozenset(manifest_hashes.upper().split()) + if MANIFEST2_REQUIRED_HASH not in manifest_hashes: + repo_name = _get_repo_name(repo_location, cached=repo_name) + warnings.warn((_("Repository named '%(repo_name)s' has a " + "'manifest-hashes' setting that does not contain " + "the '%(hash)s' hash which is required by this " + "portage version. You will have to upgrade portage " + "if you want to generate valid manifests for this " + "repository: %(layout_filename)s") % + {"repo_name": repo_name or 'unspecified', + "hash":MANIFEST2_REQUIRED_HASH, + "layout_filename":layout_filename}), + DeprecationWarning) + unsupported_hashes = manifest_hashes.difference( + MANIFEST2_HASH_FUNCTIONS) + if unsupported_hashes: + repo_name = _get_repo_name(repo_location, cached=repo_name) + warnings.warn((_("Repository named '%(repo_name)s' has a " + "'manifest-hashes' setting that contains one " + "or more hash types '%(hashes)s' which are not supported by " + "this portage version. You will have to upgrade " + "portage if you want to generate valid manifests for " + "this repository: %(layout_filename)s") % + {"repo_name": repo_name or 'unspecified', + "hashes":" ".join(sorted(unsupported_hashes)), + "layout_filename":layout_filename}), + DeprecationWarning) + data['manifest-hashes'] = manifest_hashes + + data['update-changelog'] = layout_data.get('update-changelog', 'false').lower() \ + == 'true' + + raw_formats = layout_data.get('profile-formats') + if raw_formats is None: + if eapi in ('4-python',): + raw_formats = ('portage-1',) + else: + raw_formats = ('portage-1-compat',) + else: + raw_formats = set(raw_formats.split()) + unknown = raw_formats.difference(_valid_profile_formats) + if unknown: + repo_name = _get_repo_name(repo_location, cached=repo_name) + warnings.warn((_("Repository named '%(repo_name)s' has unsupported " + "profiles in use ('profile-formats = %(unknown_fmts)s' setting in " + "'%(layout_filename)s; please upgrade portage.") % + dict(repo_name=repo_name or 'unspecified', + layout_filename=layout_filename, + unknown_fmts=" ".join(unknown))), + DeprecationWarning) + raw_formats = tuple(raw_formats.intersection(_valid_profile_formats)) + data['profile-formats'] = raw_formats + + return data, layout_errors diff --git a/portage_with_autodep/pym/portage/repository/config.pyo b/portage_with_autodep/pym/portage/repository/config.pyo Binary files differnew file mode 100644 index 0000000..f9ee26d --- /dev/null +++ b/portage_with_autodep/pym/portage/repository/config.pyo diff --git a/portage_with_autodep/pym/portage/tests/__init__.py b/portage_with_autodep/pym/portage/tests/__init__.py index a647aa2..492ece4 100644 --- a/portage_with_autodep/pym/portage/tests/__init__.py +++ b/portage_with_autodep/pym/portage/tests/__init__.py @@ -1,10 +1,13 @@ # tests/__init__.py -- Portage Unit Test functionality -# Copyright 2006-2010 Gentoo Foundation +# Copyright 2006-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 +from __future__ import print_function + import sys import time import unittest +from optparse import OptionParser, OptionValueError try: from unittest.runner import _TextTestResult # new in python-2.7 @@ -16,35 +19,33 @@ from portage import _encodings from portage import _unicode_decode def main(): - - TEST_FILE = b'__test__' - svn_dirname = b'.svn' suite = unittest.TestSuite() basedir = os.path.dirname(os.path.realpath(__file__)) - testDirs = [] - if len(sys.argv) > 1: - suite.addTests(getTestFromCommandLine(sys.argv[1:], basedir)) - return TextTestRunner(verbosity=2).run(suite) + usage = "usage: %s [options] [tests to run]" % os.path.basename(sys.argv[0]) + parser = OptionParser(usage=usage) + parser.add_option("-l", "--list", help="list all tests", + action="store_true", dest="list_tests") + (options, args) = parser.parse_args(args=sys.argv) - # the os.walk help mentions relative paths as being quirky - # I was tired of adding dirs to the list, so now we add __test__ - # to each dir we want tested. - for root, dirs, files in os.walk(basedir): - if svn_dirname in dirs: - dirs.remove(svn_dirname) - try: - root = _unicode_decode(root, - encoding=_encodings['fs'], errors='strict') - except UnicodeDecodeError: - continue + if options.list_tests: + testdir = os.path.dirname(sys.argv[0]) + for mydir in getTestDirs(basedir): + testsubdir = os.path.basename(mydir) + for name in getTestNames(mydir): + print("%s/%s/%s.py" % (testdir, testsubdir, name)) + return os.EX_OK - if TEST_FILE in files: - testDirs.append(root) + if len(args) > 1: + suite.addTests(getTestFromCommandLine(args[1:], basedir)) + else: + for mydir in getTestDirs(basedir): + suite.addTests(getTests(os.path.join(basedir, mydir), basedir)) - for mydir in testDirs: - suite.addTests(getTests(os.path.join(basedir, mydir), basedir) ) - return TextTestRunner(verbosity=2).run(suite) + result = TextTestRunner(verbosity=2).run(suite) + if not result.wasSuccessful(): + return 1 + return os.EX_OK def my_import(name): mod = __import__(name) @@ -54,7 +55,7 @@ def my_import(name): return mod def getTestFromCommandLine(args, base_path): - ret = [] + result = [] for arg in args: realpath = os.path.realpath(arg) path = os.path.dirname(realpath) @@ -64,28 +65,39 @@ def getTestFromCommandLine(args, base_path): raise Exception("Invalid argument: '%s'" % arg) mymodule = f[:-3] + result.extend(getTestsFromFiles(path, base_path, [mymodule])) + return result - parent_path = path[len(base_path)+1:] - parent_module = ".".join(("portage", "tests", parent_path)) - parent_module = parent_module.replace('/', '.') - result = [] +def getTestDirs(base_path): + TEST_FILE = b'__test__' + svn_dirname = b'.svn' + testDirs = [] - # Make the trailing / a . for module importing - modname = ".".join((parent_module, mymodule)) - mod = my_import(modname) - ret.append(unittest.TestLoader().loadTestsFromModule(mod)) - return ret + # the os.walk help mentions relative paths as being quirky + # I was tired of adding dirs to the list, so now we add __test__ + # to each dir we want tested. + for root, dirs, files in os.walk(base_path): + if svn_dirname in dirs: + dirs.remove(svn_dirname) + try: + root = _unicode_decode(root, + encoding=_encodings['fs'], errors='strict') + except UnicodeDecodeError: + continue -def getTests(path, base_path): - """ + if TEST_FILE in files: + testDirs.append(root) - path is the path to a given subdir ( 'portage/' for example) - This does a simple filter on files in that dir to give us modules - to import + testDirs.sort() + return testDirs - """ +def getTestNames(path): files = os.listdir(path) files = [ f[:-3] for f in files if f.startswith("test") and f.endswith(".py") ] + files.sort() + return files + +def getTestsFromFiles(path, base_path, files): parent_path = path[len(base_path)+1:] parent_module = ".".join(("portage", "tests", parent_path)) parent_module = parent_module.replace('/', '.') @@ -97,6 +109,16 @@ def getTests(path, base_path): result.append(unittest.TestLoader().loadTestsFromModule(mod)) return result +def getTests(path, base_path): + """ + + path is the path to a given subdir ( 'portage/' for example) + This does a simple filter on files in that dir to give us modules + to import + + """ + return getTestsFromFiles(path, base_path, getTestNames(path)) + class TextTestResult(_TextTestResult): """ We need a subclass of unittest._TextTestResult to handle tests with TODO @@ -109,6 +131,7 @@ class TextTestResult(_TextTestResult): def __init__(self, stream, descriptions, verbosity): super(TextTestResult, self).__init__(stream, descriptions, verbosity) self.todoed = [] + self.portage_skipped = [] def addTodo(self, test, info): self.todoed.append((test,info)) @@ -117,12 +140,20 @@ class TextTestResult(_TextTestResult): elif self.dots: self.stream.write(".") + def addPortageSkip(self, test, info): + self.portage_skipped.append((test,info)) + if self.showAll: + self.stream.writeln("SKIP") + elif self.dots: + self.stream.write(".") + def printErrors(self): if self.dots or self.showAll: self.stream.writeln() self.printErrorList('ERROR', self.errors) self.printErrorList('FAIL', self.failures) self.printErrorList('TODO', self.todoed) + self.printErrorList('SKIP', self.portage_skipped) class TestCase(unittest.TestCase): """ @@ -131,15 +162,12 @@ class TestCase(unittest.TestCase): and then fix the code later. This may not be a great approach (broken code!!??!11oneone) but it does happen at times. """ - - def __init__(self, methodName='runTest'): - # This method exists because unittest.py in python 2.4 stores - # the methodName as __testMethodName while 2.5 uses - # _testMethodName. - self._testMethodName = methodName - unittest.TestCase.__init__(self, methodName) + + def __init__(self, *pargs, **kwargs): + unittest.TestCase.__init__(self, *pargs, **kwargs) self.todo = False - + self.portage_skip = None + def defaultTestResult(self): return TextTestResult() @@ -162,7 +190,13 @@ class TestCase(unittest.TestCase): testMethod() ok = True except self.failureException: - if self.todo: + if self.portage_skip is not None: + if self.portage_skip is True: + result.addPortageSkip(self, "%s: SKIP" % testMethod) + else: + result.addPortageSkip(self, "%s: SKIP: %s" % + (testMethod, self.portage_skip)) + elif self.todo: result.addTodo(self,"%s: TODO" % testMethod) else: result.addFailure(self, sys.exc_info()) @@ -192,21 +226,21 @@ class TestCase(unittest.TestCase): unexpected exception. """ try: - callableObj(*args, **kwargs) + callableObj(*args, **kwargs) except excClass: - return + return else: - if hasattr(excClass,'__name__'): excName = excClass.__name__ - else: excName = str(excClass) - raise self.failureException("%s not raised: %s" % (excName, msg)) - + if hasattr(excClass,'__name__'): excName = excClass.__name__ + else: excName = str(excClass) + raise self.failureException("%s not raised: %s" % (excName, msg)) + class TextTestRunner(unittest.TextTestRunner): """ We subclass unittest.TextTestRunner to output SKIP for tests that fail but are skippable """ - + def _makeResult(self): - return TextTestResult(self.stream, self.descriptions, self.verbosity) + return TextTestResult(self.stream, self.descriptions, self.verbosity) def run(self, test): """ @@ -236,7 +270,7 @@ class TextTestRunner(unittest.TextTestRunner): else: self.stream.writeln("OK") return result - + test_cps = ['sys-apps/portage','virtual/portage'] test_versions = ['1.0', '1.0-r1','2.3_p4','1.0_alpha57'] test_slots = [ None, '1','gentoo-sources-2.6.17','spankywashere'] diff --git a/portage_with_autodep/pym/portage/tests/__init__.pyo b/portage_with_autodep/pym/portage/tests/__init__.pyo Binary files differnew file mode 100644 index 0000000..0e961b8 --- /dev/null +++ b/portage_with_autodep/pym/portage/tests/__init__.pyo diff --git a/portage_with_autodep/pym/portage/tests/bin/__init__.py b/portage_with_autodep/pym/portage/tests/bin/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/bin/__init__.py +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/bin/__test__ b/portage_with_autodep/pym/portage/tests/bin/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/bin/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/bin/setup_env.py b/portage_with_autodep/pym/portage/tests/bin/setup_env.py deleted file mode 100644 index e07643d..0000000 --- a/portage_with_autodep/pym/portage/tests/bin/setup_env.py +++ /dev/null @@ -1,85 +0,0 @@ -# setup_env.py -- Make sure bin subdir has sane env for testing -# Copyright 2007-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import tempfile - -from portage import os -from portage import shutil -from portage.tests import TestCase -from portage.process import spawn - -basepath = os.path.join(os.path.dirname(os.path.dirname( - os.path.abspath(__file__))), - "..", "..", "..") -bindir = os.path.join(basepath, "bin") -pymdir = os.path.join(basepath, "pym") -basedir = None -env = None - -def binTestsCleanup(): - global basedir - if basedir is None: - return - if os.access(basedir, os.W_OK): - shutil.rmtree(basedir) - basedir = None - -def binTestsInit(): - binTestsCleanup() - global basedir, env - basedir = tempfile.mkdtemp() - env = os.environ.copy() - env["D"] = os.path.join(basedir, "image") - env["T"] = os.path.join(basedir, "temp") - env["S"] = os.path.join(basedir, "workdir") - env["PF"] = "portage-tests-0.09-r1" - env["PATH"] = bindir + ":" + env["PATH"] - env["PORTAGE_BIN_PATH"] = bindir - env["PORTAGE_PYM_PATH"] = pymdir - os.mkdir(env["D"]) - os.mkdir(env["T"]) - os.mkdir(env["S"]) - -class BinTestCase(TestCase): - def init(self): - binTestsInit() - def cleanup(self): - binTestsCleanup() - -def _exists_in_D(path): - # Note: do not use os.path.join() here, we assume D to end in / - return os.access(env["D"] + path, os.W_OK) -def exists_in_D(path): - if not _exists_in_D(path): - raise TestCase.failureException -def xexists_in_D(path): - if _exists_in_D(path): - raise TestCase.failureException - -def portage_func(func, args, exit_status=0): - # we don't care about the output of the programs, - # just their exit value and the state of $D - global env - f = open('/dev/null', 'wb') - fd_pipes = {0:0,1:f.fileno(),2:f.fileno()} - def pre_exec(): - os.chdir(env["S"]) - spawn([func] + args.split(), env=env, - fd_pipes=fd_pipes, pre_exec=pre_exec) - f.close() - -def create_portage_wrapper(bin): - def derived_func(*args): - newargs = list(args) - newargs.insert(0, bin) - return portage_func(*newargs) - return derived_func - -for bin in os.listdir(os.path.join(bindir, "ebuild-helpers")): - if bin.startswith("do") or \ - bin.startswith("new") or \ - bin.startswith("prep") or \ - bin in ["ecompress","ecompressdir","fowners","fperms"]: - globals()[bin] = create_portage_wrapper( - os.path.join(bindir, "ebuild-helpers", bin)) diff --git a/portage_with_autodep/pym/portage/tests/bin/test_dobin.py b/portage_with_autodep/pym/portage/tests/bin/test_dobin.py deleted file mode 100644 index 6f50d7a..0000000 --- a/portage_with_autodep/pym/portage/tests/bin/test_dobin.py +++ /dev/null @@ -1,16 +0,0 @@ -# test_dobin.py -- Portage Unit Testing Functionality -# Copyright 2007-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests.bin.setup_env import BinTestCase, dobin, xexists_in_D - -class DoBin(BinTestCase): - def testDoBin(self): - self.init() - try: - dobin("does-not-exist", 1) - xexists_in_D("does-not-exist") - xexists_in_D("/bin/does-not-exist") - xexists_in_D("/usr/bin/does-not-exist") - finally: - self.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/bin/test_dodir.py b/portage_with_autodep/pym/portage/tests/bin/test_dodir.py deleted file mode 100644 index f4eb9b2..0000000 --- a/portage_with_autodep/pym/portage/tests/bin/test_dodir.py +++ /dev/null @@ -1,16 +0,0 @@ -# test_dodir.py -- Portage Unit Testing Functionality -# Copyright 2007-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests.bin.setup_env import BinTestCase, dodir, exists_in_D - -class DoDir(BinTestCase): - def testDoDir(self): - self.init() - try: - dodir("usr /usr") - exists_in_D("/usr") - dodir("/var/lib/moocow") - exists_in_D("/var/lib/moocow") - finally: - self.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/dbapi/__test__ b/portage_with_autodep/pym/portage/tests/dbapi/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/dbapi/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/dbapi/test_fakedbapi.py b/portage_with_autodep/pym/portage/tests/dbapi/test_fakedbapi.py deleted file mode 100644 index a2c5f77..0000000 --- a/portage_with_autodep/pym/portage/tests/dbapi/test_fakedbapi.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import shutil -import tempfile - -from portage import os -from portage.dbapi.virtual import fakedbapi -from portage.package.ebuild.config import config -from portage.tests import TestCase - -class TestFakedbapi(TestCase): - - def testFakedbapi(self): - packages = ( - ("sys-apps/portage-2.1.10", { - "EAPI" : "2", - "IUSE" : "ipc doc", - "repository" : "gentoo", - "SLOT" : "0", - "USE" : "ipc missing-iuse", - }), - ("virtual/package-manager-0", { - "EAPI" : "0", - "repository" : "gentoo", - "SLOT" : "0", - }), - ) - - match_tests = ( - ("sys-apps/portage:0[ipc]", ["sys-apps/portage-2.1.10"]), - ("sys-apps/portage:0[-ipc]", []), - ("sys-apps/portage:0[doc]", []), - ("sys-apps/portage:0[-doc]", ["sys-apps/portage-2.1.10"]), - ("sys-apps/portage:0", ["sys-apps/portage-2.1.10"]), - ("sys-apps/portage:0[missing-iuse]", []), - ("sys-apps/portage:0[-missing-iuse]", []), - ("sys-apps/portage:0::gentoo[ipc]", ["sys-apps/portage-2.1.10"]), - ("sys-apps/portage:0::multilib[ipc]", []), - ("virtual/package-manager", ["virtual/package-manager-0"]), - ) - - tempdir = tempfile.mkdtemp() - try: - portdir = os.path.join(tempdir, "usr/portage") - os.makedirs(portdir) - env = { - "PORTDIR": portdir, - } - fakedb = fakedbapi(settings=config(config_profile_path="", - env=env, _eprefix=tempdir)) - for cpv, metadata in packages: - fakedb.cpv_inject(cpv, metadata=metadata) - - for atom, expected_result in match_tests: - self.assertEqual( fakedb.match(atom), expected_result ) - finally: - shutil.rmtree(tempdir) diff --git a/portage_with_autodep/pym/portage/tests/dep/__init__.py b/portage_with_autodep/pym/portage/tests/dep/__init__.py deleted file mode 100644 index 9c3f524..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# tests/portage.dep/__init__.py -- Portage Unit Test functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/tests/dep/__test__ b/portage_with_autodep/pym/portage/tests/dep/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/dep/testAtom.py b/portage_with_autodep/pym/portage/tests/dep/testAtom.py deleted file mode 100644 index 092cacf..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/testAtom.py +++ /dev/null @@ -1,315 +0,0 @@ -# Copyright 2006, 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import Atom -from portage.exception import InvalidAtom - -class TestAtom(TestCase): - - def testAtom(self): - - tests = ( - ( "=sys-apps/portage-2.1-r1:0[doc,a=,!b=,c?,!d?,-e]", - ('=', 'sys-apps/portage', '2.1-r1', '0', '[doc,a=,!b=,c?,!d?,-e]', None), False, False ), - ( "=sys-apps/portage-2.1-r1*:0[doc]", - ('=*', 'sys-apps/portage', '2.1-r1', '0', '[doc]', None), False, False ), - ( "sys-apps/portage:0[doc]", - (None, 'sys-apps/portage', None, '0', '[doc]', None), False, False ), - ( "sys-apps/portage:0[doc]", - (None, 'sys-apps/portage', None, '0', '[doc]', None), False, False ), - ( "*/*", - (None, '*/*', None, None, None, None), True, False ), - ( "sys-apps/*", - (None, 'sys-apps/*', None, None, None, None), True, False ), - ( "*/portage", - (None, '*/portage', None, None, None, None), True, False ), - ( "s*s-*/portage:1", - (None, 's*s-*/portage', None, '1', None, None), True, False ), - ( "*/po*ge:2", - (None, '*/po*ge', None, '2', None, None), True, False ), - ( "!dev-libs/A", - (None, 'dev-libs/A', None, None, None, None), True, True ), - ( "!!dev-libs/A", - (None, 'dev-libs/A', None, None, None, None), True, True ), - ( "!!dev-libs/A", - (None, 'dev-libs/A', None, None, None, None), True, True ), - ( "dev-libs/A[foo(+)]", - (None, 'dev-libs/A', None, None, "[foo(+)]", None), True, True ), - ( "dev-libs/A[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", - (None, 'dev-libs/A', None, None, "[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", None), True, True ), - ( "dev-libs/A:2[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", - (None, 'dev-libs/A', None, "2", "[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", None), True, True ), - - ( "=sys-apps/portage-2.1-r1:0::repo_name[doc,a=,!b=,c?,!d?,-e]", - ('=', 'sys-apps/portage', '2.1-r1', '0', '[doc,a=,!b=,c?,!d?,-e]', 'repo_name'), False, True ), - ( "=sys-apps/portage-2.1-r1*:0::repo_name[doc]", - ('=*', 'sys-apps/portage', '2.1-r1', '0', '[doc]', 'repo_name'), False, True ), - ( "sys-apps/portage:0::repo_name[doc]", - (None, 'sys-apps/portage', None, '0', '[doc]', 'repo_name'), False, True ), - - ( "*/*::repo_name", - (None, '*/*', None, None, None, 'repo_name'), True, True ), - ( "sys-apps/*::repo_name", - (None, 'sys-apps/*', None, None, None, 'repo_name'), True, True ), - ( "*/portage::repo_name", - (None, '*/portage', None, None, None, 'repo_name'), True, True ), - ( "s*s-*/portage:1::repo_name", - (None, 's*s-*/portage', None, '1', None, 'repo_name'), True, True ), - ) - - tests_xfail = ( - ( Atom("sys-apps/portage"), False, False ), - ( "cat/pkg[a!]", False, False ), - ( "cat/pkg[!a]", False, False ), - ( "cat/pkg[!a!]", False, False ), - ( "cat/pkg[!a-]", False, False ), - ( "cat/pkg[-a=]", False, False ), - ( "cat/pkg[-a?]", False, False ), - ( "cat/pkg[-a!]", False, False ), - ( "cat/pkg[=a]", False, False ), - ( "cat/pkg[=a=]", False, False ), - ( "cat/pkg[=a?]", False, False ), - ( "cat/pkg[=a!]", False, False ), - ( "cat/pkg[=a-]", False, False ), - ( "cat/pkg[?a]", False, False ), - ( "cat/pkg[?a=]", False, False ), - ( "cat/pkg[?a?]", False, False ), - ( "cat/pkg[?a!]", False, False ), - ( "cat/pkg[?a-]", False, False ), - ( "sys-apps/portage[doc]:0", False, False ), - ( "*/*", False, False ), - ( "sys-apps/*", False, False ), - ( "*/portage", False, False ), - ( "*/**", True, False ), - ( "*/portage[use]", True, False ), - ( "cat/pkg[a()]", False, False ), - ( "cat/pkg[a(]", False, False ), - ( "cat/pkg[a)]", False, False ), - ( "cat/pkg[a(,b]", False, False ), - ( "cat/pkg[a),b]", False, False ), - ( "cat/pkg[a(*)]", False, False ), - ( "cat/pkg[a(*)]", True, False ), - ( "cat/pkg[a(+-)]", False, False ), - ( "cat/pkg[a()]", False, False ), - ( "cat/pkg[(+)a]", False, False ), - ( "cat/pkg[a=(+)]", False, False ), - ( "cat/pkg[!(+)a=]", False, False ), - ( "cat/pkg[!a=(+)]", False, False ), - ( "cat/pkg[a?(+)]", False, False ), - ( "cat/pkg[!a?(+)]", False, False ), - ( "cat/pkg[!(+)a?]", False, False ), - ( "cat/pkg[-(+)a]", False, False ), - ( "cat/pkg[a(+),-a]", False, False ), - ( "cat/pkg[a(-),-a]", False, False ), - ( "cat/pkg[-a,a(+)]", False, False ), - ( "cat/pkg[-a,a(-)]", False, False ), - ( "cat/pkg[-a(+),a(-)]", False, False ), - ( "cat/pkg[-a(-),a(+)]", False, False ), - ( "sys-apps/portage[doc]::repo_name", False, False ), - ( "sys-apps/portage:0[doc]::repo_name", False, False ), - ( "sys-apps/portage[doc]:0::repo_name", False, False ), - ( "=sys-apps/portage-2.1-r1:0::repo_name[doc,a=,!b=,c?,!d?,-e]", False, False ), - ( "=sys-apps/portage-2.1-r1*:0::repo_name[doc]", False, False ), - ( "sys-apps/portage:0::repo_name[doc]", False, False ), - ( "*/*::repo_name", True, False ), - ) - - for atom, parts, allow_wildcard, allow_repo in tests: - a = Atom(atom, allow_wildcard=allow_wildcard, allow_repo=allow_repo) - op, cp, ver, slot, use, repo = parts - self.assertEqual( op, a.operator, - msg="Atom('%s').operator = %s == '%s'" % ( atom, a.operator, op ) ) - self.assertEqual( cp, a.cp, - msg="Atom('%s').cp = %s == '%s'" % ( atom, a.cp, cp ) ) - if ver is not None: - cpv = "%s-%s" % (cp, ver) - else: - cpv = cp - self.assertEqual( cpv, a.cpv, - msg="Atom('%s').cpv = %s == '%s'" % ( atom, a.cpv, cpv ) ) - self.assertEqual( slot, a.slot, - msg="Atom('%s').slot = %s == '%s'" % ( atom, a.slot, slot ) ) - self.assertEqual( repo, a.repo, - msg="Atom('%s').repo == %s == '%s'" % ( atom, a.repo, repo ) ) - - if a.use: - returned_use = str(a.use) - else: - returned_use = None - self.assertEqual( use, returned_use, - msg="Atom('%s').use = %s == '%s'" % ( atom, returned_use, use ) ) - - for atom, allow_wildcard, allow_repo in tests_xfail: - self.assertRaisesMsg(atom, (InvalidAtom, TypeError), Atom, atom, \ - allow_wildcard=allow_wildcard, allow_repo=allow_repo) - - def test_intersects(self): - test_cases = ( - ("dev-libs/A", "dev-libs/A", True), - ("dev-libs/A", "dev-libs/B", False), - ("dev-libs/A", "sci-libs/A", False), - ("dev-libs/A[foo]", "sci-libs/A[bar]", False), - ("dev-libs/A[foo(+)]", "sci-libs/A[foo(-)]", False), - ("=dev-libs/A-1", "=dev-libs/A-1-r1", False), - ("~dev-libs/A-1", "=dev-libs/A-1", False), - ("=dev-libs/A-1:1", "=dev-libs/A-1", True), - ("=dev-libs/A-1:1", "=dev-libs/A-1:1", True), - ("=dev-libs/A-1:1", "=dev-libs/A-1:2", False), - ) - - for atom, other, expected_result in test_cases: - self.assertEqual(Atom(atom).intersects(Atom(other)), expected_result, \ - "%s and %s should intersect: %s" % (atom, other, expected_result)) - - def test_violated_conditionals(self): - test_cases = ( - ("dev-libs/A", ["foo"], ["foo"], None, "dev-libs/A"), - ("dev-libs/A[foo]", [], ["foo"], None, "dev-libs/A[foo]"), - ("dev-libs/A[foo]", ["foo"], ["foo"], None, "dev-libs/A"), - ("dev-libs/A[foo]", [], ["foo"], [], "dev-libs/A[foo]"), - ("dev-libs/A[foo]", ["foo"], ["foo"], [], "dev-libs/A"), - - ("dev-libs/A:0[foo]", ["foo"], ["foo"], [], "dev-libs/A:0"), - - ("dev-libs/A[foo,-bar]", [], ["foo", "bar"], None, "dev-libs/A[foo]"), - ("dev-libs/A[-foo,bar]", [], ["foo", "bar"], None, "dev-libs/A[bar]"), - - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", [], ["a", "b", "c", "d", "e", "f"], [], "dev-libs/A[a,!c=]"), - - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["a"], ["a", "b", "c", "d", "e", "f"], [], "dev-libs/A[!c=]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["b"], ["a", "b", "c", "d", "e", "f"], [], "dev-libs/A[a,b=,!c=]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["c"], ["a", "b", "c", "d", "e", "f"], [], "dev-libs/A[a]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["d"], ["a", "b", "c", "d", "e", "f"], [], "dev-libs/A[a,!c=]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["e"], ["a", "b", "c", "d", "e", "f"], [], "dev-libs/A[a,!c=,!e?]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["f"], ["a", "b", "c", "d", "e", "f"], [], "dev-libs/A[a,!c=,-f]"), - - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["a"], ["a", "b", "c", "d", "e", "f"], ["a"], "dev-libs/A[!c=]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["b"], ["a", "b", "c", "d", "e", "f"], ["b"], "dev-libs/A[a,!c=]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["c"], ["a", "b", "c", "d", "e", "f"], ["c"], "dev-libs/A[a,!c=]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["d"], ["a", "b", "c", "d", "e", "f"], ["d"], "dev-libs/A[a,!c=]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["e"], ["a", "b", "c", "d", "e", "f"], ["e"], "dev-libs/A[a,!c=]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["f"], ["a", "b", "c", "d", "e", "f"], ["f"], "dev-libs/A[a,!c=,-f]"), - - ("dev-libs/A[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", ["a"], ["a", "b", "c", "d", "e", "f"], ["a"], "dev-libs/A[!c(+)=]"), - ("dev-libs/A[a(-),b(+)=,!c(-)=,d(+)?,!e(-)?,-f(+)]", ["b"], ["a", "b", "c", "d", "e", "f"], ["b"], "dev-libs/A[a(-),!c(-)=]"), - ("dev-libs/A[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", ["c"], ["a", "b", "c", "d", "e", "f"], ["c"], "dev-libs/A[a(+),!c(+)=]"), - ("dev-libs/A[a(-),b(+)=,!c(-)=,d(+)?,!e(-)?,-f(+)]", ["d"], ["a", "b", "c", "d", "e", "f"], ["d"], "dev-libs/A[a(-),!c(-)=]"), - ("dev-libs/A[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", ["e"], ["a", "b", "c", "d", "e", "f"], ["e"], "dev-libs/A[a(+),!c(+)=]"), - ("dev-libs/A[a(-),b(+)=,!c(-)=,d(+)?,!e(-)?,-f(+)]", ["f"], ["a", "b", "c", "d", "e", "f"], ["f"], "dev-libs/A[a(-),!c(-)=,-f(+)]"), - - ("dev-libs/A[a(+),b(+)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", ["a"], ["a"], ["a"], "dev-libs/A[b(+)=,!e(+)?]"), - ("dev-libs/A[a(-),b(+)=,!c(-)=,d(+)?,!e(-)?,-f(+)]", ["b"], ["b"], ["b"], "dev-libs/A[a(-),!c(-)=,-f(+)]"), - ("dev-libs/A[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", ["c"], ["c"], ["c"], "dev-libs/A[!c(+)=,!e(+)?]"), - ("dev-libs/A[a(-),b(+)=,!c(-)=,d(+)?,!e(-)?,-f(+)]", ["d"], ["d"], ["d"], "dev-libs/A[a(-),b(+)=,!c(-)=,-f(+)]"), - ("dev-libs/A[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", ["e"], ["e"], ["e"], "dev-libs/A"), - ("dev-libs/A[a(-),b(+)=,!c(-)=,d(+)?,!e(-)?,-f(+)]", ["f"], ["f"], ["f"], "dev-libs/A[a(-),b(+)=,!c(-)=,-f(+)]"), - - #Some more test cases to trigger all remaining code paths - ("dev-libs/B[x?]", [], ["x"], ["x"], "dev-libs/B[x?]"), - ("dev-libs/B[x(+)?]", [], [], ["x"], "dev-libs/B"), - ("dev-libs/B[x(-)?]", [], [], ["x"], "dev-libs/B[x(-)?]"), - - ("dev-libs/C[x=]", [], ["x"], ["x"], "dev-libs/C[x=]"), - ("dev-libs/C[x(+)=]", [], [], ["x"], "dev-libs/C"), - ("dev-libs/C[x(-)=]", [], [], ["x"], "dev-libs/C[x(-)=]"), - - ("dev-libs/D[!x=]", [], ["x"], ["x"], "dev-libs/D"), - ("dev-libs/D[!x(+)=]", [], [], ["x"], "dev-libs/D[!x(+)=]"), - ("dev-libs/D[!x(-)=]", [], [], ["x"], "dev-libs/D"), - - #Missing IUSE test cases - ("dev-libs/B[x]", [], [], [], "dev-libs/B[x]"), - ("dev-libs/B[-x]", [], [], [], "dev-libs/B[-x]"), - ("dev-libs/B[x?]", [], [], [], "dev-libs/B[x?]"), - ("dev-libs/B[x=]", [], [], [], "dev-libs/B[x=]"), - ("dev-libs/B[!x=]", [], [], ["x"], "dev-libs/B[!x=]"), - ("dev-libs/B[!x?]", [], [], ["x"], "dev-libs/B[!x?]"), - ) - - test_cases_xfail = ( - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", [], ["a", "b", "c", "d", "e", "f"], None), - ) - - class use_flag_validator(object): - def __init__(self, iuse): - self.iuse = iuse - - def is_valid_flag(self, flag): - return flag in iuse - - for atom, other_use, iuse, parent_use, expected_violated_atom in test_cases: - a = Atom(atom) - validator = use_flag_validator(iuse) - violated_atom = a.violated_conditionals(other_use, validator.is_valid_flag, parent_use) - if parent_use is None: - fail_msg = "Atom: %s, other_use: %s, iuse: %s, parent_use: %s, got: %s, expected: %s" % \ - (atom, " ".join(other_use), " ".join(iuse), "None", str(violated_atom), expected_violated_atom) - else: - fail_msg = "Atom: %s, other_use: %s, iuse: %s, parent_use: %s, got: %s, expected: %s" % \ - (atom, " ".join(other_use), " ".join(iuse), " ".join(parent_use), str(violated_atom), expected_violated_atom) - self.assertEqual(str(violated_atom), expected_violated_atom, fail_msg) - - for atom, other_use, iuse, parent_use in test_cases_xfail: - a = Atom(atom) - validator = use_flag_validator(iuse) - self.assertRaisesMsg(atom, InvalidAtom, \ - a.violated_conditionals, other_use, validator.is_valid_flag, parent_use) - - def test_evaluate_conditionals(self): - test_cases = ( - ("dev-libs/A[foo]", [], "dev-libs/A[foo]"), - ("dev-libs/A[foo]", ["foo"], "dev-libs/A[foo]"), - - ("dev-libs/A:0[foo=]", ["foo"], "dev-libs/A:0[foo]"), - - ("dev-libs/A[foo,-bar]", [], "dev-libs/A[foo,-bar]"), - ("dev-libs/A[-foo,bar]", [], "dev-libs/A[-foo,bar]"), - - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", [], "dev-libs/A[a,-b,c,-e,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["a"], "dev-libs/A[a,-b,c,-e,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["b"], "dev-libs/A[a,b,c,-e,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["c"], "dev-libs/A[a,-b,-c,-e,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["d"], "dev-libs/A[a,-b,c,d,-e,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["e"], "dev-libs/A[a,-b,c,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["f"], "dev-libs/A[a,-b,c,-e,-f]"), - ("dev-libs/A[a(-),b(+)=,!c(-)=,d(+)?,!e(-)?,-f(+)]", ["d"], "dev-libs/A[a(-),-b(+),c(-),d(+),-e(-),-f(+)]"), - ("dev-libs/A[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", ["f"], "dev-libs/A[a(+),-b(-),c(+),-e(+),-f(-)]"), - ) - - for atom, use, expected_atom in test_cases: - a = Atom(atom) - b = a.evaluate_conditionals(use) - self.assertEqual(str(b), expected_atom) - self.assertEqual(str(b.unevaluated_atom), atom) - - def test__eval_qa_conditionals(self): - test_cases = ( - ("dev-libs/A[foo]", [], [], "dev-libs/A[foo]"), - ("dev-libs/A[foo]", ["foo"], [], "dev-libs/A[foo]"), - ("dev-libs/A[foo]", [], ["foo"], "dev-libs/A[foo]"), - - ("dev-libs/A:0[foo]", [], [], "dev-libs/A:0[foo]"), - ("dev-libs/A:0[foo]", ["foo"], [], "dev-libs/A:0[foo]"), - ("dev-libs/A:0[foo]", [], ["foo"], "dev-libs/A:0[foo]"), - ("dev-libs/A:0[foo=]", [], ["foo"], "dev-libs/A:0[foo]"), - - ("dev-libs/A[foo,-bar]", ["foo"], ["bar"], "dev-libs/A[foo,-bar]"), - ("dev-libs/A[-foo,bar]", ["foo", "bar"], [], "dev-libs/A[-foo,bar]"), - - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["a", "b", "c"], [], "dev-libs/A[a,-b,c,d,-e,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", [], ["a", "b", "c"], "dev-libs/A[a,b,-c,d,-e,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", ["d", "e", "f"], [], "dev-libs/A[a,b,-b,c,-c,-e,-f]"), - ("dev-libs/A[a,b=,!c=,d?,!e?,-f]", [], ["d", "e", "f"], "dev-libs/A[a,b,-b,c,-c,d,-f]"), - - ("dev-libs/A[a(-),b(+)=,!c(-)=,d(+)?,!e(-)?,-f(+)]", \ - ["a", "b", "c", "d", "e", "f"], [], "dev-libs/A[a(-),-b(+),c(-),-e(-),-f(+)]"), - ("dev-libs/A[a(+),b(-)=,!c(+)=,d(-)?,!e(+)?,-f(-)]", \ - [], ["a", "b", "c", "d", "e", "f"], "dev-libs/A[a(+),b(-),-c(+),d(-),-f(-)]"), - ) - - for atom, use_mask, use_force, expected_atom in test_cases: - a = Atom(atom) - b = a._eval_qa_conditionals(use_mask, use_force) - self.assertEqual(str(b), expected_atom) - self.assertEqual(str(b.unevaluated_atom), atom) diff --git a/portage_with_autodep/pym/portage/tests/dep/testCheckRequiredUse.py b/portage_with_autodep/pym/portage/tests/dep/testCheckRequiredUse.py deleted file mode 100644 index 54791e0..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/testCheckRequiredUse.py +++ /dev/null @@ -1,219 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import check_required_use -from portage.exception import InvalidDependString - -class TestCheckRequiredUse(TestCase): - - def testCheckRequiredUse(self): - test_cases = ( - ( "|| ( a b )", [], ["a", "b"], False), - ( "|| ( a b )", ["a"], ["a", "b"], True), - ( "|| ( a b )", ["b"], ["a", "b"], True), - ( "|| ( a b )", ["a", "b"], ["a", "b"], True), - - ( "^^ ( a b )", [], ["a", "b"], False), - ( "^^ ( a b )", ["a"], ["a", "b"], True), - ( "^^ ( a b )", ["b"], ["a", "b"], True), - ( "^^ ( a b )", ["a", "b"], ["a", "b"], False), - - ( "^^ ( || ( a b ) c )", [], ["a", "b", "c"], False), - ( "^^ ( || ( a b ) c )", ["a"], ["a", "b", "c"], True), - - ( "^^ ( || ( ( a b ) ) ( c ) )", [], ["a", "b", "c"], False), - ( "( ^^ ( ( || ( ( a ) ( b ) ) ) ( ( c ) ) ) )", ["a"], ["a", "b", "c"], True), - - ( "a || ( b c )", ["a"], ["a", "b", "c"], False), - ( "|| ( b c ) a", ["a"], ["a", "b", "c"], False), - - ( "|| ( a b c )", ["a"], ["a", "b", "c"], True), - ( "|| ( a b c )", ["b"], ["a", "b", "c"], True), - ( "|| ( a b c )", ["c"], ["a", "b", "c"], True), - - ( "^^ ( a b c )", ["a"], ["a", "b", "c"], True), - ( "^^ ( a b c )", ["b"], ["a", "b", "c"], True), - ( "^^ ( a b c )", ["c"], ["a", "b", "c"], True), - ( "^^ ( a b c )", ["a", "b"], ["a", "b", "c"], False), - ( "^^ ( a b c )", ["b", "c"], ["a", "b", "c"], False), - ( "^^ ( a b c )", ["a", "c"], ["a", "b", "c"], False), - ( "^^ ( a b c )", ["a", "b", "c"], ["a", "b", "c"], False), - - ( "a? ( ^^ ( b c ) )", [], ["a", "b", "c"], True), - ( "a? ( ^^ ( b c ) )", ["a"], ["a", "b", "c"], False), - ( "a? ( ^^ ( b c ) )", ["b"], ["a", "b", "c"], True), - ( "a? ( ^^ ( b c ) )", ["c"], ["a", "b", "c"], True), - ( "a? ( ^^ ( b c ) )", ["a", "b"], ["a", "b", "c"], True), - ( "a? ( ^^ ( b c ) )", ["a", "b", "c"], ["a", "b", "c"], False), - - ( "^^ ( a? ( !b ) !c? ( d ) )", [], ["a", "b", "c", "d"], False), - ( "^^ ( a? ( !b ) !c? ( d ) )", ["a"], ["a", "b", "c", "d"], True), - ( "^^ ( a? ( !b ) !c? ( d ) )", ["c"], ["a", "b", "c", "d"], True), - ( "^^ ( a? ( !b ) !c? ( d ) )", ["a", "c"], ["a", "b", "c", "d"], True), - ( "^^ ( a? ( !b ) !c? ( d ) )", ["a", "b", "c"], ["a", "b", "c", "d"], False), - ( "^^ ( a? ( !b ) !c? ( d ) )", ["a", "b", "d"], ["a", "b", "c", "d"], True), - ( "^^ ( a? ( !b ) !c? ( d ) )", ["a", "b", "d"], ["a", "b", "c", "d"], True), - ( "^^ ( a? ( !b ) !c? ( d ) )", ["a", "d"], ["a", "b", "c", "d"], False), - - ( "|| ( ^^ ( a b ) ^^ ( b c ) )", [], ["a", "b", "c"], False), - ( "|| ( ^^ ( a b ) ^^ ( b c ) )", ["a"], ["a", "b", "c"], True), - ( "|| ( ^^ ( a b ) ^^ ( b c ) )", ["b"], ["a", "b", "c"], True), - ( "|| ( ^^ ( a b ) ^^ ( b c ) )", ["c"], ["a", "b", "c"], True), - ( "|| ( ^^ ( a b ) ^^ ( b c ) )", ["a", "b"], ["a", "b", "c"], True), - ( "|| ( ^^ ( a b ) ^^ ( b c ) )", ["a", "c"], ["a", "b", "c"], True), - ( "|| ( ^^ ( a b ) ^^ ( b c ) )", ["b", "c"], ["a", "b", "c"], True), - ( "|| ( ^^ ( a b ) ^^ ( b c ) )", ["a", "b", "c"], ["a", "b", "c"], False), - - ( "^^ ( || ( a b ) ^^ ( b c ) )", [], ["a", "b", "c"], False), - ( "^^ ( || ( a b ) ^^ ( b c ) )", ["a"], ["a", "b", "c"], True), - ( "^^ ( || ( a b ) ^^ ( b c ) )", ["b"], ["a", "b", "c"], False), - ( "^^ ( || ( a b ) ^^ ( b c ) )", ["c"], ["a", "b", "c"], True), - ( "^^ ( || ( a b ) ^^ ( b c ) )", ["a", "b"], ["a", "b", "c"], False), - ( "^^ ( || ( a b ) ^^ ( b c ) )", ["a", "c"], ["a", "b", "c"], False), - ( "^^ ( || ( a b ) ^^ ( b c ) )", ["b", "c"], ["a", "b", "c"], True), - ( "^^ ( || ( a b ) ^^ ( b c ) )", ["a", "b", "c"], ["a", "b", "c"], True), - - ( "|| ( ( a b ) c )", ["a", "b", "c"], ["a", "b", "c"], True), - ( "|| ( ( a b ) c )", ["b", "c"], ["a", "b", "c"], True), - ( "|| ( ( a b ) c )", ["a", "c"], ["a", "b", "c"], True), - ( "|| ( ( a b ) c )", ["a", "b"], ["a", "b", "c"], True), - ( "|| ( ( a b ) c )", ["a"], ["a", "b", "c"], False), - ( "|| ( ( a b ) c )", ["b"], ["a", "b", "c"], False), - ( "|| ( ( a b ) c )", ["c"], ["a", "b", "c"], True), - ( "|| ( ( a b ) c )", [], ["a", "b", "c"], False), - - ( "^^ ( ( a b ) c )", ["a", "b", "c"], ["a", "b", "c"], False), - ( "^^ ( ( a b ) c )", ["b", "c"], ["a", "b", "c"], True), - ( "^^ ( ( a b ) c )", ["a", "c"], ["a", "b", "c"], True), - ( "^^ ( ( a b ) c )", ["a", "b"], ["a", "b", "c"], True), - ( "^^ ( ( a b ) c )", ["a"], ["a", "b", "c"], False), - ( "^^ ( ( a b ) c )", ["b"], ["a", "b", "c"], False), - ( "^^ ( ( a b ) c )", ["c"], ["a", "b", "c"], True), - ( "^^ ( ( a b ) c )", [], ["a", "b", "c"], False), - ) - - test_cases_xfail = ( - ( "^^ ( || ( a b ) ^^ ( b c ) )", [], ["a", "b"]), - ( "^^ ( || ( a b ) ^^ ( b c )", [], ["a", "b", "c"]), - ( "^^( || ( a b ) ^^ ( b c ) )", [], ["a", "b", "c"]), - ( "^^ || ( a b ) ^^ ( b c )", [], ["a", "b", "c"]), - ( "^^ ( ( || ) ( a b ) ^^ ( b c ) )", [], ["a", "b", "c"]), - ( "^^ ( || ( a b ) ) ^^ ( b c ) )", [], ["a", "b", "c"]), - ) - - for required_use, use, iuse, expected in test_cases: - self.assertEqual(bool(check_required_use(required_use, use, iuse.__contains__)), \ - expected, required_use + ", USE = " + " ".join(use)) - - for required_use, use, iuse in test_cases_xfail: - self.assertRaisesMsg(required_use + ", USE = " + " ".join(use), \ - InvalidDependString, check_required_use, required_use, use, iuse.__contains__) - - def testCheckRequiredUseFilterSatisfied(self): - """ - Test filtering of satisfied parts of REQUIRED_USE, - in order to reduce noise for bug #353234. - """ - test_cases = ( - ( - "bindist? ( !amr !faac !win32codecs ) cdio? ( !cdparanoia !cddb ) dvdnav? ( dvd )", - ("cdio", "cdparanoia"), - "cdio? ( !cdparanoia )" - ), - ( - "|| ( !amr !faac !win32codecs ) cdio? ( !cdparanoia !cddb ) ^^ ( foo bar )", - ["cdio", "cdparanoia", "foo"], - "cdio? ( !cdparanoia )" - ), - ( - "^^ ( || ( a b ) c )", - ("a", "b", "c"), - "^^ ( || ( a b ) c )" - ), - ( - "^^ ( || ( ( a b ) ) ( c ) )", - ("a", "b", "c"), - "^^ ( ( a b ) c )" - ), - ( - "a? ( ( c e ) ( b d ) )", - ("a", "c", "e"), - "a? ( b d )" - ), - ( - "a? ( ( c e ) ( b d ) )", - ("a", "b", "c", "e"), - "a? ( d )" - ), - ( - "a? ( ( c e ) ( c e b c d e c ) )", - ("a", "c", "e"), - "a? ( b d )" - ), - ( - "^^ ( || ( a b ) ^^ ( b c ) )", - ("a", "b"), - "^^ ( || ( a b ) ^^ ( b c ) )" - ), - ( - "^^ ( || ( a b ) ^^ ( b c ) )", - ["a", "c"], - "^^ ( || ( a b ) ^^ ( b c ) )" - ), - ( - "^^ ( || ( a b ) ^^ ( b c ) )", - ["b", "c"], - "" - ), - ( - "^^ ( || ( a b ) ^^ ( b c ) )", - ["a", "b", "c"], - "" - ), - ( - "^^ ( ( a b c ) ( b c d ) )", - ["a", "b", "c"], - "" - ), - ( - "^^ ( ( a b c ) ( b c d ) )", - ["a", "b", "c", "d"], - "^^ ( ( a b c ) ( b c d ) )" - ), - ( - "^^ ( ( a b c ) ( b c !d ) )", - ["a", "b", "c"], - "^^ ( ( a b c ) ( b c !d ) )" - ), - ( - "^^ ( ( a b c ) ( b c !d ) )", - ["a", "b", "c", "d"], - "" - ), - ( - "( ( ( a ) ) ( ( ( b c ) ) ) )", - [""], - "a b c" - ), - ( - "|| ( ( ( ( a ) ) ( ( ( b c ) ) ) ) )", - [""], - "a b c" - ), - ( - "|| ( ( a ( ( ) ( ) ) ( ( ) ) ( b ( ) c ) ) )", - [""], - "a b c" - ), - ( - "|| ( ( a b c ) ) || ( ( d e f ) )", - [""], - "a b c d e f" - ), - ) - for required_use, use, expected in test_cases: - result = check_required_use(required_use, use, lambda k: True).tounicode() - self.assertEqual(result, expected, - "REQUIRED_USE = '%s', USE = '%s', '%s' != '%s'" % \ - (required_use, " ".join(use), result, expected)) diff --git a/portage_with_autodep/pym/portage/tests/dep/testExtendedAtomDict.py b/portage_with_autodep/pym/portage/tests/dep/testExtendedAtomDict.py deleted file mode 100644 index 69d092e..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/testExtendedAtomDict.py +++ /dev/null @@ -1,18 +0,0 @@ -# test_isvalidatom.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import ExtendedAtomDict - -class TestExtendedAtomDict(TestCase): - - def testExtendedAtomDict(self): - d = ExtendedAtomDict(dict) - d["*/*"] = { "test1": "x" } - d["dev-libs/*"] = { "test2": "y" } - d.setdefault("sys-apps/portage", {})["test3"] = "z" - self.assertEqual(d.get("dev-libs/A"), { "test1": "x", "test2": "y" }) - self.assertEqual(d.get("sys-apps/portage"), { "test1": "x", "test3": "z" }) - self.assertEqual(d["dev-libs/*"], { "test2": "y" }) - self.assertEqual(d["sys-apps/portage"], {'test1': 'x', 'test3': 'z'}) diff --git a/portage_with_autodep/pym/portage/tests/dep/testExtractAffectingUSE.py b/portage_with_autodep/pym/portage/tests/dep/testExtractAffectingUSE.py deleted file mode 100644 index 026a552..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/testExtractAffectingUSE.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import extract_affecting_use -from portage.exception import InvalidDependString - -class TestExtractAffectingUSE(TestCase): - - def testExtractAffectingUSE(self): - test_cases = ( - ("a? ( A ) !b? ( B ) !c? ( C ) d? ( D )", "A", ("a",)), - ("a? ( A ) !b? ( B ) !c? ( C ) d? ( D )", "B", ("b",)), - ("a? ( A ) !b? ( B ) !c? ( C ) d? ( D )", "C", ("c",)), - ("a? ( A ) !b? ( B ) !c? ( C ) d? ( D )", "D", ("d",)), - - ("a? ( b? ( AB ) )", "AB", ("a", "b")), - ("a? ( b? ( c? ( ABC ) ) )", "ABC", ("a", "b", "c")), - - ("a? ( A b? ( c? ( ABC ) AB ) )", "A", ("a",)), - ("a? ( A b? ( c? ( ABC ) AB ) )", "AB", ("a", "b")), - ("a? ( A b? ( c? ( ABC ) AB ) )", "ABC", ("a", "b", "c")), - ("a? ( A b? ( c? ( ABC ) AB ) ) X", "X", []), - ("X a? ( A b? ( c? ( ABC ) AB ) )", "X", []), - - ("ab? ( || ( A B ) )", "A", ("ab",)), - ("!ab? ( || ( A B ) )", "B", ("ab",)), - ("ab? ( || ( A || ( b? ( || ( B C ) ) ) ) )", "A", ("ab",)), - ("ab? ( || ( A || ( b? ( || ( B C ) ) ) ) )", "B", ("ab", "b")), - ("ab? ( || ( A || ( b? ( || ( B C ) ) ) ) )", "C", ("ab", "b")), - - ("( ab? ( || ( ( A ) || ( b? ( ( ( || ( B ( C ) ) ) ) ) ) ) ) )", "A", ("ab",)), - ("( ab? ( || ( ( A ) || ( b? ( ( ( || ( B ( C ) ) ) ) ) ) ) ) )", "B", ("ab", "b")), - ("( ab? ( || ( ( A ) || ( b? ( ( ( || ( B ( C ) ) ) ) ) ) ) ) )", "C", ("ab", "b")), - - ("a? ( A )", "B", []), - - ("a? ( || ( A B ) )", "B", ["a"]), - - # test USE dep defaults for bug #363073 - ("a? ( >=dev-lang/php-5.2[pcre(+)] )", ">=dev-lang/php-5.2[pcre(+)]", ["a"]), - ) - - test_cases_xfail = ( - ("? ( A )", "A"), - ("!? ( A )", "A"), - ("( A", "A"), - ("A )", "A"), - - ("||( A B )", "A"), - ("|| (A B )", "A"), - ("|| ( A B)", "A"), - ("|| ( A B", "A"), - ("|| A B )", "A"), - ("|| A B", "A"), - ("|| ( A B ) )", "A"), - ("|| || B C", "A"), - ("|| ( A B || )", "A"), - ("a? A", "A"), - ("( || ( || || ( A ) foo? ( B ) ) )", "A"), - ("( || ( || bar? ( A ) foo? ( B ) ) )", "A"), - ) - - for dep, atom, expected in test_cases: - expected = set(expected) - result = extract_affecting_use(dep, atom, eapi="0") - fail_msg = "dep: " + dep + ", atom: " + atom + ", got: " + \ - " ".join(sorted(result)) + ", expected: " + " ".join(sorted(expected)) - self.assertEqual(result, expected, fail_msg) - - for dep, atom in test_cases_xfail: - fail_msg = "dep: " + dep + ", atom: " + atom + ", got: " + \ - " ".join(sorted(result)) + ", expected: " + " ".join(sorted(expected)) - self.assertRaisesMsg(fail_msg, \ - InvalidDependString, extract_affecting_use, dep, atom, eapi="0") diff --git a/portage_with_autodep/pym/portage/tests/dep/testStandalone.py b/portage_with_autodep/pym/portage/tests/dep/testStandalone.py deleted file mode 100644 index e9f01df..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/testStandalone.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import cpvequal -from portage.exception import PortageException - -class TestStandalone(TestCase): - """ Test some small functions portage.dep - """ - - def testCPVequal(self): - - test_cases = ( - ( "sys-apps/portage-2.1","sys-apps/portage-2.1", True ), - ( "sys-apps/portage-2.1","sys-apps/portage-2.0", False ), - ( "sys-apps/portage-2.1","sys-apps/portage-2.1-r1", False ), - ( "sys-apps/portage-2.1-r1","sys-apps/portage-2.1", False ), - ( "sys-apps/portage-2.1_alpha3","sys-apps/portage-2.1", False ), - ( "sys-apps/portage-2.1_alpha3_p6","sys-apps/portage-2.1_alpha3", False ), - ( "sys-apps/portage-2.1_alpha3","sys-apps/portage-2.1", False ), - ( "sys-apps/portage-2.1","sys-apps/X-2.1", False ), - ( "sys-apps/portage-2.1","portage-2.1", False ), - ) - - test_cases_xfail = ( - ( "sys-apps/portage","sys-apps/portage" ), - ( "sys-apps/portage-2.1-6","sys-apps/portage-2.1-6" ), - ) - - for cpv1, cpv2, expected_result in test_cases: - self.assertEqual(cpvequal(cpv1, cpv2), expected_result) - - for cpv1, cpv2 in test_cases_xfail: - self.assertRaisesMsg("cpvequal("+cpv1+", "+cpv2+")", \ - PortageException, cpvequal, cpv1, cpv2) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_best_match_to_list.py b/portage_with_autodep/pym/portage/tests/dep/test_best_match_to_list.py deleted file mode 100644 index d050adc..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_best_match_to_list.py +++ /dev/null @@ -1,43 +0,0 @@ -# test_best_match_to_list.py -- Portage Unit Testing Functionality -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import Atom, best_match_to_list - -class Test_best_match_to_list(TestCase): - - def best_match_to_list_wrapper(self, mypkg, mylist): - """ - This function uses best_match_to_list to create sorted - list of matching atoms. - """ - ret = [] - while mylist: - m = best_match_to_list(mypkg, mylist) - if m is not None: - ret.append(m) - mylist.remove(m) - else: - break - - return ret - - def testBest_match_to_list(self): - tests = [ - ("dev-libs/A-1", [Atom("dev-libs/A"), Atom("=dev-libs/A-1")], \ - [Atom("=dev-libs/A-1"), Atom("dev-libs/A")]), - ("dev-libs/A-1", [Atom("dev-libs/B"), Atom("=dev-libs/A-1:0")], \ - [Atom("=dev-libs/A-1:0")]), - ("dev-libs/A-1", [Atom("dev-libs/*", allow_wildcard=True), Atom("=dev-libs/A-1:0")], \ - [Atom("=dev-libs/A-1:0"), Atom("dev-libs/*", allow_wildcard=True)]), - ("dev-libs/A-1:0", [Atom("dev-*/*", allow_wildcard=True), Atom("dev-*/*:0", allow_wildcard=True),\ - Atom("dev-libs/A"), Atom("<=dev-libs/A-2"), Atom("dev-libs/A:0"), \ - Atom("=dev-libs/A-1*"), Atom("~dev-libs/A-1"), Atom("=dev-libs/A-1")], \ - [Atom("=dev-libs/A-1"), Atom("~dev-libs/A-1"), Atom("=dev-libs/A-1*"), \ - Atom("dev-libs/A:0"), Atom("<=dev-libs/A-2"), Atom("dev-libs/A"), \ - Atom("dev-*/*:0", allow_wildcard=True), Atom("dev-*/*", allow_wildcard=True)]) - ] - - for pkg, atom_list, result in tests: - self.assertEqual( self.best_match_to_list_wrapper( pkg, atom_list ), result ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_dep_getcpv.py b/portage_with_autodep/pym/portage/tests/dep/test_dep_getcpv.py deleted file mode 100644 index 8a0a8aa..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_dep_getcpv.py +++ /dev/null @@ -1,35 +0,0 @@ -# test_dep_getcpv.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import dep_getcpv - -class DepGetCPV(TestCase): - """ A simple testcase for isvalidatom - """ - - def testDepGetCPV(self): - - prefix_ops = ["<", ">", "=", "~", "<=", - ">=", "!=", "!<", "!>", "!~"] - - bad_prefix_ops = [ ">~", "<~", "~>", "~<" ] - postfix_ops = [ ("=", "*"), ] - - cpvs = ["sys-apps/portage-2.1", "sys-apps/portage-2.1", - "sys-apps/portage-2.1"] - slots = [None, ":foo", ":2"] - for cpv in cpvs: - for slot in slots: - for prefix in prefix_ops: - mycpv = prefix + cpv - if slot: - mycpv += slot - self.assertEqual( dep_getcpv( mycpv ), cpv ) - - for prefix, postfix in postfix_ops: - mycpv = prefix + cpv + postfix - if slot: - mycpv += slot - self.assertEqual( dep_getcpv( mycpv ), cpv ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_dep_getrepo.py b/portage_with_autodep/pym/portage/tests/dep/test_dep_getrepo.py deleted file mode 100644 index 78ead8c..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_dep_getrepo.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import dep_getrepo - -class DepGetRepo(TestCase): - """ A simple testcase for isvalidatom - """ - - def testDepGetRepo(self): - - repo_char = "::" - repos = ( "a", "repo-name", "repo_name", "repo123", None ) - cpvs = ["sys-apps/portage"] - versions = ["2.1.1","2.1-r1", None] - uses = ["[use]", None] - for cpv in cpvs: - for version in versions: - for use in uses: - for repo in repos: - pkg = cpv - if version: - pkg = '=' + pkg + '-' + version - if repo is not None: - pkg = pkg + repo_char + repo - if use: - pkg = pkg + use - self.assertEqual( dep_getrepo( pkg ), repo ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_dep_getslot.py b/portage_with_autodep/pym/portage/tests/dep/test_dep_getslot.py deleted file mode 100644 index 206cecc..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_dep_getslot.py +++ /dev/null @@ -1,28 +0,0 @@ -# test_dep_getslot.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import dep_getslot - -class DepGetSlot(TestCase): - """ A simple testcase for isvalidatom - """ - - def testDepGetSlot(self): - - slot_char = ":" - slots = ( "a", "1.2", "1", "IloveVapier", None ) - cpvs = ["sys-apps/portage"] - versions = ["2.1.1","2.1-r1"] - for cpv in cpvs: - for version in versions: - for slot in slots: - mycpv = cpv - if version: - mycpv = '=' + mycpv + '-' + version - if slot is not None: - self.assertEqual( dep_getslot( - mycpv + slot_char + slot ), slot ) - else: - self.assertEqual( dep_getslot( mycpv ), slot ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_dep_getusedeps.py b/portage_with_autodep/pym/portage/tests/dep/test_dep_getusedeps.py deleted file mode 100644 index d2494f7..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_dep_getusedeps.py +++ /dev/null @@ -1,35 +0,0 @@ -# test_dep_getusedeps.py -- Portage Unit Testing Functionality -# Copyright 2007-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import dep_getusedeps - -from portage.tests import test_cps, test_slots, test_versions, test_usedeps - -class DepGetUseDeps(TestCase): - """ A simple testcase for dep_getusedeps - """ - - def testDepGetUseDeps(self): - - for mycpv in test_cps: - for version in test_versions: - for slot in test_slots: - for use in test_usedeps: - cpv = mycpv[:] - if version: - cpv += version - if slot: - cpv += ":" + slot - if isinstance(use, tuple): - cpv += "[%s]" % (",".join(use),) - self.assertEqual( dep_getusedeps( - cpv ), use ) - else: - if len(use): - self.assertEqual( dep_getusedeps( - cpv + "[" + use + "]" ), (use,) ) - else: - self.assertEqual( dep_getusedeps( - cpv + "[" + use + "]" ), () ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_get_operator.py b/portage_with_autodep/pym/portage/tests/dep/test_get_operator.py deleted file mode 100644 index 4f9848f..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_get_operator.py +++ /dev/null @@ -1,33 +0,0 @@ -# test_get_operator.py -- Portage Unit Testing Functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import get_operator - -class GetOperator(TestCase): - - def testGetOperator(self): - - # get_operator does not validate operators - tests = [ ( "~", "~" ), ( "=", "=" ), ( ">", ">" ), - ( ">=", ">=" ), ( "<=", "<=" ), - ] - - test_cpvs = ["sys-apps/portage-2.1"] - slots = [ None,"1","linux-2.5.6" ] - for cpv in test_cpvs: - for test in tests: - for slot in slots: - atom = cpv[:] - if slot: - atom += ":" + slot - result = get_operator( test[0] + atom ) - self.assertEqual( result, test[1], - msg="get_operator(%s) != %s" % (test[0] + atom, test[1]) ) - - result = get_operator( "sys-apps/portage" ) - self.assertEqual( result, None ) - - result = get_operator( "=sys-apps/portage-2.1*" ) - self.assertEqual( result , "=*" ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_get_required_use_flags.py b/portage_with_autodep/pym/portage/tests/dep/test_get_required_use_flags.py deleted file mode 100644 index 06f8110..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_get_required_use_flags.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import get_required_use_flags -from portage.exception import InvalidDependString - -class TestCheckRequiredUse(TestCase): - - def testCheckRequiredUse(self): - test_cases = ( - ("a b c", ["a", "b", "c"]), - - ("|| ( a b c )", ["a", "b", "c"]), - ("^^ ( a b c )", ["a", "b", "c"]), - - ("|| ( a b ^^ ( d e f ) )", ["a", "b", "d", "e", "f"]), - ("^^ ( a b || ( d e f ) )", ["a", "b", "d", "e", "f"]), - - ("( ^^ ( a ( b ) ( || ( ( d e ) ( f ) ) ) ) )", ["a", "b", "d", "e", "f"]), - - ("a? ( ^^ ( b c ) )", ["a", "b", "c"]), - ("a? ( ^^ ( !b !d? ( c ) ) )", ["a", "b", "c", "d"]), - ) - - test_cases_xfail = ( - ("^^ ( || ( a b ) ^^ ( b c )"), - ("^^( || ( a b ) ^^ ( b c ) )"), - ("^^ || ( a b ) ^^ ( b c )"), - ("^^ ( ( || ) ( a b ) ^^ ( b c ) )"), - ("^^ ( || ( a b ) ) ^^ ( b c ) )"), - ) - - for required_use, expected in test_cases: - result = get_required_use_flags(required_use) - expected = set(expected) - self.assertEqual(result, expected, \ - "REQUIRED_USE: '%s', expected: '%s', got: '%s'" % (required_use, expected, result)) - - for required_use in test_cases_xfail: - self.assertRaisesMsg("REQUIRED_USE: '%s'" % (required_use,), \ - InvalidDependString, get_required_use_flags, required_use) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_isjustname.py b/portage_with_autodep/pym/portage/tests/dep/test_isjustname.py deleted file mode 100644 index c16fb54..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_isjustname.py +++ /dev/null @@ -1,24 +0,0 @@ -# test_isjustname.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import isjustname - -class IsJustName(TestCase): - - def testIsJustName(self): - - cats = ( "", "sys-apps/", "foo/", "virtual/" ) - pkgs = ( "portage", "paludis", "pkgcore", "notARealPkg" ) - vers = ( "", "-2.0-r3", "-1.0_pre2", "-3.1b" ) - - for pkg in pkgs: - for cat in cats: - for ver in vers: - if len(ver): - self.assertFalse( isjustname( cat + pkg + ver ), - msg="isjustname(%s) is True!" % (cat + pkg + ver) ) - else: - self.assertTrue( isjustname( cat + pkg + ver ), - msg="isjustname(%s) is False!" % (cat + pkg + ver) ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_isvalidatom.py b/portage_with_autodep/pym/portage/tests/dep/test_isvalidatom.py deleted file mode 100644 index 173ab0d..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_isvalidatom.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2006-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import isvalidatom - -class IsValidAtomTestCase(object): - def __init__(self, atom, expected, allow_wildcard=False, allow_repo=False): - self.atom = atom - self.expected = expected - self.allow_wildcard = allow_wildcard - self.allow_repo = allow_repo - -class IsValidAtom(TestCase): - - def testIsValidAtom(self): - - test_cases = ( - IsValidAtomTestCase("sys-apps/portage", True), - IsValidAtomTestCase("=sys-apps/portage-2.1", True), - IsValidAtomTestCase("=sys-apps/portage-2.1*", True), - IsValidAtomTestCase(">=sys-apps/portage-2.1", True), - IsValidAtomTestCase("<=sys-apps/portage-2.1", True), - IsValidAtomTestCase(">sys-apps/portage-2.1", True), - IsValidAtomTestCase("<sys-apps/portage-2.1", True), - IsValidAtomTestCase("~sys-apps/portage-2.1", True), - IsValidAtomTestCase("sys-apps/portage:foo", True), - IsValidAtomTestCase("sys-apps/portage-2.1:foo", False), - IsValidAtomTestCase( "sys-apps/portage-2.1:", False), - IsValidAtomTestCase("sys-apps/portage-2.1:", False), - IsValidAtomTestCase("sys-apps/portage-2.1:[foo]", False), - IsValidAtomTestCase("sys-apps/portage", True), - IsValidAtomTestCase("sys-apps/portage", True), - IsValidAtomTestCase("sys-apps/portage", True), - IsValidAtomTestCase("sys-apps/portage", True), - IsValidAtomTestCase("sys-apps/portage", True), - IsValidAtomTestCase("sys-apps/portage", True), - IsValidAtomTestCase("sys-apps/portage", True), - - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[bar?,!baz?,!doc=,build=]", True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[doc?]", True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[!doc?]", True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[doc=]", True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[!doc=]", True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[!doc]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[!-doc]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[!-doc=]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[!-doc?]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[-doc?]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[-doc=]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[-doc!=]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[-doc=]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[bar][-baz][doc?][!build?]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[bar,-baz,doc?,!build?]", True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[bar,-baz,doc?,!build?,]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[,bar,-baz,doc?,!build?]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[bar,-baz][doc?,!build?]", False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo[bar][doc,build]", False), - IsValidAtomTestCase(">~cate-gory/foo-1.0", False), - IsValidAtomTestCase(">~category/foo-1.0", False), - IsValidAtomTestCase("<~category/foo-1.0", False), - IsValidAtomTestCase("###cat/foo-1.0", False), - IsValidAtomTestCase("~sys-apps/portage", False), - IsValidAtomTestCase("portage", False), - IsValidAtomTestCase("=portage", False), - IsValidAtomTestCase(">=portage-2.1", False), - IsValidAtomTestCase("~portage-2.1", False), - IsValidAtomTestCase("=portage-2.1*", False), - IsValidAtomTestCase("null/portage", True), - IsValidAtomTestCase("null/portage*:0", False), - IsValidAtomTestCase(">=null/portage-2.1", True), - IsValidAtomTestCase(">=null/portage", False), - IsValidAtomTestCase(">null/portage", False), - IsValidAtomTestCase("=null/portage*", False), - IsValidAtomTestCase("=null/portage", False), - IsValidAtomTestCase("~null/portage", False), - IsValidAtomTestCase("<=null/portage", False), - IsValidAtomTestCase("<null/portage", False), - IsValidAtomTestCase("~null/portage-2.1", True), - IsValidAtomTestCase("=null/portage-2.1*", True), - IsValidAtomTestCase("null/portage-2.1*", False), - IsValidAtomTestCase("app-doc/php-docs-20071125", False), - IsValidAtomTestCase("app-doc/php-docs-20071125-r2", False), - IsValidAtomTestCase("=foo/bar-1-r1-1-r1", False), - IsValidAtomTestCase("foo/-z-1", False), - - # These are invalid because pkg name must not end in hyphen - # followed by numbers - IsValidAtomTestCase("=foo/bar-1-r1-1-r1", False), - IsValidAtomTestCase("=foo/bar-123-1", False), - IsValidAtomTestCase("=foo/bar-123-1*", False), - IsValidAtomTestCase("foo/bar-123", False), - IsValidAtomTestCase("=foo/bar-123-1-r1", False), - IsValidAtomTestCase("=foo/bar-123-1-r1*", False), - IsValidAtomTestCase("foo/bar-123-r1", False), - IsValidAtomTestCase("foo/bar-1", False), - - IsValidAtomTestCase("=foo/bar--baz-1-r1", True), - IsValidAtomTestCase("=foo/bar-baz--1-r1", True), - IsValidAtomTestCase("=foo/bar-baz---1-r1", True), - IsValidAtomTestCase("=foo/bar-baz---1", True), - IsValidAtomTestCase("=foo/bar-baz-1--r1", False), - IsValidAtomTestCase("games-strategy/ufo2000", True), - IsValidAtomTestCase("~games-strategy/ufo2000-0.1", True), - IsValidAtomTestCase("=media-libs/x264-20060810", True), - IsValidAtomTestCase("foo/b", True), - IsValidAtomTestCase("app-text/7plus", True), - IsValidAtomTestCase("foo/666", True), - IsValidAtomTestCase("=dev-libs/poppler-qt3-0.11*", True), - - #Testing atoms with repositories - IsValidAtomTestCase("sys-apps/portage::repo_123-name", True, allow_repo=True), - IsValidAtomTestCase("=sys-apps/portage-2.1::repo", True, allow_repo=True), - IsValidAtomTestCase("=sys-apps/portage-2.1*::repo", True, allow_repo=True), - IsValidAtomTestCase("sys-apps/portage:foo::repo", True, allow_repo=True), - IsValidAtomTestCase("sys-apps/portage-2.1:foo::repo", False, allow_repo=True), - IsValidAtomTestCase("sys-apps/portage-2.1:::repo", False, allow_repo=True), - IsValidAtomTestCase("sys-apps/portage-2.1:::repo[foo]", False, allow_repo=True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[bar?,!baz?,!doc=,build=]", True, allow_repo=True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[doc?]", True, allow_repo=True), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[!doc]", False, allow_repo=True), - IsValidAtomTestCase("###cat/foo-1.0::repo", False, allow_repo=True), - IsValidAtomTestCase("~sys-apps/portage::repo", False, allow_repo=True), - IsValidAtomTestCase("portage::repo", False, allow_repo=True), - IsValidAtomTestCase("=portage::repo", False, allow_repo=True), - IsValidAtomTestCase("null/portage::repo", True, allow_repo=True), - IsValidAtomTestCase("app-doc/php-docs-20071125::repo", False, allow_repo=True), - IsValidAtomTestCase("=foo/bar-1-r1-1-r1::repo", False, allow_repo=True), - - IsValidAtomTestCase("sys-apps/portage::repo_123-name", False, allow_repo=False), - IsValidAtomTestCase("=sys-apps/portage-2.1::repo", False, allow_repo=False), - IsValidAtomTestCase("=sys-apps/portage-2.1*::repo", False, allow_repo=False), - IsValidAtomTestCase("sys-apps/portage:foo::repo", False, allow_repo=False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[bar?,!baz?,!doc=,build=]", False, allow_repo=False), - IsValidAtomTestCase("=sys-apps/portage-2.2*:foo::repo[doc?]", False, allow_repo=False), - IsValidAtomTestCase("null/portage::repo", False, allow_repo=False), - ) - - for test_case in test_cases: - if test_case.expected: - atom_type = "valid" - else: - atom_type = "invalid" - self.assertEqual( bool(isvalidatom(test_case.atom, allow_wildcard=test_case.allow_wildcard, \ - allow_repo=test_case.allow_repo)), test_case.expected, - msg="isvalidatom(%s) != %s" % ( test_case.atom, test_case.expected ) ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_match_from_list.py b/portage_with_autodep/pym/portage/tests/dep/test_match_from_list.py deleted file mode 100644 index afba414..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_match_from_list.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2006, 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import sys -from portage.tests import TestCase -from portage.dep import Atom, match_from_list, _repo_separator -from portage.versions import catpkgsplit - -if sys.hexversion >= 0x3000000: - basestring = str - -class Package(object): - """ - Provides a minimal subset of attributes of _emerge.Package.Package - """ - def __init__(self, atom): - atom = Atom(atom, allow_repo=True) - self.cp = atom.cp - self.cpv = atom.cpv - self.cpv_split = catpkgsplit(self.cpv) - self.slot = atom.slot - self.repo = atom.repo - if atom.use: - self.use = self._use_class(atom.use.enabled) - self.iuse = self._iuse_class(atom.use.required) - else: - self.use = self._use_class([]) - self.iuse = self._iuse_class([]) - - class _use_class(object): - def __init__(self, use): - self.enabled = frozenset(use) - - class _iuse_class(object): - def __init__(self, iuse): - self.all = frozenset(iuse) - - def is_valid_flag(self, flags): - if isinstance(flags, basestring): - flags = [flags] - for flag in flags: - if not flag in self.all: - return False - return True - -class Test_match_from_list(TestCase): - - def testMatch_from_list(self): - tests = ( - ("=sys-apps/portage-45*", [], [] ), - ("=sys-apps/portage-45*", ["sys-apps/portage-045"], ["sys-apps/portage-045"] ), - ("!=sys-apps/portage-45*", ["sys-apps/portage-045"], ["sys-apps/portage-045"] ), - ("!!=sys-apps/portage-45*", ["sys-apps/portage-045"], ["sys-apps/portage-045"] ), - ("=sys-apps/portage-045", ["sys-apps/portage-045"], ["sys-apps/portage-045"] ), - ("=sys-apps/portage-045", ["sys-apps/portage-046"], [] ), - ("~sys-apps/portage-045", ["sys-apps/portage-045-r1"], ["sys-apps/portage-045-r1"] ), - ("~sys-apps/portage-045", ["sys-apps/portage-046-r1"], [] ), - ("<=sys-apps/portage-045", ["sys-apps/portage-045"], ["sys-apps/portage-045"] ), - ("<=sys-apps/portage-045", ["sys-apps/portage-046"], [] ), - ("<sys-apps/portage-046", ["sys-apps/portage-045"], ["sys-apps/portage-045"] ), - ("<sys-apps/portage-046", ["sys-apps/portage-046"], [] ), - (">=sys-apps/portage-045", ["sys-apps/portage-045"], ["sys-apps/portage-045"] ), - (">=sys-apps/portage-047", ["sys-apps/portage-046-r1"], [] ), - (">sys-apps/portage-044", ["sys-apps/portage-045"], ["sys-apps/portage-045"] ), - (">sys-apps/portage-047", ["sys-apps/portage-046-r1"], [] ), - ("sys-apps/portage:0", [Package("=sys-apps/portage-045:0")], ["sys-apps/portage-045"] ), - ("sys-apps/portage:0", [Package("=sys-apps/portage-045:1")], [] ), - ("=sys-fs/udev-1*", ["sys-fs/udev-123"], ["sys-fs/udev-123"]), - ("=sys-fs/udev-4*", ["sys-fs/udev-456"], ["sys-fs/udev-456"] ), - ("*/*", ["sys-fs/udev-456"], ["sys-fs/udev-456"] ), - ("sys-fs/*", ["sys-fs/udev-456"], ["sys-fs/udev-456"] ), - ("*/udev", ["sys-fs/udev-456"], ["sys-fs/udev-456"] ), - ("=sys-apps/portage-2*", ["sys-apps/portage-2.1"], ["sys-apps/portage-2.1"] ), - ("=sys-apps/portage-2.1*", ["sys-apps/portage-2.1.2"], ["sys-apps/portage-2.1.2"] ), - ("dev-libs/*", ["sys-apps/portage-2.1.2"], [] ), - ("*/tar", ["sys-apps/portage-2.1.2"], [] ), - ("*/*", ["dev-libs/A-1", "dev-libs/B-1"], ["dev-libs/A-1", "dev-libs/B-1"] ), - ("dev-libs/*", ["dev-libs/A-1", "sci-libs/B-1"], ["dev-libs/A-1"] ), - - ("dev-libs/A[foo]", [Package("=dev-libs/A-1[foo]"), Package("=dev-libs/A-2[-foo]")], ["dev-libs/A-1"] ), - ("dev-libs/A[-foo]", [Package("=dev-libs/A-1[foo]"), Package("=dev-libs/A-2[-foo]")], ["dev-libs/A-2"] ), - ("dev-libs/A[-foo]", [Package("=dev-libs/A-1[foo]"), Package("=dev-libs/A-2")], [] ), - ("dev-libs/A[foo,bar]", [Package("=dev-libs/A-1[foo]"), Package("=dev-libs/A-2[-foo]")], [] ), - ("dev-libs/A[foo,bar]", [Package("=dev-libs/A-1[foo]"), Package("=dev-libs/A-2[-foo,bar]")], [] ), - ("dev-libs/A[foo,bar]", [Package("=dev-libs/A-1[foo]"), Package("=dev-libs/A-2[foo,bar]")], ["dev-libs/A-2"] ), - ("dev-libs/A[foo,bar(+)]", [Package("=dev-libs/A-1[-foo]"), Package("=dev-libs/A-2[foo]")], ["dev-libs/A-2"] ), - ("dev-libs/A[foo,bar(-)]", [Package("=dev-libs/A-1[-foo]"), Package("=dev-libs/A-2[foo]")], [] ), - ("dev-libs/A[foo,-bar(-)]", [Package("=dev-libs/A-1[-foo,bar]"), Package("=dev-libs/A-2[foo]")], ["dev-libs/A-2"] ), - - ("dev-libs/A::repo1", [Package("=dev-libs/A-1::repo1"), Package("=dev-libs/A-1::repo2")], ["dev-libs/A-1::repo1"] ), - ("dev-libs/A::repo2", [Package("=dev-libs/A-1::repo1"), Package("=dev-libs/A-1::repo2")], ["dev-libs/A-1::repo2"] ), - ("dev-libs/A::repo2[foo]", [Package("=dev-libs/A-1::repo1[foo]"), Package("=dev-libs/A-1::repo2[-foo]")], [] ), - ("dev-libs/A::repo2[foo]", [Package("=dev-libs/A-1::repo1[-foo]"), Package("=dev-libs/A-1::repo2[foo]")], ["dev-libs/A-1::repo2"] ), - ("dev-libs/A:1::repo2[foo]", [Package("=dev-libs/A-1:1::repo1"), Package("=dev-libs/A-1:2::repo2")], [] ), - ("dev-libs/A:1::repo2[foo]", [Package("=dev-libs/A-1:2::repo1"), Package("=dev-libs/A-1:1::repo2[foo]")], ["dev-libs/A-1::repo2"] ), - ) - - for atom, cpv_list, expected_result in tests: - result = [] - for pkg in match_from_list( atom, cpv_list ): - if isinstance(pkg, Package): - if pkg.repo: - result.append(pkg.cpv + _repo_separator + pkg.repo) - else: - result.append(pkg.cpv) - else: - result.append(pkg) - self.assertEqual( result, expected_result ) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_paren_reduce.py b/portage_with_autodep/pym/portage/tests/dep/test_paren_reduce.py deleted file mode 100644 index 9a147a0..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_paren_reduce.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.dep import paren_reduce -from portage.exception import InvalidDependString - -class TestParenReduce(TestCase): - - def testParenReduce(self): - - test_cases = ( - ( "A", ["A"]), - ( "( A )", ["A"]), - ( "|| ( A B )", [ "||", ["A", "B"] ]), - ( "|| ( A || ( B C ) )", [ "||", ["A", "||", ["B", "C"]]]), - ( "|| ( A || ( B C D ) )", [ "||", ["A", "||", ["B", "C", "D"]] ]), - ( "|| ( A || ( B || ( C D ) E ) )", [ "||", ["A", "||", ["B", "||", ["C", "D"], "E"]] ]), - ( "a? ( A )", ["a?", ["A"]]), - - ( "( || ( ( ( A ) B ) ) )", ["A", "B"]), - ( "( || ( || ( ( A ) B ) ) )", [ "||", ["A", "B"] ]), - ( "|| ( A )", ["A"]), - ( "( || ( || ( || ( A ) foo? ( B ) ) ) )", [ "||", ["A", "foo?", ["B"] ]]), - ( "( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )", [ "||", ["bar?", ["A"], "foo?", ["B"] ]]), - ( "A || ( ) foo? ( ) B", ["A", "B"]), - - ( "|| ( A ) || ( B )", ["A", "B"]), - ( "foo? ( A ) foo? ( B )", ["foo?", ["A"], "foo?", ["B"]]), - - ( "|| ( ( A B ) C )", [ "||", [ ["A", "B"], "C"] ]), - ( "|| ( ( A B ) ( C ) )", [ "||", [ ["A", "B"], "C"] ]), - # test USE dep defaults for bug #354003 - ( ">=dev-lang/php-5.2[pcre(+)]", [ ">=dev-lang/php-5.2[pcre(+)]" ]), - ) - - test_cases_xfail = ( - "( A", - "A )", - - "||( A B )", - "|| (A B )", - "|| ( A B)", - "|| ( A B", - "|| A B )", - - "|| A B", - "|| ( A B ) )", - "|| || B C", - - "|| ( A B || )", - - "a? A", - - ( "( || ( || || ( A ) foo? ( B ) ) )"), - ( "( || ( || bar? ( A ) foo? ( B ) ) )"), - ) - - for dep_str, expected_result in test_cases: - self.assertEqual(paren_reduce(dep_str), expected_result, - "input: '%s' result: %s != %s" % (dep_str, - paren_reduce(dep_str), expected_result)) - - for dep_str in test_cases_xfail: - self.assertRaisesMsg(dep_str, - InvalidDependString, paren_reduce, dep_str) diff --git a/portage_with_autodep/pym/portage/tests/dep/test_use_reduce.py b/portage_with_autodep/pym/portage/tests/dep/test_use_reduce.py deleted file mode 100644 index 1618430..0000000 --- a/portage_with_autodep/pym/portage/tests/dep/test_use_reduce.py +++ /dev/null @@ -1,627 +0,0 @@ -# Copyright 2009-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.exception import InvalidDependString -from portage.dep import Atom, use_reduce - -class UseReduceTestCase(object): - def __init__(self, deparray, uselist=[], masklist=[], \ - matchall=0, excludeall=[], is_src_uri=False, \ - eapi="0", opconvert=False, flat=False, expected_result=None, \ - is_valid_flag=None, token_class=None): - self.deparray = deparray - self.uselist = uselist - self.masklist = masklist - self.matchall = matchall - self.excludeall = excludeall - self.is_src_uri = is_src_uri - self.eapi = eapi - self.opconvert = opconvert - self.flat = flat - self.is_valid_flag = is_valid_flag - self.token_class = token_class - self.expected_result = expected_result - - def run(self): - try: - return use_reduce(self.deparray, self.uselist, self.masklist, \ - self.matchall, self.excludeall, self.is_src_uri, self.eapi, \ - self.opconvert, self.flat, self.is_valid_flag, self.token_class) - except InvalidDependString as e: - raise InvalidDependString("%s: %s" % (e, self.deparray)) - -class UseReduce(TestCase): - - def always_true(self, ununsed_parameter): - return True - - def always_false(self, ununsed_parameter): - return False - - def testUseReduce(self): - - EAPI_WITH_SRC_URI_ARROWS = "2" - EAPI_WITHOUT_SRC_URI_ARROWS = "0" - - test_cases = ( - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - uselist = ["a", "b", "c", "d"], - expected_result = ["A", "B"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - uselist = ["a", "b", "c"], - expected_result = ["A", "B", "D"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - uselist = ["b", "c"], - expected_result = ["B", "D"] - ), - - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - matchall = True, - expected_result = ["A", "B", "C", "D"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - masklist = ["a", "c"], - expected_result = ["C", "D"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - matchall = True, - masklist = ["a", "c"], - expected_result = ["B", "C", "D"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - uselist = ["a", "b"], - masklist = ["a", "c"], - expected_result = ["B", "C", "D"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - excludeall = ["a", "c"], - expected_result = ["D"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - uselist = ["b"], - excludeall = ["a", "c"], - expected_result = ["B", "D"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - matchall = True, - excludeall = ["a", "c"], - expected_result = ["A", "B", "D"] - ), - UseReduceTestCase( - "a? ( A ) b? ( B ) !c? ( C ) !d? ( D )", - matchall = True, - excludeall = ["a", "c"], - masklist = ["b"], - expected_result = ["A", "D"] - ), - - - UseReduceTestCase( - "a? ( b? ( AB ) )", - uselist = ["a", "b"], - expected_result = ["AB"] - ), - UseReduceTestCase( - "a? ( b? ( AB ) C )", - uselist = ["a"], - expected_result = ["C"] - ), - UseReduceTestCase( - "a? ( b? ( || ( AB CD ) ) )", - uselist = ["a", "b"], - expected_result = ["||", ["AB", "CD"]] - ), - UseReduceTestCase( - "|| ( || ( a? ( A ) b? ( B ) ) )", - uselist = ["a", "b"], - expected_result = ["||", ["A", "B"]] - ), - UseReduceTestCase( - "|| ( || ( a? ( A ) b? ( B ) ) )", - uselist = ["a"], - expected_result = ["A"] - ), - UseReduceTestCase( - "|| ( || ( a? ( A ) b? ( B ) ) )", - uselist = [], - expected_result = [] - ), - UseReduceTestCase( - "|| ( || ( a? ( || ( A c? ( C ) ) ) b? ( B ) ) )", - uselist = [], - expected_result = [] - ), - UseReduceTestCase( - "|| ( || ( a? ( || ( A c? ( C ) ) ) b? ( B ) ) )", - uselist = ["a"], - expected_result = ["A"] - ), - UseReduceTestCase( - "|| ( || ( a? ( || ( A c? ( C ) ) ) b? ( B ) ) )", - uselist = ["b"], - expected_result = ["B"] - ), - UseReduceTestCase( - "|| ( || ( a? ( || ( A c? ( C ) ) ) b? ( B ) ) )", - uselist = ["c"], - expected_result = [] - ), - UseReduceTestCase( - "|| ( || ( a? ( || ( A c? ( C ) ) ) b? ( B ) ) )", - uselist = ["a", "c"], - expected_result = ["||", [ "A", "C"]] - ), - - #paren_reduce tests - UseReduceTestCase( - "A", - expected_result = ["A"]), - UseReduceTestCase( - "( A )", - expected_result = ["A"]), - UseReduceTestCase( - "|| ( A B )", - expected_result = [ "||", ["A", "B"] ]), - UseReduceTestCase( - "|| ( ( A B ) C )", - expected_result = [ "||", [ ["A", "B"], "C"] ]), - UseReduceTestCase( - "|| ( ( A B ) ( C ) )", - expected_result = [ "||", [ ["A", "B"], "C"] ]), - UseReduceTestCase( - "|| ( A || ( B C ) )", - expected_result = [ "||", ["A", "B", "C"]]), - UseReduceTestCase( - "|| ( A || ( B C D ) )", - expected_result = [ "||", ["A", "B", "C", "D"] ]), - UseReduceTestCase( - "|| ( A || ( B || ( C D ) E ) )", - expected_result = [ "||", ["A", "B", "C", "D", "E"] ]), - UseReduceTestCase( - "( || ( ( ( A ) B ) ) )", - expected_result = ["A", "B"] ), - UseReduceTestCase( - "( || ( || ( ( A ) B ) ) )", - expected_result = [ "||", ["A", "B"] ]), - UseReduceTestCase( - "( || ( || ( ( A ) B ) ) )", - expected_result = [ "||", ["A", "B"] ]), - UseReduceTestCase( - "|| ( A )", - expected_result = ["A"]), - UseReduceTestCase( - "( || ( || ( || ( A ) foo? ( B ) ) ) )", - expected_result = ["A"]), - UseReduceTestCase( - "( || ( || ( || ( A ) foo? ( B ) ) ) )", - uselist = ["foo"], - expected_result = [ "||", ["A", "B"] ]), - UseReduceTestCase( - "( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )", - expected_result = []), - UseReduceTestCase( - "( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )", - uselist = ["foo", "bar"], - expected_result = [ "||", [ "A", "B" ] ]), - UseReduceTestCase( - "A || ( bar? ( C ) ) foo? ( bar? ( C ) ) B", - expected_result = ["A", "B"]), - UseReduceTestCase( - "|| ( A ) || ( B )", - expected_result = ["A", "B"]), - UseReduceTestCase( - "foo? ( A ) foo? ( B )", - expected_result = []), - UseReduceTestCase( - "foo? ( A ) foo? ( B )", - uselist = ["foo"], - expected_result = ["A", "B"]), - UseReduceTestCase( - "|| ( A B ) C", - expected_result = ['||', ['A', 'B'], 'C']), - UseReduceTestCase( - "A || ( B C )", - expected_result = ['A', '||', ['B', 'C']]), - - #SRC_URI stuff - UseReduceTestCase( - "http://foo/bar -> blah.tbz2", - is_src_uri = True, - eapi = EAPI_WITH_SRC_URI_ARROWS, - expected_result = ["http://foo/bar", "->", "blah.tbz2"]), - UseReduceTestCase( - "foo? ( http://foo/bar -> blah.tbz2 )", - uselist = [], - is_src_uri = True, - eapi = EAPI_WITH_SRC_URI_ARROWS, - expected_result = []), - UseReduceTestCase( - "foo? ( http://foo/bar -> blah.tbz2 )", - uselist = ["foo"], - is_src_uri = True, - eapi = EAPI_WITH_SRC_URI_ARROWS, - expected_result = ["http://foo/bar", "->", "blah.tbz2"]), - UseReduceTestCase( - "http://foo/bar -> bar.tbz2 foo? ( ftp://foo/a )", - uselist = [], - is_src_uri = True, - eapi = EAPI_WITH_SRC_URI_ARROWS, - expected_result = ["http://foo/bar", "->", "bar.tbz2"]), - UseReduceTestCase( - "http://foo/bar -> bar.tbz2 foo? ( ftp://foo/a )", - uselist = ["foo"], - is_src_uri = True, - eapi = EAPI_WITH_SRC_URI_ARROWS, - expected_result = ["http://foo/bar", "->", "bar.tbz2", "ftp://foo/a"]), - UseReduceTestCase( - "http://foo.com/foo http://foo/bar -> blah.tbz2", - uselist = ["foo"], - is_src_uri = True, - eapi = EAPI_WITH_SRC_URI_ARROWS, - expected_result = ["http://foo.com/foo", "http://foo/bar", "->", "blah.tbz2"]), - - #opconvert tests - UseReduceTestCase( - "A", - opconvert = True, - expected_result = ["A"]), - UseReduceTestCase( - "( A )", - opconvert = True, - expected_result = ["A"]), - UseReduceTestCase( - "|| ( A B )", - opconvert = True, - expected_result = [['||', 'A', 'B']]), - UseReduceTestCase( - "|| ( ( A B ) C )", - opconvert = True, - expected_result = [['||', ['A', 'B'], 'C']]), - UseReduceTestCase( - "|| ( A || ( B C ) )", - opconvert = True, - expected_result = [['||', 'A', 'B', 'C']]), - UseReduceTestCase( - "|| ( A || ( B C D ) )", - opconvert = True, - expected_result = [['||', 'A', 'B', 'C', 'D']]), - UseReduceTestCase( - "|| ( A || ( B || ( C D ) E ) )", - expected_result = [ "||", ["A", "B", "C", "D", "E"] ]), - UseReduceTestCase( - "( || ( ( ( A ) B ) ) )", - opconvert = True, - expected_result = [ "A", "B" ] ), - UseReduceTestCase( - "( || ( || ( ( A ) B ) ) )", - opconvert = True, - expected_result = [['||', 'A', 'B']]), - UseReduceTestCase( - "|| ( A B ) C", - opconvert = True, - expected_result = [['||', 'A', 'B'], 'C']), - UseReduceTestCase( - "A || ( B C )", - opconvert = True, - expected_result = ['A', ['||', 'B', 'C']]), - UseReduceTestCase( - "A foo? ( || ( B || ( bar? ( || ( C D E ) ) !bar? ( F ) ) ) ) G", - uselist = ["foo", "bar"], - opconvert = True, - expected_result = ['A', ['||', 'B', 'C', 'D', 'E'], 'G']), - UseReduceTestCase( - "A foo? ( || ( B || ( bar? ( || ( C D E ) ) !bar? ( F ) ) ) ) G", - uselist = ["foo", "bar"], - opconvert = False, - expected_result = ['A', '||', ['B', 'C', 'D', 'E'], 'G']), - - UseReduceTestCase( - "|| ( A )", - opconvert = True, - expected_result = ["A"]), - UseReduceTestCase( - "( || ( || ( || ( A ) foo? ( B ) ) ) )", - expected_result = ["A"]), - UseReduceTestCase( - "( || ( || ( || ( A ) foo? ( B ) ) ) )", - uselist = ["foo"], - opconvert = True, - expected_result = [['||', 'A', 'B']]), - UseReduceTestCase( - "( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )", - opconvert = True, - expected_result = []), - UseReduceTestCase( - "( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )", - uselist = ["foo", "bar"], - opconvert = True, - expected_result = [['||', 'A', 'B']]), - UseReduceTestCase( - "A || ( bar? ( C ) ) foo? ( bar? ( C ) ) B", - opconvert = True, - expected_result = ["A", "B"]), - UseReduceTestCase( - "|| ( A ) || ( B )", - opconvert = True, - expected_result = ["A", "B"]), - UseReduceTestCase( - "foo? ( A ) foo? ( B )", - opconvert = True, - expected_result = []), - UseReduceTestCase( - "foo? ( A ) foo? ( B )", - uselist = ["foo"], - opconvert = True, - expected_result = ["A", "B"]), - UseReduceTestCase( - "|| ( foo? ( || ( A B ) ) )", - uselist = ["foo"], - opconvert = True, - expected_result = [['||', 'A', 'B']]), - - UseReduceTestCase( - "|| ( ( A B ) foo? ( || ( C D ) ) )", - uselist = ["foo"], - opconvert = True, - expected_result = [['||', ['A', 'B'], 'C', 'D']]), - - UseReduceTestCase( - "|| ( ( A B ) foo? ( || ( C D ) ) )", - uselist = ["foo"], - opconvert = False, - expected_result = ['||', [['A', 'B'], 'C', 'D']]), - - UseReduceTestCase( - "|| ( ( A B ) || ( C D ) )", - expected_result = ['||', [['A', 'B'], 'C', 'D']]), - - UseReduceTestCase( - "|| ( ( A B ) || ( C D || ( E ( F G ) || ( H ) ) ) )", - expected_result = ['||', [['A', 'B'], 'C', 'D', 'E', ['F', 'G'], 'H']]), - - UseReduceTestCase( - "|| ( ( A B ) || ( C D || ( E ( F G ) || ( H ) ) ) )", - opconvert = True, - expected_result = [['||', ['A', 'B'], 'C', 'D', 'E', ['F', 'G'], 'H']]), - - UseReduceTestCase( - "|| ( foo? ( A B ) )", - uselist = ["foo"], - expected_result = ['A', 'B']), - - UseReduceTestCase( - "|| ( || ( foo? ( A B ) ) )", - uselist = ["foo"], - expected_result = ['A', 'B']), - - UseReduceTestCase( - "|| ( || ( || ( a? ( b? ( c? ( || ( || ( || ( d? ( e? ( f? ( A B ) ) ) ) ) ) ) ) ) ) ) )", - uselist = ["a", "b", "c", "d", "e", "f"], - expected_result = ['A', 'B']), - - UseReduceTestCase( - "|| ( || ( ( || ( a? ( ( b? ( c? ( || ( || ( || ( ( d? ( e? ( f? ( A B ) ) ) ) ) ) ) ) ) ) ) ) ) ) )", - uselist = ["a", "b", "c", "d", "e", "f"], - expected_result = ['A', 'B']), - - UseReduceTestCase( - "|| ( ( A ( || ( B ) ) ) )", - expected_result = ['A', 'B']), - - UseReduceTestCase( - "|| ( ( A B ) || ( foo? ( bar? ( ( C D || ( baz? ( E ) ( F G ) || ( H ) ) ) ) ) ) )", - uselist = ["foo", "bar", "baz"], - expected_result = ['||', [['A', 'B'], ['C', 'D', '||', ['E', ['F', 'G'], 'H']]]]), - - UseReduceTestCase( - "|| ( ( A B ) || ( foo? ( bar? ( ( C D || ( baz? ( E ) ( F G ) || ( H ) ) ) ) ) ) )", - uselist = ["foo", "bar", "baz"], - opconvert = True, - expected_result = [['||', ['A', 'B'], ['C', 'D', ['||', 'E', ['F', 'G'], 'H']]]]), - - UseReduceTestCase( - "|| ( foo? ( A B ) )", - uselist = ["foo"], - opconvert=True, - expected_result = ['A', 'B']), - - UseReduceTestCase( - "|| ( || ( foo? ( A B ) ) )", - uselist = ["foo"], - opconvert=True, - expected_result = ['A', 'B']), - - UseReduceTestCase( - "|| ( || ( || ( a? ( b? ( c? ( || ( || ( || ( d? ( e? ( f? ( A B ) ) ) ) ) ) ) ) ) ) ) )", - uselist = ["a", "b", "c", "d", "e", "f"], - opconvert=True, - expected_result = ['A', 'B']), - - #flat test - UseReduceTestCase( - "A", - flat = True, - expected_result = ["A"]), - UseReduceTestCase( - "( A )", - flat = True, - expected_result = ["A"]), - UseReduceTestCase( - "|| ( A B )", - flat = True, - expected_result = [ "||", "A", "B" ] ), - UseReduceTestCase( - "|| ( A || ( B C ) )", - flat = True, - expected_result = [ "||", "A", "||", "B", "C" ]), - UseReduceTestCase( - "|| ( A || ( B C D ) )", - flat = True, - expected_result = [ "||", "A", "||", "B", "C", "D" ]), - UseReduceTestCase( - "|| ( A || ( B || ( C D ) E ) )", - flat = True, - expected_result = [ "||", "A", "||", "B", "||", "C", "D", "E" ]), - UseReduceTestCase( - "( || ( ( ( A ) B ) ) )", - flat = True, - expected_result = [ "||", "A", "B"] ), - UseReduceTestCase( - "( || ( || ( ( A ) B ) ) )", - flat = True, - expected_result = [ "||", "||", "A", "B" ]), - UseReduceTestCase( - "( || ( || ( ( A ) B ) ) )", - flat = True, - expected_result = [ "||", "||", "A", "B" ]), - UseReduceTestCase( - "|| ( A )", - flat = True, - expected_result = ["||", "A"]), - UseReduceTestCase( - "( || ( || ( || ( A ) foo? ( B ) ) ) )", - expected_result = ["A"]), - UseReduceTestCase( - "( || ( || ( || ( A ) foo? ( B ) ) ) )", - uselist = ["foo"], - flat = True, - expected_result = [ "||", "||","||", "A", "B" ]), - UseReduceTestCase( - "( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )", - flat = True, - expected_result = ["||", "||","||"]), - UseReduceTestCase( - "( || ( || ( bar? ( A ) || ( foo? ( B ) ) ) ) )", - uselist = ["foo", "bar"], - flat = True, - expected_result = [ "||", "||", "A", "||", "B" ]), - UseReduceTestCase( - "A || ( bar? ( C ) ) foo? ( bar? ( C ) ) B", - flat = True, - expected_result = ["A", "||", "B"]), - UseReduceTestCase( - "|| ( A ) || ( B )", - flat = True, - expected_result = ["||", "A", "||", "B"]), - UseReduceTestCase( - "foo? ( A ) foo? ( B )", - flat = True, - expected_result = []), - UseReduceTestCase( - "foo? ( A ) foo? ( B )", - uselist = ["foo"], - flat = True, - expected_result = ["A", "B"]), - - #use flag validation - UseReduceTestCase( - "foo? ( A )", - uselist = ["foo"], - is_valid_flag = self.always_true, - expected_result = ["A"]), - UseReduceTestCase( - "foo? ( A )", - is_valid_flag = self.always_true, - expected_result = []), - - #token_class - UseReduceTestCase( - "foo? ( dev-libs/A )", - uselist = ["foo"], - token_class=Atom, - expected_result = ["dev-libs/A"]), - UseReduceTestCase( - "foo? ( dev-libs/A )", - token_class=Atom, - expected_result = []), - ) - - test_cases_xfail = ( - UseReduceTestCase("? ( A )"), - UseReduceTestCase("!? ( A )"), - UseReduceTestCase("( A"), - UseReduceTestCase("A )"), - UseReduceTestCase("||( A B )"), - UseReduceTestCase("|| (A B )"), - UseReduceTestCase("|| ( A B)"), - UseReduceTestCase("|| ( A B"), - UseReduceTestCase("|| A B )"), - UseReduceTestCase("|| A B"), - UseReduceTestCase("|| ( A B ) )"), - UseReduceTestCase("|| || B C"), - UseReduceTestCase("|| ( A B || )"), - UseReduceTestCase("a? A"), - UseReduceTestCase("( || ( || || ( A ) foo? ( B ) ) )"), - UseReduceTestCase("( || ( || bar? ( A ) foo? ( B ) ) )"), - UseReduceTestCase("foo?"), - UseReduceTestCase("foo? || ( A )"), - UseReduceTestCase("|| ( )"), - UseReduceTestCase("foo? ( )"), - - #SRC_URI stuff - UseReduceTestCase("http://foo/bar -> blah.tbz2", is_src_uri = True, eapi = EAPI_WITHOUT_SRC_URI_ARROWS), - UseReduceTestCase("|| ( http://foo/bar -> blah.tbz2 )", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - UseReduceTestCase("http://foo/bar -> foo? ( ftp://foo/a )", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - UseReduceTestCase("http://foo/bar blah.tbz2 ->", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - UseReduceTestCase("-> http://foo/bar blah.tbz2 )", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - UseReduceTestCase("http://foo/bar ->", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - UseReduceTestCase("http://foo/bar -> foo? ( http://foo.com/foo )", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - UseReduceTestCase("foo? ( http://foo/bar -> ) blah.tbz2", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - UseReduceTestCase("http://foo/bar -> foo/blah.tbz2", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - UseReduceTestCase("http://foo/bar -> -> bar.tbz2 foo? ( ftp://foo/a )", is_src_uri = True, eapi = EAPI_WITH_SRC_URI_ARROWS), - - UseReduceTestCase("http://foo/bar -> bar.tbz2 foo? ( ftp://foo/a )", is_src_uri = False, eapi = EAPI_WITH_SRC_URI_ARROWS), - - UseReduceTestCase( - "A", - opconvert = True, - flat = True), - - #use flag validation - UseReduceTestCase("1.0? ( A )"), - UseReduceTestCase("!1.0? ( A )"), - UseReduceTestCase("!? ( A )"), - UseReduceTestCase("!?? ( A )"), - UseReduceTestCase( - "foo? ( A )", - is_valid_flag = self.always_false, - ), - UseReduceTestCase( - "foo? ( A )", - uselist = ["foo"], - is_valid_flag = self.always_false, - ), - - #token_class - UseReduceTestCase( - "foo? ( A )", - uselist = ["foo"], - token_class=Atom), - UseReduceTestCase( - "A(B", - token_class=Atom), - ) - - for test_case in test_cases: - # If it fails then show the input, since lots of our - # test cases have the same output but different input, - # making it difficult deduce which test has failed. - self.assertEqual(test_case.run(), test_case.expected_result, - "input: '%s' result: %s != %s" % (test_case.deparray, - test_case.run(), test_case.expected_result)) - - for test_case in test_cases_xfail: - self.assertRaisesMsg(test_case.deparray, (InvalidDependString, ValueError), test_case.run) diff --git a/portage_with_autodep/pym/portage/tests/ebuild/__init__.py b/portage_with_autodep/pym/portage/tests/ebuild/__init__.py deleted file mode 100644 index e2d487e..0000000 --- a/portage_with_autodep/pym/portage/tests/ebuild/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 1998-2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/tests/ebuild/__test__ b/portage_with_autodep/pym/portage/tests/ebuild/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/ebuild/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/ebuild/test_array_fromfile_eof.py b/portage_with_autodep/pym/portage/tests/ebuild/test_array_fromfile_eof.py deleted file mode 100644 index d8277f2..0000000 --- a/portage_with_autodep/pym/portage/tests/ebuild/test_array_fromfile_eof.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2009 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import array -import tempfile - -from portage import _unicode_decode -from portage import _unicode_encode -from portage.tests import TestCase - -class ArrayFromfileEofTestCase(TestCase): - - def testArrayFromfileEof(self): - # This tests if the following python issue is fixed - # in the currently running version of python: - # http://bugs.python.org/issue5334 - - input_data = "an arbitrary string" - input_bytes = _unicode_encode(input_data, - encoding='utf_8', errors='strict') - f = tempfile.TemporaryFile() - f.write(input_bytes) - - f.seek(0) - data = [] - eof = False - while not eof: - a = array.array('B') - try: - a.fromfile(f, len(input_bytes) + 1) - except (EOFError, IOError): - # python-3.0 lost data here - eof = True - - if not a: - eof = True - else: - data.append(_unicode_decode(a.tostring(), - encoding='utf_8', errors='strict')) - - f.close() - - self.assertEqual(input_data, ''.join(data)) diff --git a/portage_with_autodep/pym/portage/tests/ebuild/test_config.py b/portage_with_autodep/pym/portage/tests/ebuild/test_config.py deleted file mode 100644 index 7bec8c6..0000000 --- a/portage_with_autodep/pym/portage/tests/ebuild/test_config.py +++ /dev/null @@ -1,198 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import portage -from portage import os -from portage.package.ebuild.config import config -from portage.package.ebuild._config.LicenseManager import LicenseManager -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class ConfigTestCase(TestCase): - - def testClone(self): - """ - Test the clone via constructor. - """ - - ebuilds = { - "dev-libs/A-1": { }, - } - - playground = ResolverPlayground(ebuilds=ebuilds) - try: - settings = config(clone=playground.settings) - result = playground.run(["=dev-libs/A-1"]) - pkg, existing_node = result.depgraph._select_package( - playground.root, "=dev-libs/A-1") - settings.setcpv(pkg) - - # clone after setcpv tests deepcopy of LazyItemsDict - settings2 = config(clone=settings) - finally: - playground.cleanup() - - def testFeaturesMutation(self): - """ - Test whether mutation of config.features updates the FEATURES - variable and persists through config.regenerate() calls. Also - verify that features_set._prune_overrides() works correctly. - """ - playground = ResolverPlayground() - try: - settings = config(clone=playground.settings) - - settings.features.add('noclean') - self.assertEqual('noclean' in settings['FEATURES'].split(), True) - settings.regenerate() - self.assertEqual('noclean' in settings['FEATURES'].split(),True) - - settings.features.discard('noclean') - self.assertEqual('noclean' in settings['FEATURES'].split(), False) - settings.regenerate() - self.assertEqual('noclean' in settings['FEATURES'].split(), False) - - settings.features.add('noclean') - self.assertEqual('noclean' in settings['FEATURES'].split(), True) - settings.regenerate() - self.assertEqual('noclean' in settings['FEATURES'].split(),True) - - # before: ['noclean', '-noclean', 'noclean'] - settings.features._prune_overrides() - # after: ['noclean'] - self.assertEqual(settings._features_overrides.count('noclean'), 1) - self.assertEqual(settings._features_overrides.count('-noclean'), 0) - - settings.features.remove('noclean') - - # before: ['noclean', '-noclean'] - settings.features._prune_overrides() - # after: ['-noclean'] - self.assertEqual(settings._features_overrides.count('noclean'), 0) - self.assertEqual(settings._features_overrides.count('-noclean'), 1) - finally: - playground.cleanup() - - def testLicenseManager(self): - - user_config = { - "package.license": - ( - "dev-libs/* TEST", - "dev-libs/A -TEST2", - "=dev-libs/A-2 TEST3 @TEST", - "*/* @EULA TEST2", - "=dev-libs/C-1 *", - "=dev-libs/C-2 -*", - ), - } - - playground = ResolverPlayground(user_config=user_config) - try: - portage.util.noiselimit = -2 - - license_group_locations = (os.path.join(playground.portdir, "profiles"),) - pkg_license = os.path.join(playground.eroot, "etc", "portage") - - lic_man = LicenseManager(license_group_locations, pkg_license) - - self.assertEqual(lic_man._accept_license_str, None) - self.assertEqual(lic_man._accept_license, None) - self.assertEqual(lic_man._license_groups, {"EULA": frozenset(["TEST"])}) - self.assertEqual(lic_man._undef_lic_groups, set(["TEST"])) - - self.assertEqual(lic_man.extract_global_changes(), "TEST TEST2") - self.assertEqual(lic_man.extract_global_changes(), "") - - lic_man.set_accept_license_str("TEST TEST2") - self.assertEqual(lic_man._getPkgAcceptLicense("dev-libs/B-1", "0", None), ["TEST", "TEST2", "TEST"]) - self.assertEqual(lic_man._getPkgAcceptLicense("dev-libs/A-1", "0", None), ["TEST", "TEST2", "TEST", "-TEST2"]) - self.assertEqual(lic_man._getPkgAcceptLicense("dev-libs/A-2", "0", None), ["TEST", "TEST2", "TEST", "-TEST2", "TEST3", "@TEST"]) - - self.assertEqual(lic_man.get_prunned_accept_license("dev-libs/B-1", [], "TEST", "0", None), "TEST") - self.assertEqual(lic_man.get_prunned_accept_license("dev-libs/A-1", [], "-TEST2", "0", None), "") - self.assertEqual(lic_man.get_prunned_accept_license("dev-libs/A-2", [], "|| ( TEST TEST2 )", "0", None), "TEST") - self.assertEqual(lic_man.get_prunned_accept_license("dev-libs/C-1", [], "TEST5", "0", None), "TEST5") - self.assertEqual(lic_man.get_prunned_accept_license("dev-libs/C-2", [], "TEST2", "0", None), "") - - self.assertEqual(lic_man.getMissingLicenses("dev-libs/B-1", [], "TEST", "0", None), []) - self.assertEqual(lic_man.getMissingLicenses("dev-libs/A-1", [], "-TEST2", "0", None), ["-TEST2"]) - self.assertEqual(lic_man.getMissingLicenses("dev-libs/A-2", [], "|| ( TEST TEST2 )", "0", None), []) - self.assertEqual(lic_man.getMissingLicenses("dev-libs/A-3", [], "|| ( TEST2 || ( TEST3 TEST4 ) )", "0", None), ["TEST2", "TEST3", "TEST4"]) - self.assertEqual(lic_man.getMissingLicenses("dev-libs/C-1", [], "TEST5", "0", None), []) - self.assertEqual(lic_man.getMissingLicenses("dev-libs/C-2", [], "TEST2", "0", None), ["TEST2"]) - self.assertEqual(lic_man.getMissingLicenses("dev-libs/D-1", [], "", "0", None), []) - finally: - portage.util.noiselimit = 0 - playground.cleanup() - - def testPackageMaskOrder(self): - - ebuilds = { - "dev-libs/A-1": { }, - "dev-libs/B-1": { }, - "dev-libs/C-1": { }, - "dev-libs/D-1": { }, - "dev-libs/E-1": { }, - } - - repo_configs = { - "test_repo": { - "package.mask": - ( - "dev-libs/A", - "dev-libs/C", - ), - } - } - - profile = { - "package.mask": - ( - "-dev-libs/A", - "dev-libs/B", - "-dev-libs/B", - "dev-libs/D", - ), - } - - user_config = { - "package.mask": - ( - "-dev-libs/C", - "-dev-libs/D", - "dev-libs/E", - ), - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["dev-libs/A"], - options = { "--autounmask": 'n' }, - success = False), - ResolverPlaygroundTestCase( - ["dev-libs/B"], - success = True, - mergelist = ["dev-libs/B-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/C"], - success = True, - mergelist = ["dev-libs/C-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/D"], - success = True, - mergelist = ["dev-libs/D-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/E"], - options = { "--autounmask": 'n' }, - success = False), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, repo_configs=repo_configs, \ - profile=profile, user_config=user_config) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/ebuild/test_doebuild_spawn.py b/portage_with_autodep/pym/portage/tests/ebuild/test_doebuild_spawn.py deleted file mode 100644 index ed08b2a..0000000 --- a/portage_with_autodep/pym/portage/tests/ebuild/test_doebuild_spawn.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage import _python_interpreter -from portage import _shell_quote -from portage.const import EBUILD_SH_BINARY -from portage.package.ebuild.config import config -from portage.package.ebuild.doebuild import spawn as doebuild_spawn -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground -from _emerge.EbuildPhase import EbuildPhase -from _emerge.MiscFunctionsProcess import MiscFunctionsProcess -from _emerge.Package import Package -from _emerge.PollScheduler import PollScheduler - -class DoebuildSpawnTestCase(TestCase): - """ - Invoke portage.package.ebuild.doebuild.spawn() with a - minimal environment. This gives coverage to some of - the ebuild execution internals, like ebuild.sh, - AbstractEbuildProcess, and EbuildIpcDaemon. - """ - - def testDoebuildSpawn(self): - playground = ResolverPlayground() - try: - settings = config(clone=playground.settings) - cpv = 'sys-apps/portage-2.1' - metadata = { - 'EAPI' : '2', - 'INHERITED' : 'python eutils', - 'IUSE' : 'build doc epydoc python3 selinux', - 'LICENSE' : 'GPL-2', - 'PROVIDE' : 'virtual/portage', - 'RDEPEND' : '>=app-shells/bash-3.2_p17 >=dev-lang/python-2.6', - 'SLOT' : '0', - } - root_config = playground.trees[playground.root]['root_config'] - pkg = Package(built=False, cpv=cpv, installed=False, - metadata=metadata, root_config=root_config, - type_name='ebuild') - settings.setcpv(pkg) - settings['PORTAGE_PYTHON'] = _python_interpreter - settings['PORTAGE_BUILDDIR'] = os.path.join( - settings['PORTAGE_TMPDIR'], cpv) - settings['T'] = os.path.join( - settings['PORTAGE_BUILDDIR'], 'temp') - for x in ('PORTAGE_BUILDDIR', 'T'): - os.makedirs(settings[x]) - # Create a fake environment, to pretend as if the ebuild - # has been sourced already. - open(os.path.join(settings['T'], 'environment'), 'wb') - - scheduler = PollScheduler().sched_iface - for phase in ('_internal_test',): - - # Test EbuildSpawnProcess by calling doebuild.spawn() with - # returnpid=False. This case is no longer used by portage - # internals since EbuildPhase is used instead and that passes - # returnpid=True to doebuild.spawn(). - rval = doebuild_spawn("%s %s" % (_shell_quote( - os.path.join(settings["PORTAGE_BIN_PATH"], - os.path.basename(EBUILD_SH_BINARY))), phase), - settings, free=1) - self.assertEqual(rval, os.EX_OK) - - ebuild_phase = EbuildPhase(background=False, - phase=phase, scheduler=scheduler, - settings=settings) - ebuild_phase.start() - ebuild_phase.wait() - self.assertEqual(ebuild_phase.returncode, os.EX_OK) - - ebuild_phase = MiscFunctionsProcess(background=False, - commands=['success_hooks'], - scheduler=scheduler, settings=settings) - ebuild_phase.start() - ebuild_phase.wait() - self.assertEqual(ebuild_phase.returncode, os.EX_OK) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/ebuild/test_ipc_daemon.py b/portage_with_autodep/pym/portage/tests/ebuild/test_ipc_daemon.py deleted file mode 100644 index b5b4796..0000000 --- a/portage_with_autodep/pym/portage/tests/ebuild/test_ipc_daemon.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import shutil -import tempfile -import time -from portage import os -from portage import _python_interpreter -from portage.tests import TestCase -from portage.const import PORTAGE_BIN_PATH -from portage.const import PORTAGE_PYM_PATH -from portage.const import BASH_BINARY -from portage.package.ebuild._ipc.ExitCommand import ExitCommand -from portage.util import ensure_dirs -from _emerge.SpawnProcess import SpawnProcess -from _emerge.EbuildBuildDir import EbuildBuildDir -from _emerge.EbuildIpcDaemon import EbuildIpcDaemon -from _emerge.TaskScheduler import TaskScheduler - -class IpcDaemonTestCase(TestCase): - - _SCHEDULE_TIMEOUT = 40000 # 40 seconds - - def testIpcDaemon(self): - tmpdir = tempfile.mkdtemp() - build_dir = None - try: - env = {} - - # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they - # need to be inherited by ebuild subprocesses. - if 'PORTAGE_USERNAME' in os.environ: - env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] - if 'PORTAGE_GRPNAME' in os.environ: - env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] - - env['PORTAGE_PYTHON'] = _python_interpreter - env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH - env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH - env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') - - task_scheduler = TaskScheduler(max_jobs=2) - build_dir = EbuildBuildDir( - scheduler=task_scheduler.sched_iface, - settings=env) - build_dir.lock() - ensure_dirs(env['PORTAGE_BUILDDIR']) - - input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') - output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') - os.mkfifo(input_fifo) - os.mkfifo(output_fifo) - - for exitcode in (0, 1, 2): - exit_command = ExitCommand() - commands = {'exit' : exit_command} - daemon = EbuildIpcDaemon(commands=commands, - input_fifo=input_fifo, - output_fifo=output_fifo, - scheduler=task_scheduler.sched_iface) - proc = SpawnProcess( - args=[BASH_BINARY, "-c", - '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode], - env=env, scheduler=task_scheduler.sched_iface) - - self.received_command = False - def exit_command_callback(): - self.received_command = True - proc.cancel() - daemon.cancel() - - exit_command.reply_hook = exit_command_callback - task_scheduler.add(daemon) - task_scheduler.add(proc) - start_time = time.time() - task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT) - task_scheduler.clear() - - self.assertEqual(self.received_command, True, - "command not received after %d seconds" % \ - (time.time() - start_time,)) - self.assertEqual(proc.isAlive(), False) - self.assertEqual(daemon.isAlive(), False) - self.assertEqual(exit_command.exitcode, exitcode) - - # Intentionally short timeout test for QueueScheduler.run() - sleep_time_s = 10 # 10.000 seconds - short_timeout_ms = 10 # 0.010 seconds - - for i in range(3): - exit_command = ExitCommand() - commands = {'exit' : exit_command} - daemon = EbuildIpcDaemon(commands=commands, - input_fifo=input_fifo, - output_fifo=output_fifo, - scheduler=task_scheduler.sched_iface) - proc = SpawnProcess( - args=[BASH_BINARY, "-c", 'exec sleep %d' % sleep_time_s], - env=env, scheduler=task_scheduler.sched_iface) - - self.received_command = False - def exit_command_callback(): - self.received_command = True - proc.cancel() - daemon.cancel() - - exit_command.reply_hook = exit_command_callback - task_scheduler.add(daemon) - task_scheduler.add(proc) - start_time = time.time() - task_scheduler.run(timeout=short_timeout_ms) - task_scheduler.clear() - - self.assertEqual(self.received_command, False, - "command received after %d seconds" % \ - (time.time() - start_time,)) - self.assertEqual(proc.isAlive(), False) - self.assertEqual(daemon.isAlive(), False) - self.assertEqual(proc.returncode == os.EX_OK, False) - - finally: - if build_dir is not None: - build_dir.unlock() - shutil.rmtree(tmpdir) diff --git a/portage_with_autodep/pym/portage/tests/ebuild/test_pty_eof.py b/portage_with_autodep/pym/portage/tests/ebuild/test_pty_eof.py deleted file mode 100644 index 4b6ff21..0000000 --- a/portage_with_autodep/pym/portage/tests/ebuild/test_pty_eof.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2009-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.util._pty import _can_test_pty_eof, _test_pty_eof - -class PtyEofFdopenBufferedTestCase(TestCase): - - def testPtyEofFdopenBuffered(self): - # This tests if the following python issue is fixed yet: - # http://bugs.python.org/issue5380 - # Since it might not be fixed, mark as todo. - self.todo = True - # The result is only valid if openpty does not raise EnvironmentError. - if _can_test_pty_eof(): - try: - self.assertEqual(_test_pty_eof(fdopen_buffered=True), True) - except EnvironmentError: - pass - -class PtyEofFdopenUnBufferedTestCase(TestCase): - def testPtyEofFdopenUnBuffered(self): - # New development: It appears that array.fromfile() is usable - # with python3 as long as fdopen is called with a bufsize - # argument of 0. - - # The result is only valid if openpty does not raise EnvironmentError. - if _can_test_pty_eof(): - try: - self.assertEqual(_test_pty_eof(), True) - except EnvironmentError: - pass diff --git a/portage_with_autodep/pym/portage/tests/ebuild/test_spawn.py b/portage_with_autodep/pym/portage/tests/ebuild/test_spawn.py deleted file mode 100644 index fea4738..0000000 --- a/portage_with_autodep/pym/portage/tests/ebuild/test_spawn.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 1998-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import errno -import io -import sys -import tempfile -from portage import os -from portage import _encodings -from portage import _unicode_encode -from portage.const import BASH_BINARY -from portage.tests import TestCase -from _emerge.SpawnProcess import SpawnProcess -from _emerge.PollScheduler import PollScheduler - -class SpawnTestCase(TestCase): - - def testLogfile(self): - logfile = None - try: - fd, logfile = tempfile.mkstemp() - os.close(fd) - null_fd = os.open('/dev/null', os.O_RDWR) - test_string = 2 * "blah blah blah\n" - scheduler = PollScheduler().sched_iface - proc = SpawnProcess( - args=[BASH_BINARY, "-c", - "echo -n '%s'" % test_string], - env={}, fd_pipes={0:sys.stdin.fileno(), 1:null_fd, 2:null_fd}, - scheduler=scheduler, - logfile=logfile) - proc.start() - os.close(null_fd) - self.assertEqual(proc.wait(), os.EX_OK) - f = io.open(_unicode_encode(logfile, - encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['content'], errors='strict') - log_content = f.read() - f.close() - # When logging passes through a pty, this comparison will fail - # unless the oflag terminal attributes have the termios.OPOST - # bit disabled. Otherwise, tranformations such as \n -> \r\n - # may occur. - self.assertEqual(test_string, log_content) - finally: - if logfile: - try: - os.unlink(logfile) - except EnvironmentError as e: - if e.errno != errno.ENOENT: - raise - del e diff --git a/portage_with_autodep/pym/portage/tests/env/__init__.py b/portage_with_autodep/pym/portage/tests/env/__init__.py deleted file mode 100644 index cbeabe5..0000000 --- a/portage_with_autodep/pym/portage/tests/env/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# tests/portage/env/__init__.py -- Portage Unit Test functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - diff --git a/portage_with_autodep/pym/portage/tests/env/__test__ b/portage_with_autodep/pym/portage/tests/env/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/env/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/env/config/__init__.py b/portage_with_autodep/pym/portage/tests/env/config/__init__.py deleted file mode 100644 index ef5cc43..0000000 --- a/portage_with_autodep/pym/portage/tests/env/config/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# tests/portage/env/config/__init__.py -- Portage Unit Test functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - diff --git a/portage_with_autodep/pym/portage/tests/env/config/__test__ b/portage_with_autodep/pym/portage/tests/env/config/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/env/config/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/env/config/test_PackageKeywordsFile.py b/portage_with_autodep/pym/portage/tests/env/config/test_PackageKeywordsFile.py deleted file mode 100644 index f1e9e98..0000000 --- a/portage_with_autodep/pym/portage/tests/env/config/test_PackageKeywordsFile.py +++ /dev/null @@ -1,40 +0,0 @@ -# test_PackageKeywordsFile.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage.tests import TestCase -from portage.env.config import PackageKeywordsFile -from tempfile import mkstemp - -class PackageKeywordsFileTestCase(TestCase): - - cpv = ['sys-apps/portage'] - keywords = ['~x86', 'amd64', '-mips'] - - def testPackageKeywordsFile(self): - """ - A simple test to ensure the load works properly - """ - - self.BuildFile() - try: - f = PackageKeywordsFile(self.fname) - f.load() - i = 0 - for cpv, keyword in f.items(): - self.assertEqual( cpv, self.cpv[i] ) - [k for k in keyword if self.assertTrue(k in self.keywords)] - i = i + 1 - finally: - self.NukeFile() - - def BuildFile(self): - fd, self.fname = mkstemp() - f = os.fdopen(fd, 'w') - for c in self.cpv: - f.write("%s %s\n" % (c,' '.join(self.keywords))) - f.close() - - def NukeFile(self): - os.unlink(self.fname) diff --git a/portage_with_autodep/pym/portage/tests/env/config/test_PackageMaskFile.py b/portage_with_autodep/pym/portage/tests/env/config/test_PackageMaskFile.py deleted file mode 100644 index 0c5b30f..0000000 --- a/portage_with_autodep/pym/portage/tests/env/config/test_PackageMaskFile.py +++ /dev/null @@ -1,29 +0,0 @@ -# test_PackageMaskFile.py -- Portage Unit Testing Functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage.env.config import PackageMaskFile -from portage.tests import TestCase, test_cps -from tempfile import mkstemp - -class PackageMaskFileTestCase(TestCase): - - def testPackageMaskFile(self): - self.BuildFile() - try: - f = PackageMaskFile(self.fname) - f.load() - for atom in f: - self.assertTrue(atom in test_cps) - finally: - self.NukeFile() - - def BuildFile(self): - fd, self.fname = mkstemp() - f = os.fdopen(fd, 'w') - f.write("\n".join(test_cps)) - f.close() - - def NukeFile(self): - os.unlink(self.fname) diff --git a/portage_with_autodep/pym/portage/tests/env/config/test_PackageUseFile.py b/portage_with_autodep/pym/portage/tests/env/config/test_PackageUseFile.py deleted file mode 100644 index 7a38067..0000000 --- a/portage_with_autodep/pym/portage/tests/env/config/test_PackageUseFile.py +++ /dev/null @@ -1,37 +0,0 @@ -# test_PackageUseFile.py -- Portage Unit Testing Functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage.tests import TestCase -from portage.env.config import PackageUseFile -from tempfile import mkstemp - - -class PackageUseFileTestCase(TestCase): - - cpv = 'sys-apps/portage' - useflags = ['cdrom', 'far', 'boo', 'flag', 'blat'] - - def testPackageUseFile(self): - """ - A simple test to ensure the load works properly - """ - self.BuildFile() - try: - f = PackageUseFile(self.fname) - f.load() - for cpv, use in f.items(): - self.assertEqual( cpv, self.cpv ) - [flag for flag in use if self.assertTrue(flag in self.useflags)] - finally: - self.NukeFile() - - def BuildFile(self): - fd, self.fname = mkstemp() - f = os.fdopen(fd, 'w') - f.write("%s %s" % (self.cpv, ' '.join(self.useflags))) - f.close() - - def NukeFile(self): - os.unlink(self.fname) diff --git a/portage_with_autodep/pym/portage/tests/env/config/test_PortageModulesFile.py b/portage_with_autodep/pym/portage/tests/env/config/test_PortageModulesFile.py deleted file mode 100644 index 2cd1a8a..0000000 --- a/portage_with_autodep/pym/portage/tests/env/config/test_PortageModulesFile.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2006-2009 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage.tests import TestCase -from portage.env.config import PortageModulesFile -from tempfile import mkstemp - -class PortageModulesFileTestCase(TestCase): - - keys = ['foo.bar','baz','bob','extra_key'] - invalid_keys = ['',""] - modules = ['spanky','zmedico','antarus','ricer','5','6'] - - def setUp(self): - self.items = {} - for k, v in zip(self.keys + self.invalid_keys, - self.modules): - self.items[k] = v - - def testPortageModulesFile(self): - self.BuildFile() - f = PortageModulesFile(self.fname) - f.load() - for k in self.keys: - self.assertEqual(f[k], self.items[k]) - for ik in self.invalid_keys: - self.assertEqual(False, ik in f) - self.NukeFile() - - def BuildFile(self): - fd, self.fname = mkstemp() - f = os.fdopen(fd, 'w') - for k, v in self.items.items(): - f.write('%s=%s\n' % (k,v)) - f.close() - - def NukeFile(self): - os.unlink(self.fname) diff --git a/portage_with_autodep/pym/portage/tests/lafilefixer/__init__.py b/portage_with_autodep/pym/portage/tests/lafilefixer/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/lafilefixer/__init__.py +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/lafilefixer/__test__ b/portage_with_autodep/pym/portage/tests/lafilefixer/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/lafilefixer/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/lafilefixer/test_lafilefixer.py b/portage_with_autodep/pym/portage/tests/lafilefixer/test_lafilefixer.py deleted file mode 100644 index 0bcffaa..0000000 --- a/portage_with_autodep/pym/portage/tests/lafilefixer/test_lafilefixer.py +++ /dev/null @@ -1,145 +0,0 @@ -# test_lafilefixer.py -- Portage Unit Testing Functionality -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.exception import InvalidData - -class test_lafilefixer(TestCase): - - def get_test_cases_clean(self): - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' -lm'\n" + \ - b"current=6\n" + \ - b"age=0\n" + \ - b"revision=2\n" + \ - b"installed=yes\n" + \ - b"dlopen=''\n" + \ - b"dlpreopen=''\n" + \ - b"libdir='/usr/lib64'\n" - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' -lm'\n" + \ - b"current=6\n" + \ - b"age=0\n" + \ - b"revision=2\n" + \ - b"installed=yes\n" + \ - b"dlopen=''\n" + \ - b"dlpreopen=''\n" + \ - b"libdir='/usr/lib64'\n" - yield b"dependency_libs=' liba.la /usr/lib64/bar.la -lc'\n" - - def get_test_cases_update(self): - #.la -> -l* - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc'\n", \ - b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' -L/usr/lib64 -la -lb -lc'\n" - #move stuff into inherited_linker_flags - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' /usr/lib64/liba.la -pthread /usr/lib64/libb.la -lc'\n" + \ - b"inherited_linker_flags=''\n", \ - b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' -L/usr/lib64 -la -lb -lc'\n" + \ - b"inherited_linker_flags=' -pthread'\n" - #reorder - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' /usr/lib64/liba.la -R/usr/lib64 /usr/lib64/libb.la -lc'\n", \ - b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' -R/usr/lib64 -L/usr/lib64 -la -lb -lc'\n" - #remove duplicates from dependency_libs (the original version didn't do it for inherited_linker_flags) - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' /usr/lib64/liba.la /usr/lib64/libc.la -pthread -mt" + \ - b" -L/usr/lib -R/usr/lib64 -lc /usr/lib64/libb.la -lc'\n" +\ - b"inherited_linker_flags=' -pthread -pthread'\n", \ - b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' -R/usr/lib64 -L/usr/lib64 -L/usr/lib -la -lc -lb'\n" +\ - b"inherited_linker_flags=' -pthread -pthread -mt'\n" - #-L rewriting - yield b"dependency_libs=' -L/usr/X11R6/lib'\n", \ - b"dependency_libs=' -L/usr/lib'\n" - yield b"dependency_libs=' -L/usr/local/lib'\n", \ - b"dependency_libs=' -L/usr/lib'\n" - yield b"dependency_libs=' -L/usr/lib64/pkgconfig/../..'\n", \ - b"dependency_libs=' -L/usr'\n" - yield b"dependency_libs=' -L/usr/lib/pkgconfig/..'\n", \ - b"dependency_libs=' -L/usr/lib'\n" - yield b"dependency_libs=' -L/usr/lib/pkgconfig/../.. -L/usr/lib/pkgconfig/..'\n", \ - b"dependency_libs=' -L/usr -L/usr/lib'\n" - #we once got a backtrace on this one - yield b"dependency_libs=' /usr/lib64/libMagickCore.la -L/usr/lib64 -llcms2 /usr/lib64/libtiff.la " + \ - b"-ljbig -lc /usr/lib64/libfreetype.la /usr/lib64/libjpeg.la /usr/lib64/libXext.la " + \ - b"/usr/lib64/libXt.la /usr/lib64/libSM.la -lICE -luuid /usr/lib64/libICE.la /usr/lib64/libX11.la " + \ - b"/usr/lib64/libxcb.la /usr/lib64/libXau.la /usr/lib64/libXdmcp.la -lbz2 -lz -lm " + \ - b"/usr/lib/gcc/x86_64-pc-linux-gnu/4.4.4/libgomp.la -lrt -lpthread /usr/lib64/libltdl.la -ldl " + \ - b"/usr/lib64/libfpx.la -lstdc++'", \ - b"dependency_libs=' -L/usr/lib64 -L/usr/lib/gcc/x86_64-pc-linux-gnu/4.4.4 -lMagickCore -llcms2 " + \ - b"-ltiff -ljbig -lc -lfreetype -ljpeg -lXext -lXt -lSM -lICE -luuid -lX11 -lxcb -lXau -lXdmcp " + \ - b"-lbz2 -lz -lm -lgomp -lrt -lpthread -lltdl -ldl -lfpx -lstdc++'" - - - def get_test_cases_broken(self): - yield b"" - #no dependency_libs - yield b"dlname='libfoo.so.1'\n" + \ - b"current=6\n" + \ - b"age=0\n" + \ - b"revision=2\n" - #borken dependency_libs - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc' \n" - #borken dependency_libs - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc\n" - #crap in dependency_libs - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc /-lstdc++'\n" - #dependency_libs twice - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc /-lstdc++'\n" +\ - b"dependency_libs=' /usr/lib64/liba.la /usr/lib64/libb.la -lc /-lstdc++'\n" - #inherited_linker_flags twice - yield b"dlname='libfoo.so.1'\n" + \ - b"library_names='libfoo.so.1.0.2 libfoo.so.1 libfoo.so'\n" + \ - b"old_library='libpdf.a'\n" + \ - b"inherited_linker_flags=''\n" +\ - b"inherited_linker_flags=''\n" - - def testlafilefixer(self): - from portage.util.lafilefixer import _parse_lafile_contents, rewrite_lafile - - for clean_contents in self.get_test_cases_clean(): - self.assertEqual(rewrite_lafile(clean_contents), (False, None)) - - for original_contents, fixed_contents in self.get_test_cases_update(): - self.assertEqual(rewrite_lafile(original_contents), (True, fixed_contents)) - - for broken_contents in self.get_test_cases_broken(): - self.assertRaises(InvalidData, rewrite_lafile, broken_contents) diff --git a/portage_with_autodep/pym/portage/tests/lazyimport/__init__.py b/portage_with_autodep/pym/portage/tests/lazyimport/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/lazyimport/__init__.py +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/lazyimport/__test__ b/portage_with_autodep/pym/portage/tests/lazyimport/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/lazyimport/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/lazyimport/test_lazy_import_portage_baseline.py b/portage_with_autodep/pym/portage/tests/lazyimport/test_lazy_import_portage_baseline.py deleted file mode 100644 index 08ccfa7..0000000 --- a/portage_with_autodep/pym/portage/tests/lazyimport/test_lazy_import_portage_baseline.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import re -import portage -from portage import os -from portage.const import PORTAGE_PYM_PATH -from portage.tests import TestCase - -from _emerge.PollScheduler import PollScheduler -from _emerge.PipeReader import PipeReader -from _emerge.SpawnProcess import SpawnProcess - -class LazyImportPortageBaselineTestCase(TestCase): - - _module_re = re.compile(r'^(portage|repoman|_emerge)\.') - - _baseline_imports = frozenset([ - 'portage.const', 'portage.localization', - 'portage.proxy', 'portage.proxy.lazyimport', - 'portage.proxy.objectproxy', - 'portage._selinux', - ]) - - _baseline_import_cmd = [portage._python_interpreter, '-c', ''' -import os -import sys -sys.path.insert(0, os.environ["PORTAGE_PYM_PATH"]) -import portage -sys.stdout.write(" ".join(k for k in sys.modules - if sys.modules[k] is not None)) -'''] - - def testLazyImportPortageBaseline(self): - """ - Check what modules are imported by a baseline module import. - """ - - env = os.environ.copy() - pythonpath = env.get('PYTHONPATH') - if pythonpath is not None and not pythonpath.strip(): - pythonpath = None - if pythonpath is None: - pythonpath = '' - else: - pythonpath = ':' + pythonpath - pythonpath = PORTAGE_PYM_PATH + pythonpath - env[pythonpath] = pythonpath - - # If python is patched to insert the path of the - # currently installed portage module into sys.path, - # then the above PYTHONPATH override doesn't help. - env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH - - scheduler = PollScheduler().sched_iface - master_fd, slave_fd = os.pipe() - master_file = os.fdopen(master_fd, 'rb', 0) - slave_file = os.fdopen(slave_fd, 'wb') - producer = SpawnProcess( - args=self._baseline_import_cmd, - env=env, fd_pipes={1:slave_fd}, - scheduler=scheduler) - producer.start() - slave_file.close() - - consumer = PipeReader( - input_files={"producer" : master_file}, - scheduler=scheduler) - - consumer.start() - consumer.wait() - self.assertEqual(producer.wait(), os.EX_OK) - self.assertEqual(consumer.wait(), os.EX_OK) - - output = consumer.getvalue().decode('ascii', 'replace').split() - - unexpected_modules = " ".join(sorted(x for x in output \ - if self._module_re.match(x) is not None and \ - x not in self._baseline_imports)) - - self.assertEqual("", unexpected_modules) diff --git a/portage_with_autodep/pym/portage/tests/lazyimport/test_preload_portage_submodules.py b/portage_with_autodep/pym/portage/tests/lazyimport/test_preload_portage_submodules.py deleted file mode 100644 index 9d20eba..0000000 --- a/portage_with_autodep/pym/portage/tests/lazyimport/test_preload_portage_submodules.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import portage -from portage.tests import TestCase - -class PreloadPortageSubmodulesTestCase(TestCase): - - def testPreloadPortageSubmodules(self): - """ - Verify that _preload_portage_submodules() doesn't leave any - remaining proxies that refer to the portage.* namespace. - """ - portage.proxy.lazyimport._preload_portage_submodules() - for name in portage.proxy.lazyimport._module_proxies: - self.assertEqual(name.startswith('portage.'), False) diff --git a/portage_with_autodep/pym/portage/tests/lint/__init__.pyo b/portage_with_autodep/pym/portage/tests/lint/__init__.pyo Binary files differnew file mode 100644 index 0000000..a1241e5 --- /dev/null +++ b/portage_with_autodep/pym/portage/tests/lint/__init__.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo Binary files differnew file mode 100644 index 0000000..a7ddc80 --- /dev/null +++ b/portage_with_autodep/pym/portage/tests/lint/test_bash_syntax.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo Binary files differnew file mode 100644 index 0000000..7b1460d --- /dev/null +++ b/portage_with_autodep/pym/portage/tests/lint/test_compile_modules.pyo diff --git a/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo Binary files differnew file mode 100644 index 0000000..b3a1d61 --- /dev/null +++ b/portage_with_autodep/pym/portage/tests/lint/test_import_modules.pyo diff --git a/portage_with_autodep/pym/portage/tests/locks/__test__ b/portage_with_autodep/pym/portage/tests/locks/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/locks/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/locks/test_asynchronous_lock.py b/portage_with_autodep/pym/portage/tests/locks/test_asynchronous_lock.py deleted file mode 100644 index 8946caf..0000000 --- a/portage_with_autodep/pym/portage/tests/locks/test_asynchronous_lock.py +++ /dev/null @@ -1,124 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import shutil -import signal -import tempfile - -from portage import os -from portage.tests import TestCase -from _emerge.AsynchronousLock import AsynchronousLock -from _emerge.PollScheduler import PollScheduler - -class AsynchronousLockTestCase(TestCase): - - def testAsynchronousLock(self): - scheduler = PollScheduler().sched_iface - tempdir = tempfile.mkdtemp() - try: - path = os.path.join(tempdir, 'lock_me') - for force_async in (True, False): - for force_dummy in (True, False): - async_lock = AsynchronousLock(path=path, - scheduler=scheduler, _force_async=force_async, - _force_thread=True, - _force_dummy=force_dummy) - async_lock.start() - self.assertEqual(async_lock.wait(), os.EX_OK) - self.assertEqual(async_lock.returncode, os.EX_OK) - async_lock.unlock() - - async_lock = AsynchronousLock(path=path, - scheduler=scheduler, _force_async=force_async, - _force_process=True) - async_lock.start() - self.assertEqual(async_lock.wait(), os.EX_OK) - self.assertEqual(async_lock.returncode, os.EX_OK) - async_lock.unlock() - - finally: - shutil.rmtree(tempdir) - - def testAsynchronousLockWait(self): - scheduler = PollScheduler().sched_iface - tempdir = tempfile.mkdtemp() - try: - path = os.path.join(tempdir, 'lock_me') - lock1 = AsynchronousLock(path=path, scheduler=scheduler) - lock1.start() - self.assertEqual(lock1.wait(), os.EX_OK) - self.assertEqual(lock1.returncode, os.EX_OK) - - # lock2 requires _force_async=True since the portage.locks - # module is not designed to work as intended here if the - # same process tries to lock the same file more than - # one time concurrently. - lock2 = AsynchronousLock(path=path, scheduler=scheduler, - _force_async=True, _force_process=True) - lock2.start() - # lock2 should be waiting for lock1 to release - self.assertEqual(lock2.poll(), None) - self.assertEqual(lock2.returncode, None) - - lock1.unlock() - self.assertEqual(lock2.wait(), os.EX_OK) - self.assertEqual(lock2.returncode, os.EX_OK) - lock2.unlock() - finally: - shutil.rmtree(tempdir) - - def testAsynchronousLockWaitCancel(self): - scheduler = PollScheduler().sched_iface - tempdir = tempfile.mkdtemp() - try: - path = os.path.join(tempdir, 'lock_me') - lock1 = AsynchronousLock(path=path, scheduler=scheduler) - lock1.start() - self.assertEqual(lock1.wait(), os.EX_OK) - self.assertEqual(lock1.returncode, os.EX_OK) - lock2 = AsynchronousLock(path=path, scheduler=scheduler, - _force_async=True, _force_process=True) - lock2.start() - # lock2 should be waiting for lock1 to release - self.assertEqual(lock2.poll(), None) - self.assertEqual(lock2.returncode, None) - - # Cancel lock2 and then check wait() and returncode results. - lock2.cancel() - self.assertEqual(lock2.wait() == os.EX_OK, False) - self.assertEqual(lock2.returncode == os.EX_OK, False) - self.assertEqual(lock2.returncode is None, False) - lock1.unlock() - finally: - shutil.rmtree(tempdir) - - def testAsynchronousLockWaitKill(self): - scheduler = PollScheduler().sched_iface - tempdir = tempfile.mkdtemp() - try: - path = os.path.join(tempdir, 'lock_me') - lock1 = AsynchronousLock(path=path, scheduler=scheduler) - lock1.start() - self.assertEqual(lock1.wait(), os.EX_OK) - self.assertEqual(lock1.returncode, os.EX_OK) - lock2 = AsynchronousLock(path=path, scheduler=scheduler, - _force_async=True, _force_process=True) - lock2.start() - # lock2 should be waiting for lock1 to release - self.assertEqual(lock2.poll(), None) - self.assertEqual(lock2.returncode, None) - - # Kill lock2's process and then check wait() and - # returncode results. This is intended to simulate - # a SIGINT sent via the controlling tty. - self.assertEqual(lock2._imp is not None, True) - self.assertEqual(lock2._imp._proc is not None, True) - self.assertEqual(lock2._imp._proc.pid is not None, True) - lock2._imp._kill_test = True - os.kill(lock2._imp._proc.pid, signal.SIGTERM) - self.assertEqual(lock2.wait() == os.EX_OK, False) - self.assertEqual(lock2.returncode == os.EX_OK, False) - self.assertEqual(lock2.returncode is None, False) - lock1.unlock() - finally: - shutil.rmtree(tempdir) diff --git a/portage_with_autodep/pym/portage/tests/locks/test_lock_nonblock.py b/portage_with_autodep/pym/portage/tests/locks/test_lock_nonblock.py deleted file mode 100644 index d5748ad..0000000 --- a/portage_with_autodep/pym/portage/tests/locks/test_lock_nonblock.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import shutil -import tempfile -import traceback - -import portage -from portage import os -from portage.tests import TestCase - -class LockNonblockTestCase(TestCase): - - def testLockNonblock(self): - tempdir = tempfile.mkdtemp() - try: - path = os.path.join(tempdir, 'lock_me') - lock1 = portage.locks.lockfile(path) - pid = os.fork() - if pid == 0: - portage.process._setup_pipes({0:0, 1:1, 2:2}) - rval = 2 - try: - try: - lock2 = portage.locks.lockfile(path, flags=os.O_NONBLOCK) - except portage.exception.TryAgain: - rval = os.EX_OK - else: - rval = 1 - portage.locks.unlockfile(lock2) - except SystemExit: - raise - except: - traceback.print_exc() - finally: - os._exit(rval) - - self.assertEqual(pid > 0, True) - pid, status = os.waitpid(pid, 0) - self.assertEqual(os.WIFEXITED(status), True) - self.assertEqual(os.WEXITSTATUS(status), os.EX_OK) - - portage.locks.unlockfile(lock1) - finally: - shutil.rmtree(tempdir) - diff --git a/portage_with_autodep/pym/portage/tests/news/__init__.py b/portage_with_autodep/pym/portage/tests/news/__init__.py deleted file mode 100644 index 28a753f..0000000 --- a/portage_with_autodep/pym/portage/tests/news/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# tests/portage.news/__init__.py -- Portage Unit Test functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/tests/news/__test__ b/portage_with_autodep/pym/portage/tests/news/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/news/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/news/test_NewsItem.py b/portage_with_autodep/pym/portage/tests/news/test_NewsItem.py deleted file mode 100644 index a4e76f3..0000000 --- a/portage_with_autodep/pym/portage/tests/news/test_NewsItem.py +++ /dev/null @@ -1,95 +0,0 @@ -# test_NewsItem.py -- Portage Unit Testing Functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage.tests import TestCase -from portage.news import NewsItem -from portage.dbapi.virtual import testdbapi -from tempfile import mkstemp -# TODO(antarus) Make newsitem use a loader so we can load using a string instead of a tempfile - -class NewsItemTestCase(TestCase): - """These tests suck: they use your running config instead of making their own""" - fakeItem = """ -Title: YourSQL Upgrades from 4.0 to 4.1 -Author: Ciaran McCreesh <ciaranm@gentoo.org> -Content-Type: text/plain -Posted: 01-Nov-2005 -Revision: 1 -#Display-If-Installed: -#Display-If-Profile: -#Display-If-Arch: - -YourSQL databases created using YourSQL version 4.0 are incompatible -with YourSQL version 4.1 or later. There is no reliable way to -automate the database format conversion, so action from the system -administrator is required before an upgrade can take place. - -Please see the Gentoo YourSQL Upgrade Guide for instructions: - - http://www.gentoo.org/doc/en/yoursql-upgrading.xml - -Also see the official YourSQL documentation: - - http://dev.yoursql.com/doc/refman/4.1/en/upgrading-from-4-0.html - -After upgrading, you should also recompile any packages which link -against YourSQL: - - revdep-rebuild --library=libyoursqlclient.so.12 - -The revdep-rebuild tool is provided by app-portage/gentoolkit. -""" - def setUp(self): - self.profile = "/usr/portage/profiles/default-linux/x86/2007.0/" - self.keywords = "x86" - # Use fake/test dbapi to avoid slow tests - self.vardb = testdbapi() - # self.vardb.inject_cpv('sys-apps/portage-2.0', { 'SLOT' : 0 }) - # Consumers only use ARCH, so avoid portage.settings by using a dict - self.settings = { 'ARCH' : 'x86' } - - def testDisplayIfProfile(self): - tmpItem = self.fakeItem[:].replace("#Display-If-Profile:", "Display-If-Profile: %s" % - self.profile) - - item = self._processItem(tmpItem) - try: - self.assertTrue(item.isRelevant(self.vardb, self.settings, self.profile), - msg="Expected %s to be relevant, but it was not!" % tmpItem) - finally: - os.unlink(item.path) - - def testDisplayIfInstalled(self): - tmpItem = self.fakeItem[:].replace("#Display-If-Installed:", "Display-If-Installed: %s" % - "sys-apps/portage") - - try: - item = self._processItem(tmpItem) - self.assertTrue(item.isRelevant(self.vardb, self.settings, self.profile), - msg="Expected %s to be relevant, but it was not!" % tmpItem) - finally: - os.unlink(item.path) - - def testDisplayIfKeyword(self): - tmpItem = self.fakeItem[:].replace("#Display-If-Keyword:", "Display-If-Keyword: %s" % - self.keywords) - - try: - item = self._processItem(tmpItem) - self.assertTrue(item.isRelevant(self.vardb, self.settings, self.profile), - msg="Expected %s to be relevant, but it was not!" % tmpItem) - finally: - os.unlink(item.path) - - def _processItem(self, item): - filename = None - fd, filename = mkstemp() - f = os.fdopen(fd, 'w') - f.write(item) - f.close() - try: - return NewsItem(filename, 0) - except TypeError: - self.fail("Error while processing news item %s" % filename) diff --git a/portage_with_autodep/pym/portage/tests/process/__init__.py b/portage_with_autodep/pym/portage/tests/process/__init__.py deleted file mode 100644 index d19e353..0000000 --- a/portage_with_autodep/pym/portage/tests/process/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 1998-2008 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/tests/process/__test__ b/portage_with_autodep/pym/portage/tests/process/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/process/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/process/test_poll.py b/portage_with_autodep/pym/portage/tests/process/test_poll.py deleted file mode 100644 index ee6ee0c..0000000 --- a/portage_with_autodep/pym/portage/tests/process/test_poll.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 1998-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage.tests import TestCase -from _emerge.PollScheduler import PollScheduler -from _emerge.PipeReader import PipeReader -from _emerge.SpawnProcess import SpawnProcess - -class PipeReaderTestCase(TestCase): - - def testPipeReader(self): - """ - Use a poll loop to read data from a pipe and assert that - the data written to the pipe is identical to the data - read from the pipe. - """ - - test_string = 2 * "blah blah blah\n" - - scheduler = PollScheduler().sched_iface - master_fd, slave_fd = os.pipe() - master_file = os.fdopen(master_fd, 'rb', 0) - slave_file = os.fdopen(slave_fd, 'wb') - producer = SpawnProcess( - args=["bash", "-c", "echo -n '%s'" % test_string], - env=os.environ, fd_pipes={1:slave_fd}, - scheduler=scheduler) - producer.start() - slave_file.close() - - consumer = PipeReader( - input_files={"producer" : master_file}, - scheduler=scheduler) - - consumer.start() - consumer.wait() - output = consumer.getvalue().decode('ascii', 'replace') - self.assertEqual(test_string, output) diff --git a/portage_with_autodep/pym/portage/tests/resolver/ResolverPlayground.py b/portage_with_autodep/pym/portage/tests/resolver/ResolverPlayground.py deleted file mode 100644 index 6a8e3c1..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/ResolverPlayground.py +++ /dev/null @@ -1,690 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from itertools import permutations -import shutil -import sys -import tempfile -import portage -from portage import os -from portage.const import PORTAGE_BASE_PATH -from portage.dbapi.vartree import vartree -from portage.dbapi.porttree import portagetree -from portage.dbapi.bintree import binarytree -from portage.dep import Atom, _repo_separator -from portage.package.ebuild.config import config -from portage.package.ebuild.digestgen import digestgen -from portage._sets import load_default_config -from portage._sets.base import InternalPackageSet -from portage.versions import catsplit - -import _emerge -from _emerge.actions import calc_depclean -from _emerge.Blocker import Blocker -from _emerge.create_depgraph_params import create_depgraph_params -from _emerge.depgraph import backtrack_depgraph -from _emerge.RootConfig import RootConfig - -if sys.hexversion >= 0x3000000: - basestring = str - -class ResolverPlayground(object): - """ - This class helps to create the necessary files on disk and - the needed settings instances, etc. for the resolver to do - its work. - """ - - config_files = frozenset(("package.use", "package.mask", "package.keywords", \ - "package.unmask", "package.properties", "package.license", "use.mask", "use.force")) - - def __init__(self, ebuilds={}, installed={}, profile={}, repo_configs={}, \ - user_config={}, sets={}, world=[], debug=False): - """ - ebuilds: cpv -> metadata mapping simulating available ebuilds. - installed: cpv -> metadata mapping simulating installed packages. - If a metadata key is missing, it gets a default value. - profile: settings defined by the profile. - """ - self.debug = debug - self.root = "/" - self.eprefix = tempfile.mkdtemp() - self.eroot = self.root + self.eprefix.lstrip(os.sep) + os.sep - self.portdir = os.path.join(self.eroot, "usr/portage") - self.vdbdir = os.path.join(self.eroot, "var/db/pkg") - os.makedirs(self.portdir) - os.makedirs(self.vdbdir) - - if not debug: - portage.util.noiselimit = -2 - - self.repo_dirs = {} - #Make sure the main repo is always created - self._get_repo_dir("test_repo") - - self._create_ebuilds(ebuilds) - self._create_installed(installed) - self._create_profile(ebuilds, installed, profile, repo_configs, user_config, sets) - self._create_world(world) - - self.settings, self.trees = self._load_config() - - self._create_ebuild_manifests(ebuilds) - - portage.util.noiselimit = 0 - - def _get_repo_dir(self, repo): - """ - Create the repo directory if needed. - """ - if repo not in self.repo_dirs: - if repo == "test_repo": - repo_path = self.portdir - else: - repo_path = os.path.join(self.eroot, "usr", "local", repo) - - self.repo_dirs[repo] = repo_path - profile_path = os.path.join(repo_path, "profiles") - - try: - os.makedirs(profile_path) - except os.error: - pass - - repo_name_file = os.path.join(profile_path, "repo_name") - f = open(repo_name_file, "w") - f.write("%s\n" % repo) - f.close() - - return self.repo_dirs[repo] - - def _create_ebuilds(self, ebuilds): - for cpv in ebuilds: - a = Atom("=" + cpv, allow_repo=True) - repo = a.repo - if repo is None: - repo = "test_repo" - - metadata = ebuilds[cpv].copy() - eapi = metadata.pop("EAPI", 0) - lic = metadata.pop("LICENSE", "") - properties = metadata.pop("PROPERTIES", "") - slot = metadata.pop("SLOT", 0) - keywords = metadata.pop("KEYWORDS", "x86") - iuse = metadata.pop("IUSE", "") - depend = metadata.pop("DEPEND", "") - rdepend = metadata.pop("RDEPEND", None) - pdepend = metadata.pop("PDEPEND", None) - required_use = metadata.pop("REQUIRED_USE", None) - - if metadata: - raise ValueError("metadata of ebuild '%s' contains unknown keys: %s" % (cpv, metadata.keys())) - - repo_dir = self._get_repo_dir(repo) - ebuild_dir = os.path.join(repo_dir, a.cp) - ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild") - try: - os.makedirs(ebuild_dir) - except os.error: - pass - - f = open(ebuild_path, "w") - f.write('EAPI="' + str(eapi) + '"\n') - f.write('LICENSE="' + str(lic) + '"\n') - f.write('PROPERTIES="' + str(properties) + '"\n') - f.write('SLOT="' + str(slot) + '"\n') - f.write('KEYWORDS="' + str(keywords) + '"\n') - f.write('IUSE="' + str(iuse) + '"\n') - f.write('DEPEND="' + str(depend) + '"\n') - if rdepend is not None: - f.write('RDEPEND="' + str(rdepend) + '"\n') - if pdepend is not None: - f.write('PDEPEND="' + str(pdepend) + '"\n') - if required_use is not None: - f.write('REQUIRED_USE="' + str(required_use) + '"\n') - f.close() - - def _create_ebuild_manifests(self, ebuilds): - tmpsettings = config(clone=self.settings) - tmpsettings['PORTAGE_QUIET'] = '1' - for cpv in ebuilds: - a = Atom("=" + cpv, allow_repo=True) - repo = a.repo - if repo is None: - repo = "test_repo" - - repo_dir = self._get_repo_dir(repo) - ebuild_dir = os.path.join(repo_dir, a.cp) - ebuild_path = os.path.join(ebuild_dir, a.cpv.split("/")[1] + ".ebuild") - - portdb = self.trees[self.root]["porttree"].dbapi - tmpsettings['O'] = ebuild_dir - if not digestgen(mysettings=tmpsettings, myportdb=portdb): - raise AssertionError('digest creation failed for %s' % ebuild_path) - - def _create_installed(self, installed): - for cpv in installed: - a = Atom("=" + cpv, allow_repo=True) - repo = a.repo - if repo is None: - repo = "test_repo" - - vdb_pkg_dir = os.path.join(self.vdbdir, a.cpv) - try: - os.makedirs(vdb_pkg_dir) - except os.error: - pass - - metadata = installed[cpv].copy() - eapi = metadata.pop("EAPI", 0) - lic = metadata.pop("LICENSE", "") - properties = metadata.pop("PROPERTIES", "") - slot = metadata.pop("SLOT", 0) - keywords = metadata.pop("KEYWORDS", "~x86") - iuse = metadata.pop("IUSE", "") - use = metadata.pop("USE", "") - depend = metadata.pop("DEPEND", "") - rdepend = metadata.pop("RDEPEND", None) - pdepend = metadata.pop("PDEPEND", None) - required_use = metadata.pop("REQUIRED_USE", None) - - if metadata: - raise ValueError("metadata of installed '%s' contains unknown keys: %s" % (cpv, metadata.keys())) - - def write_key(key, value): - f = open(os.path.join(vdb_pkg_dir, key), "w") - f.write(str(value) + "\n") - f.close() - - write_key("EAPI", eapi) - write_key("LICENSE", lic) - write_key("PROPERTIES", properties) - write_key("SLOT", slot) - write_key("LICENSE", lic) - write_key("PROPERTIES", properties) - write_key("repository", repo) - write_key("KEYWORDS", keywords) - write_key("IUSE", iuse) - write_key("USE", use) - write_key("DEPEND", depend) - if rdepend is not None: - write_key("RDEPEND", rdepend) - if pdepend is not None: - write_key("PDEPEND", pdepend) - if required_use is not None: - write_key("REQUIRED_USE", required_use) - - def _create_profile(self, ebuilds, installed, profile, repo_configs, user_config, sets): - - for repo in self.repo_dirs: - repo_dir = self._get_repo_dir(repo) - profile_dir = os.path.join(self._get_repo_dir(repo), "profiles") - - #Create $REPO/profiles/categories - categories = set() - for cpv in ebuilds: - ebuilds_repo = Atom("="+cpv, allow_repo=True).repo - if ebuilds_repo is None: - ebuilds_repo = "test_repo" - if ebuilds_repo == repo: - categories.add(catsplit(cpv)[0]) - - categories_file = os.path.join(profile_dir, "categories") - f = open(categories_file, "w") - for cat in categories: - f.write(cat + "\n") - f.close() - - #Create $REPO/profiles/license_groups - license_file = os.path.join(profile_dir, "license_groups") - f = open(license_file, "w") - f.write("EULA TEST\n") - f.close() - - repo_config = repo_configs.get(repo) - if repo_config: - for config_file, lines in repo_config.items(): - if config_file not in self.config_files: - raise ValueError("Unknown config file: '%s'" % config_file) - - file_name = os.path.join(profile_dir, config_file) - f = open(file_name, "w") - for line in lines: - f.write("%s\n" % line) - f.close() - - #Create $profile_dir/eclass (we fail to digest the ebuilds if it's not there) - os.makedirs(os.path.join(repo_dir, "eclass")) - - if repo == "test_repo": - #Create a minimal profile in /usr/portage - sub_profile_dir = os.path.join(profile_dir, "default", "linux", "x86", "test_profile") - os.makedirs(sub_profile_dir) - - eapi_file = os.path.join(sub_profile_dir, "eapi") - f = open(eapi_file, "w") - f.write("0\n") - f.close() - - make_defaults_file = os.path.join(sub_profile_dir, "make.defaults") - f = open(make_defaults_file, "w") - f.write("ARCH=\"x86\"\n") - f.write("ACCEPT_KEYWORDS=\"x86\"\n") - f.close() - - use_force_file = os.path.join(sub_profile_dir, "use.force") - f = open(use_force_file, "w") - f.write("x86\n") - f.close() - - if profile: - for config_file, lines in profile.items(): - if config_file not in self.config_files: - raise ValueError("Unknown config file: '%s'" % config_file) - - file_name = os.path.join(sub_profile_dir, config_file) - f = open(file_name, "w") - for line in lines: - f.write("%s\n" % line) - f.close() - - #Create profile symlink - os.makedirs(os.path.join(self.eroot, "etc")) - os.symlink(sub_profile_dir, os.path.join(self.eroot, "etc", "make.profile")) - - user_config_dir = os.path.join(self.eroot, "etc", "portage") - - try: - os.makedirs(user_config_dir) - except os.error: - pass - - repos_conf_file = os.path.join(user_config_dir, "repos.conf") - f = open(repos_conf_file, "w") - priority = 0 - for repo in sorted(self.repo_dirs.keys()): - f.write("[%s]\n" % repo) - f.write("LOCATION=%s\n" % self.repo_dirs[repo]) - if repo == "test_repo": - f.write("PRIORITY=%s\n" % -1000) - else: - f.write("PRIORITY=%s\n" % priority) - priority += 1 - f.close() - - for config_file, lines in user_config.items(): - if config_file not in self.config_files: - raise ValueError("Unknown config file: '%s'" % config_file) - - file_name = os.path.join(user_config_dir, config_file) - f = open(file_name, "w") - for line in lines: - f.write("%s\n" % line) - f.close() - - #Create /usr/share/portage/config/sets/portage.conf - default_sets_conf_dir = os.path.join(self.eroot, "usr/share/portage/config/sets") - - try: - os.makedirs(default_sets_conf_dir) - except os.error: - pass - - provided_sets_portage_conf = \ - os.path.join(PORTAGE_BASE_PATH, "cnf/sets/portage.conf") - os.symlink(provided_sets_portage_conf, os.path.join(default_sets_conf_dir, "portage.conf")) - - set_config_dir = os.path.join(user_config_dir, "sets") - - try: - os.makedirs(set_config_dir) - except os.error: - pass - - for sets_file, lines in sets.items(): - file_name = os.path.join(set_config_dir, sets_file) - f = open(file_name, "w") - for line in lines: - f.write("%s\n" % line) - f.close() - - user_config_dir = os.path.join(self.eroot, "etc", "portage") - - try: - os.makedirs(user_config_dir) - except os.error: - pass - - for config_file, lines in user_config.items(): - if config_file not in self.config_files: - raise ValueError("Unknown config file: '%s'" % config_file) - - file_name = os.path.join(user_config_dir, config_file) - f = open(file_name, "w") - for line in lines: - f.write("%s\n" % line) - f.close() - - def _create_world(self, world): - #Create /var/lib/portage/world - var_lib_portage = os.path.join(self.eroot, "var", "lib", "portage") - os.makedirs(var_lib_portage) - - world_file = os.path.join(var_lib_portage, "world") - - f = open(world_file, "w") - for atom in world: - f.write("%s\n" % atom) - f.close() - - def _load_config(self): - portdir_overlay = [] - for repo_name in sorted(self.repo_dirs): - path = self.repo_dirs[repo_name] - if path != self.portdir: - portdir_overlay.append(path) - - env = { - "ACCEPT_KEYWORDS": "x86", - "PORTDIR": self.portdir, - "PORTDIR_OVERLAY": " ".join(portdir_overlay), - 'PORTAGE_TMPDIR' : os.path.join(self.eroot, 'var/tmp'), - } - - # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they - # need to be inherited by ebuild subprocesses. - if 'PORTAGE_USERNAME' in os.environ: - env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] - if 'PORTAGE_GRPNAME' in os.environ: - env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] - - settings = config(_eprefix=self.eprefix, env=env) - settings.lock() - - trees = { - self.root: { - "vartree": vartree(settings=settings), - "porttree": portagetree(self.root, settings=settings), - "bintree": binarytree(self.root, - os.path.join(self.eroot, "usr/portage/packages"), - settings=settings) - } - } - - for root, root_trees in trees.items(): - settings = root_trees["vartree"].settings - settings._init_dirs() - setconfig = load_default_config(settings, root_trees) - root_trees["root_config"] = RootConfig(settings, root_trees, setconfig) - - return settings, trees - - def run(self, atoms, options={}, action=None): - options = options.copy() - options["--pretend"] = True - if self.debug: - options["--debug"] = True - - global_noiselimit = portage.util.noiselimit - global_emergelog_disable = _emerge.emergelog._disable - try: - - if not self.debug: - portage.util.noiselimit = -2 - _emerge.emergelog._disable = True - - if options.get("--depclean"): - rval, cleanlist, ordered, req_pkg_count = \ - calc_depclean(self.settings, self.trees, None, - options, "depclean", InternalPackageSet(initial_atoms=atoms, allow_wildcard=True), None) - result = ResolverPlaygroundDepcleanResult( \ - atoms, rval, cleanlist, ordered, req_pkg_count) - else: - params = create_depgraph_params(options, action) - success, depgraph, favorites = backtrack_depgraph( - self.settings, self.trees, options, params, action, atoms, None) - depgraph._show_merge_list() - depgraph.display_problems() - result = ResolverPlaygroundResult(atoms, success, depgraph, favorites) - finally: - portage.util.noiselimit = global_noiselimit - _emerge.emergelog._disable = global_emergelog_disable - - return result - - def run_TestCase(self, test_case): - if not isinstance(test_case, ResolverPlaygroundTestCase): - raise TypeError("ResolverPlayground needs a ResolverPlaygroundTestCase") - for atoms in test_case.requests: - result = self.run(atoms, test_case.options, test_case.action) - if not test_case.compare_with_result(result): - return - - def cleanup(self): - portdb = self.trees[self.root]["porttree"].dbapi - portdb.close_caches() - portage.dbapi.porttree.portdbapi.portdbapi_instances.remove(portdb) - if self.debug: - print("\nEROOT=%s" % self.eroot) - else: - shutil.rmtree(self.eroot) - -class ResolverPlaygroundTestCase(object): - - def __init__(self, request, **kwargs): - self.all_permutations = kwargs.pop("all_permutations", False) - self.ignore_mergelist_order = kwargs.pop("ignore_mergelist_order", False) - self.ambiguous_merge_order = kwargs.pop("ambiguous_merge_order", False) - self.check_repo_names = kwargs.pop("check_repo_names", False) - self.merge_order_assertions = kwargs.pop("merge_order_assertions", False) - - if self.all_permutations: - self.requests = list(permutations(request)) - else: - self.requests = [request] - - self.options = kwargs.pop("options", {}) - self.action = kwargs.pop("action", None) - self.test_success = True - self.fail_msg = None - self._checks = kwargs.copy() - - def compare_with_result(self, result): - checks = dict.fromkeys(result.checks) - for key, value in self._checks.items(): - if not key in checks: - raise KeyError("Not an available check: '%s'" % key) - checks[key] = value - - fail_msgs = [] - for key, value in checks.items(): - got = getattr(result, key) - expected = value - - if key in result.optional_checks and expected is None: - continue - - if key == "mergelist": - if not self.check_repo_names: - #Strip repo names if we don't check them - if got: - new_got = [] - for cpv in got: - if cpv[:1] == "!": - new_got.append(cpv) - continue - a = Atom("="+cpv, allow_repo=True) - new_got.append(a.cpv) - got = new_got - if expected: - new_expected = [] - for obj in expected: - if isinstance(obj, basestring): - if obj[:1] == "!": - new_expected.append(obj) - continue - a = Atom("="+obj, allow_repo=True) - new_expected.append(a.cpv) - continue - new_expected.append(set()) - for cpv in obj: - if cpv[:1] != "!": - cpv = Atom("="+cpv, allow_repo=True).cpv - new_expected[-1].add(cpv) - expected = new_expected - if self.ignore_mergelist_order and got is not None: - got = set(got) - expected = set(expected) - - if self.ambiguous_merge_order and got: - expected_stack = list(reversed(expected)) - got_stack = list(reversed(got)) - new_expected = [] - match = True - while got_stack and expected_stack: - got_token = got_stack.pop() - expected_obj = expected_stack.pop() - if isinstance(expected_obj, basestring): - new_expected.append(expected_obj) - if got_token == expected_obj: - continue - # result doesn't match, so stop early - match = False - break - expected_obj = set(expected_obj) - try: - expected_obj.remove(got_token) - except KeyError: - # result doesn't match, so stop early - match = False - break - new_expected.append(got_token) - while got_stack and expected_obj: - got_token = got_stack.pop() - try: - expected_obj.remove(got_token) - except KeyError: - match = False - break - new_expected.append(got_token) - if not match: - # result doesn't match, so stop early - break - if expected_obj: - # result does not match, so stop early - match = False - new_expected.append(tuple(expected_obj)) - break - if expected_stack: - # result does not match, add leftovers to new_expected - match = False - expected_stack.reverse() - new_expected.extend(expected_stack) - expected = new_expected - - if match and self.merge_order_assertions: - for node1, node2 in self.merge_order_assertions: - if not (got.index(node1) < got.index(node2)): - fail_msgs.append("atoms: (" + \ - ", ".join(result.atoms) + "), key: " + \ - ("merge_order_assertions, expected: %s" % \ - str((node1, node2))) + \ - ", got: " + str(got)) - - elif key in ("unstable_keywords", "needed_p_mask_changes") and expected is not None: - expected = set(expected) - - if got != expected: - fail_msgs.append("atoms: (" + ", ".join(result.atoms) + "), key: " + \ - key + ", expected: " + str(expected) + ", got: " + str(got)) - if fail_msgs: - self.test_success = False - self.fail_msg = "\n".join(fail_msgs) - return False - return True - -class ResolverPlaygroundResult(object): - - checks = ( - "success", "mergelist", "use_changes", "license_changes", "unstable_keywords", "slot_collision_solutions", - "circular_dependency_solutions", "needed_p_mask_changes", - ) - optional_checks = ( - ) - - def __init__(self, atoms, success, mydepgraph, favorites): - self.atoms = atoms - self.success = success - self.depgraph = mydepgraph - self.favorites = favorites - self.mergelist = None - self.use_changes = None - self.license_changes = None - self.unstable_keywords = None - self.needed_p_mask_changes = None - self.slot_collision_solutions = None - self.circular_dependency_solutions = None - - if self.depgraph._dynamic_config._serialized_tasks_cache is not None: - self.mergelist = [] - for x in self.depgraph._dynamic_config._serialized_tasks_cache: - if isinstance(x, Blocker): - self.mergelist.append(x.atom) - else: - repo_str = "" - if x.metadata["repository"] != "test_repo": - repo_str = _repo_separator + x.metadata["repository"] - self.mergelist.append(x.cpv + repo_str) - - if self.depgraph._dynamic_config._needed_use_config_changes: - self.use_changes = {} - for pkg, needed_use_config_changes in \ - self.depgraph._dynamic_config._needed_use_config_changes.items(): - new_use, changes = needed_use_config_changes - self.use_changes[pkg.cpv] = changes - - if self.depgraph._dynamic_config._needed_unstable_keywords: - self.unstable_keywords = set() - for pkg in self.depgraph._dynamic_config._needed_unstable_keywords: - self.unstable_keywords.add(pkg.cpv) - - if self.depgraph._dynamic_config._needed_p_mask_changes: - self.needed_p_mask_changes = set() - for pkg in self.depgraph._dynamic_config._needed_p_mask_changes: - self.needed_p_mask_changes.add(pkg.cpv) - - if self.depgraph._dynamic_config._needed_license_changes: - self.license_changes = {} - for pkg, missing_licenses in self.depgraph._dynamic_config._needed_license_changes.items(): - self.license_changes[pkg.cpv] = missing_licenses - - if self.depgraph._dynamic_config._slot_conflict_handler is not None: - self.slot_collision_solutions = [] - handler = self.depgraph._dynamic_config._slot_conflict_handler - - for change in handler.changes: - new_change = {} - for pkg in change: - new_change[pkg.cpv] = change[pkg] - self.slot_collision_solutions.append(new_change) - - if self.depgraph._dynamic_config._circular_dependency_handler is not None: - handler = self.depgraph._dynamic_config._circular_dependency_handler - sol = handler.solutions - self.circular_dependency_solutions = dict( zip([x.cpv for x in sol.keys()], sol.values()) ) - -class ResolverPlaygroundDepcleanResult(object): - - checks = ( - "success", "cleanlist", "ordered", "req_pkg_count", - ) - optional_checks = ( - "ordered", "req_pkg_count", - ) - - def __init__(self, atoms, rval, cleanlist, ordered, req_pkg_count): - self.atoms = atoms - self.success = rval == 0 - self.cleanlist = cleanlist - self.ordered = ordered - self.req_pkg_count = req_pkg_count diff --git a/portage_with_autodep/pym/portage/tests/resolver/__test__ b/portage_with_autodep/pym/portage/tests/resolver/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_autounmask.py b/portage_with_autodep/pym/portage/tests/resolver/test_autounmask.py deleted file mode 100644 index 54c435f..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_autounmask.py +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class AutounmaskTestCase(TestCase): - - def testAutounmask(self): - - ebuilds = { - #ebuilds to test use changes - "dev-libs/A-1": { "SLOT": 1, "DEPEND": "dev-libs/B[foo]", "EAPI": 2}, - "dev-libs/A-2": { "SLOT": 2, "DEPEND": "dev-libs/B[bar]", "EAPI": 2}, - "dev-libs/B-1": { "DEPEND": "foo? ( dev-libs/C ) bar? ( dev-libs/D )", "IUSE": "foo bar"}, - "dev-libs/C-1": {}, - "dev-libs/D-1": {}, - - #ebuilds to test if we allow changing of masked or forced flags - "dev-libs/E-1": { "SLOT": 1, "DEPEND": "dev-libs/F[masked-flag]", "EAPI": 2}, - "dev-libs/E-2": { "SLOT": 2, "DEPEND": "dev-libs/G[-forced-flag]", "EAPI": 2}, - "dev-libs/F-1": { "IUSE": "masked-flag"}, - "dev-libs/G-1": { "IUSE": "forced-flag"}, - - #ebuilds to test keyword changes - "app-misc/Z-1": { "KEYWORDS": "~x86", "DEPEND": "app-misc/Y" }, - "app-misc/Y-1": { "KEYWORDS": "~x86" }, - "app-misc/W-1": {}, - "app-misc/W-2": { "KEYWORDS": "~x86" }, - "app-misc/V-1": { "KEYWORDS": "~x86", "DEPEND": ">=app-misc/W-2"}, - - #ebuilds to test mask and keyword changes - "app-text/A-1": {}, - "app-text/B-1": { "KEYWORDS": "~x86" }, - "app-text/C-1": { "KEYWORDS": "" }, - "app-text/D-1": { "KEYWORDS": "~x86" }, - "app-text/D-2": { "KEYWORDS": "" }, - - #ebuilds for mixed test for || dep handling - "sci-libs/K-1": { "DEPEND": " || ( sci-libs/L[bar] || ( sci-libs/M sci-libs/P ) )", "EAPI": 2}, - "sci-libs/K-2": { "DEPEND": " || ( sci-libs/L[bar] || ( sci-libs/P sci-libs/M ) )", "EAPI": 2}, - "sci-libs/K-3": { "DEPEND": " || ( sci-libs/M || ( sci-libs/L[bar] sci-libs/P ) )", "EAPI": 2}, - "sci-libs/K-4": { "DEPEND": " || ( sci-libs/M || ( sci-libs/P sci-libs/L[bar] ) )", "EAPI": 2}, - "sci-libs/K-5": { "DEPEND": " || ( sci-libs/P || ( sci-libs/L[bar] sci-libs/M ) )", "EAPI": 2}, - "sci-libs/K-6": { "DEPEND": " || ( sci-libs/P || ( sci-libs/M sci-libs/L[bar] ) )", "EAPI": 2}, - "sci-libs/K-7": { "DEPEND": " || ( sci-libs/M sci-libs/L[bar] )", "EAPI": 2}, - "sci-libs/K-8": { "DEPEND": " || ( sci-libs/L[bar] sci-libs/M )", "EAPI": 2}, - - "sci-libs/L-1": { "IUSE": "bar" }, - "sci-libs/M-1": { "KEYWORDS": "~x86" }, - "sci-libs/P-1": { }, - - #ebuilds to test these nice "required by cat/pkg[foo]" messages - "dev-util/Q-1": { "DEPEND": "foo? ( dev-util/R[bar] )", "IUSE": "+foo", "EAPI": 2 }, - "dev-util/Q-2": { "RDEPEND": "!foo? ( dev-util/R[bar] )", "IUSE": "foo", "EAPI": 2 }, - "dev-util/R-1": { "IUSE": "bar" }, - - #ebuilds to test interaction with REQUIRED_USE - "app-portage/A-1": { "DEPEND": "app-portage/B[foo]", "EAPI": 2 }, - "app-portage/A-2": { "DEPEND": "app-portage/B[foo=]", "IUSE": "+foo", "REQUIRED_USE": "foo", "EAPI": "4" }, - - "app-portage/B-1": { "IUSE": "foo +bar", "REQUIRED_USE": "^^ ( foo bar )", "EAPI": "4" }, - "app-portage/C-1": { "IUSE": "+foo +bar", "REQUIRED_USE": "^^ ( foo bar )", "EAPI": "4" }, - } - - test_cases = ( - #Test USE changes. - #The simple case. - - ResolverPlaygroundTestCase( - ["dev-libs/A:1"], - options = {"--autounmask": "n"}, - success = False), - ResolverPlaygroundTestCase( - ["dev-libs/A:1"], - options = {"--autounmask": True}, - success = False, - mergelist = ["dev-libs/C-1", "dev-libs/B-1", "dev-libs/A-1"], - use_changes = { "dev-libs/B-1": {"foo": True} } ), - - #Make sure we restart if needed. - ResolverPlaygroundTestCase( - ["dev-libs/A:1", "dev-libs/B"], - options = {"--autounmask": True}, - all_permutations = True, - success = False, - mergelist = ["dev-libs/C-1", "dev-libs/B-1", "dev-libs/A-1"], - use_changes = { "dev-libs/B-1": {"foo": True} } ), - ResolverPlaygroundTestCase( - ["dev-libs/A:1", "dev-libs/A:2", "dev-libs/B"], - options = {"--autounmask": True}, - all_permutations = True, - success = False, - mergelist = ["dev-libs/D-1", "dev-libs/C-1", "dev-libs/B-1", "dev-libs/A-1", "dev-libs/A-2"], - ignore_mergelist_order = True, - use_changes = { "dev-libs/B-1": {"foo": True, "bar": True} } ), - - #Test keywording. - #The simple case. - - ResolverPlaygroundTestCase( - ["app-misc/Z"], - options = {"--autounmask": "n"}, - success = False), - ResolverPlaygroundTestCase( - ["app-misc/Z"], - options = {"--autounmask": True}, - success = False, - mergelist = ["app-misc/Y-1", "app-misc/Z-1"], - unstable_keywords = ["app-misc/Y-1", "app-misc/Z-1"]), - - #Make sure that the backtracking for slot conflicts handles our mess. - - ResolverPlaygroundTestCase( - ["=app-misc/V-1", "app-misc/W"], - options = {"--autounmask": True}, - all_permutations = True, - success = False, - mergelist = ["app-misc/W-2", "app-misc/V-1"], - unstable_keywords = ["app-misc/W-2", "app-misc/V-1"]), - - #Mixed testing - #Make sure we don't change use for something in a || dep if there is another choice - #that needs no change. - - ResolverPlaygroundTestCase( - ["=sci-libs/K-1"], - options = {"--autounmask": True}, - success = True, - mergelist = ["sci-libs/P-1", "sci-libs/K-1"]), - ResolverPlaygroundTestCase( - ["=sci-libs/K-2"], - options = {"--autounmask": True}, - success = True, - mergelist = ["sci-libs/P-1", "sci-libs/K-2"]), - ResolverPlaygroundTestCase( - ["=sci-libs/K-3"], - options = {"--autounmask": True}, - success = True, - mergelist = ["sci-libs/P-1", "sci-libs/K-3"]), - ResolverPlaygroundTestCase( - ["=sci-libs/K-4"], - options = {"--autounmask": True}, - success = True, - mergelist = ["sci-libs/P-1", "sci-libs/K-4"]), - ResolverPlaygroundTestCase( - ["=sci-libs/K-5"], - options = {"--autounmask": True}, - success = True, - mergelist = ["sci-libs/P-1", "sci-libs/K-5"]), - ResolverPlaygroundTestCase( - ["=sci-libs/K-6"], - options = {"--autounmask": True}, - success = True, - mergelist = ["sci-libs/P-1", "sci-libs/K-6"]), - - #Make sure we prefer use changes over keyword changes. - ResolverPlaygroundTestCase( - ["=sci-libs/K-7"], - options = {"--autounmask": True}, - success = False, - mergelist = ["sci-libs/L-1", "sci-libs/K-7"], - use_changes = { "sci-libs/L-1": { "bar": True } }), - ResolverPlaygroundTestCase( - ["=sci-libs/K-8"], - options = {"--autounmask": True}, - success = False, - mergelist = ["sci-libs/L-1", "sci-libs/K-8"], - use_changes = { "sci-libs/L-1": { "bar": True } }), - - #Test these nice "required by cat/pkg[foo]" messages. - ResolverPlaygroundTestCase( - ["=dev-util/Q-1"], - options = {"--autounmask": True}, - success = False, - mergelist = ["dev-util/R-1", "dev-util/Q-1"], - use_changes = { "dev-util/R-1": { "bar": True } }), - ResolverPlaygroundTestCase( - ["=dev-util/Q-2"], - options = {"--autounmask": True}, - success = False, - mergelist = ["dev-util/R-1", "dev-util/Q-2"], - use_changes = { "dev-util/R-1": { "bar": True } }), - - #Test interaction with REQUIRED_USE. - ResolverPlaygroundTestCase( - ["=app-portage/A-1"], - options = { "--autounmask": True }, - use_changes = None, - success = False), - ResolverPlaygroundTestCase( - ["=app-portage/A-2"], - options = { "--autounmask": True }, - use_changes = None, - success = False), - ResolverPlaygroundTestCase( - ["=app-portage/C-1"], - options = { "--autounmask": True }, - use_changes = None, - success = False), - - #Make sure we don't change masked/forced flags. - ResolverPlaygroundTestCase( - ["dev-libs/E:1"], - options = {"--autounmask": True}, - use_changes = None, - success = False), - ResolverPlaygroundTestCase( - ["dev-libs/E:2"], - options = {"--autounmask": True}, - use_changes = None, - success = False), - - #Test mask and keyword changes. - ResolverPlaygroundTestCase( - ["app-text/A"], - options = {"--autounmask": True}, - success = False, - mergelist = ["app-text/A-1"], - needed_p_mask_changes = ["app-text/A-1"]), - ResolverPlaygroundTestCase( - ["app-text/B"], - options = {"--autounmask": True}, - success = False, - mergelist = ["app-text/B-1"], - unstable_keywords = ["app-text/B-1"], - needed_p_mask_changes = ["app-text/B-1"]), - ResolverPlaygroundTestCase( - ["app-text/C"], - options = {"--autounmask": True}, - success = False, - mergelist = ["app-text/C-1"], - unstable_keywords = ["app-text/C-1"], - needed_p_mask_changes = ["app-text/C-1"]), - #Make sure unstable keyword is preferred over missing keyword - ResolverPlaygroundTestCase( - ["app-text/D"], - options = {"--autounmask": True}, - success = False, - mergelist = ["app-text/D-1"], - unstable_keywords = ["app-text/D-1"]), - #Test missing keyword - ResolverPlaygroundTestCase( - ["=app-text/D-2"], - options = {"--autounmask": True}, - success = False, - mergelist = ["app-text/D-2"], - unstable_keywords = ["app-text/D-2"]) - ) - - profile = { - "use.mask": - ( - "masked-flag", - ), - "use.force": - ( - "forced-flag", - ), - "package.mask": - ( - "app-text/A", - "app-text/B", - "app-text/C", - ), - } - - playground = ResolverPlayground(ebuilds=ebuilds, profile=profile) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - - def testAutounmaskForLicenses(self): - - ebuilds = { - "dev-libs/A-1": { "LICENSE": "TEST" }, - "dev-libs/B-1": { "LICENSE": "TEST", "IUSE": "foo", "KEYWORDS": "~x86"}, - "dev-libs/C-1": { "DEPEND": "dev-libs/B[foo]", "EAPI": 2 }, - - "dev-libs/D-1": { "DEPEND": "dev-libs/E dev-libs/F", "LICENSE": "TEST" }, - "dev-libs/E-1": { "LICENSE": "TEST" }, - "dev-libs/E-2": { "LICENSE": "TEST" }, - "dev-libs/F-1": { "DEPEND": "=dev-libs/E-1", "LICENSE": "TEST" }, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["=dev-libs/A-1"], - options = {"--autounmask": 'n'}, - success = False), - ResolverPlaygroundTestCase( - ["=dev-libs/A-1"], - options = {"--autounmask": True}, - success = False, - mergelist = ["dev-libs/A-1"], - license_changes = { "dev-libs/A-1": set(["TEST"]) }), - - #Test license+keyword+use change at once. - ResolverPlaygroundTestCase( - ["=dev-libs/C-1"], - options = {"--autounmask": True}, - success = False, - mergelist = ["dev-libs/B-1", "dev-libs/C-1"], - license_changes = { "dev-libs/B-1": set(["TEST"]) }, - unstable_keywords = ["dev-libs/B-1"], - use_changes = { "dev-libs/B-1": { "foo": True } }), - - #Test license with backtracking. - ResolverPlaygroundTestCase( - ["=dev-libs/D-1"], - options = {"--autounmask": True}, - success = False, - mergelist = ["dev-libs/E-1", "dev-libs/F-1", "dev-libs/D-1"], - license_changes = { "dev-libs/D-1": set(["TEST"]), "dev-libs/E-1": set(["TEST"]), "dev-libs/E-2": set(["TEST"]), "dev-libs/F-1": set(["TEST"]) }), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_backtracking.py b/portage_with_autodep/pym/portage/tests/resolver/test_backtracking.py deleted file mode 100644 index fc49306..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_backtracking.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class BacktrackingTestCase(TestCase): - - def testBacktracking(self): - ebuilds = { - "dev-libs/A-1": {}, - "dev-libs/A-2": {}, - "dev-libs/B-1": { "DEPEND": "dev-libs/A" }, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["=dev-libs/A-1", "dev-libs/B"], - all_permutations = True, - mergelist = ["dev-libs/A-1", "dev-libs/B-1"], - success = True), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - - - def testHittingTheBacktrackLimit(self): - ebuilds = { - "dev-libs/A-1": {}, - "dev-libs/A-2": {}, - "dev-libs/B-1": {}, - "dev-libs/B-2": {}, - "dev-libs/C-1": { "DEPEND": "dev-libs/A dev-libs/B" }, - "dev-libs/D-1": { "DEPEND": "=dev-libs/A-1 =dev-libs/B-1" }, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["dev-libs/C", "dev-libs/D"], - all_permutations = True, - mergelist = ["dev-libs/A-1", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], - ignore_mergelist_order = True, - success = True), - #This one hits the backtrack limit. Be aware that this depends on the argument order. - ResolverPlaygroundTestCase( - ["dev-libs/D", "dev-libs/C"], - options = { "--backtrack": 1 }, - mergelist = ["dev-libs/A-1", "dev-libs/B-1", "dev-libs/A-2", "dev-libs/B-2", "dev-libs/C-1", "dev-libs/D-1"], - ignore_mergelist_order = True, - slot_collision_solutions = [], - success = False), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - - - def testBacktrackingGoodVersionFirst(self): - """ - When backtracking due to slot conflicts, we masked the version that has been pulled - in first. This is not always a good idea. Mask the highest version instead. - """ - - ebuilds = { - "dev-libs/A-1": { "DEPEND": "=dev-libs/C-1 dev-libs/B" }, - "dev-libs/B-1": { "DEPEND": "=dev-libs/C-1" }, - "dev-libs/B-2": { "DEPEND": "=dev-libs/C-2" }, - "dev-libs/C-1": { }, - "dev-libs/C-2": { }, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["dev-libs/A"], - mergelist = ["dev-libs/C-1", "dev-libs/B-1", "dev-libs/A-1", ], - success = True), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - - def testBacktrackWithoutUpdates(self): - """ - If --update is not given we might have to mask the old installed version later. - """ - - ebuilds = { - "dev-libs/A-1": { "DEPEND": "dev-libs/Z" }, - "dev-libs/B-1": { "DEPEND": ">=dev-libs/Z-2" }, - "dev-libs/Z-1": { }, - "dev-libs/Z-2": { }, - } - - installed = { - "dev-libs/Z-1": { "USE": "" }, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["dev-libs/B", "dev-libs/A"], - all_permutations = True, - mergelist = ["dev-libs/Z-2", "dev-libs/B-1", "dev-libs/A-1", ], - ignore_mergelist_order = True, - success = True), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed) - - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - - def testBacktrackMissedUpdates(self): - """ - An update is missed due to a dependency on an older version. - """ - - ebuilds = { - "dev-libs/A-1": { }, - "dev-libs/A-2": { }, - "dev-libs/B-1": { "RDEPEND": "<=dev-libs/A-1" }, - } - - installed = { - "dev-libs/A-1": { "USE": "" }, - "dev-libs/B-1": { "USE": "", "RDEPEND": "<=dev-libs/A-1" }, - } - - options = {'--update' : True, '--deep' : True, '--selective' : True} - - test_cases = ( - ResolverPlaygroundTestCase( - ["dev-libs/A", "dev-libs/B"], - options = options, - all_permutations = True, - mergelist = [], - success = True), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed) - - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_circular_dependencies.py b/portage_with_autodep/pym/portage/tests/resolver/test_circular_dependencies.py deleted file mode 100644 index f8331ac..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_circular_dependencies.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class CircularDependencyTestCase(TestCase): - - #TODO: - # use config change by autounmask - # conflict on parent's parent - # difference in RDEPEND and DEPEND - # is there anything else than priority buildtime and runtime? - # play with use.{mask,force} - # play with REQUIRED_USE - - - def testCircularDependency(self): - - ebuilds = { - "dev-libs/Z-1": { "DEPEND": "foo? ( !bar? ( dev-libs/Y ) )", "IUSE": "+foo bar", "EAPI": 1 }, - "dev-libs/Z-2": { "DEPEND": "foo? ( dev-libs/Y ) !bar? ( dev-libs/Y )", "IUSE": "+foo bar", "EAPI": 1 }, - "dev-libs/Z-3": { "DEPEND": "foo? ( !bar? ( dev-libs/Y ) ) foo? ( dev-libs/Y ) !bar? ( dev-libs/Y )", "IUSE": "+foo bar", "EAPI": 1 }, - "dev-libs/Y-1": { "DEPEND": "dev-libs/Z" }, - "dev-libs/W-1": { "DEPEND": "dev-libs/Z[foo] dev-libs/Y", "EAPI": 2 }, - "dev-libs/W-2": { "DEPEND": "dev-libs/Z[foo=] dev-libs/Y", "IUSE": "+foo", "EAPI": 2 }, - "dev-libs/W-3": { "DEPEND": "dev-libs/Z[bar] dev-libs/Y", "EAPI": 2 }, - - "app-misc/A-1": { "DEPEND": "foo? ( =app-misc/B-1 )", "IUSE": "+foo bar", "REQUIRED_USE": "^^ ( foo bar )", "EAPI": "4" }, - "app-misc/A-2": { "DEPEND": "foo? ( =app-misc/B-2 ) bar? ( =app-misc/B-2 )", "IUSE": "+foo bar", "REQUIRED_USE": "^^ ( foo bar )", "EAPI": "4" }, - "app-misc/B-1": { "DEPEND": "=app-misc/A-1" }, - "app-misc/B-2": { "DEPEND": "=app-misc/A-2" }, - } - - test_cases = ( - #Simple tests - ResolverPlaygroundTestCase( - ["=dev-libs/Z-1"], - circular_dependency_solutions = { "dev-libs/Y-1": frozenset([frozenset([("foo", False)]), frozenset([("bar", True)])])}, - success = False), - ResolverPlaygroundTestCase( - ["=dev-libs/Z-2"], - circular_dependency_solutions = { "dev-libs/Y-1": frozenset([frozenset([("foo", False), ("bar", True)])])}, - success = False), - ResolverPlaygroundTestCase( - ["=dev-libs/Z-3"], - circular_dependency_solutions = { "dev-libs/Y-1": frozenset([frozenset([("foo", False), ("bar", True)])])}, - success = False), - - #Conflict on parent - ResolverPlaygroundTestCase( - ["=dev-libs/W-1"], - circular_dependency_solutions = {}, - success = False), - ResolverPlaygroundTestCase( - ["=dev-libs/W-2"], - circular_dependency_solutions = { "dev-libs/Y-1": frozenset([frozenset([("foo", False), ("bar", True)])])}, - success = False), - - #Conflict with autounmask - ResolverPlaygroundTestCase( - ["=dev-libs/W-3"], - circular_dependency_solutions = { "dev-libs/Y-1": frozenset([frozenset([("foo", False)])])}, - use_changes = { "dev-libs/Z-3": {"bar": True}}, - success = False), - - #Conflict with REQUIRED_USE - ResolverPlaygroundTestCase( - ["=app-misc/B-1"], - circular_dependency_solutions = { "app-misc/B-1": frozenset([frozenset([("foo", False), ("bar", True)])])}, - success = False), - ResolverPlaygroundTestCase( - ["=app-misc/B-2"], - circular_dependency_solutions = {}, - success = False), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_depclean.py b/portage_with_autodep/pym/portage/tests/resolver/test_depclean.py deleted file mode 100644 index ba70144..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_depclean.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class SimpleDepcleanTestCase(TestCase): - - def testSimpleDepclean(self): - ebuilds = { - "dev-libs/A-1": {}, - "dev-libs/B-1": {}, - } - installed = { - "dev-libs/A-1": {}, - "dev-libs/B-1": {}, - } - - world = ( - "dev-libs/A", - ) - - test_cases = ( - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/B-1"]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed, world=world) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - -class DepcleanWithDepsTestCase(TestCase): - - def testDepcleanWithDeps(self): - ebuilds = { - "dev-libs/A-1": { "RDEPEND": "dev-libs/C" }, - "dev-libs/B-1": { "RDEPEND": "dev-libs/D" }, - "dev-libs/C-1": {}, - "dev-libs/D-1": { "RDEPEND": "dev-libs/E" }, - "dev-libs/E-1": { "RDEPEND": "dev-libs/F" }, - "dev-libs/F-1": {}, - } - installed = { - "dev-libs/A-1": { "RDEPEND": "dev-libs/C" }, - "dev-libs/B-1": { "RDEPEND": "dev-libs/D" }, - "dev-libs/C-1": {}, - "dev-libs/D-1": { "RDEPEND": "dev-libs/E" }, - "dev-libs/E-1": { "RDEPEND": "dev-libs/F" }, - "dev-libs/F-1": {}, - } - - world = ( - "dev-libs/A", - ) - - test_cases = ( - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/B-1", "dev-libs/D-1", - "dev-libs/E-1", "dev-libs/F-1"]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed, world=world) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - - -class DepcleanWithInstalledMaskedTestCase(TestCase): - - def testDepcleanWithInstalledMasked(self): - """ - Test case for bug 332719. - emerge --declean ignores that B is masked by license and removes C. - The next emerge -uDN world doesn't take B and installs C again. - """ - ebuilds = { - "dev-libs/A-1": { "RDEPEND": "|| ( dev-libs/B dev-libs/C )" }, - "dev-libs/B-1": { "LICENSE": "TEST", "KEYWORDS": "x86" }, - "dev-libs/C-1": { "KEYWORDS": "x86" }, - } - installed = { - "dev-libs/A-1": { "RDEPEND": "|| ( dev-libs/B dev-libs/C )" }, - "dev-libs/B-1": { "LICENSE": "TEST", "KEYWORDS": "x86" }, - "dev-libs/C-1": { "KEYWORDS": "x86" }, - } - - world = ( - "dev-libs/A", - ) - - test_cases = ( - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True}, - success = True, - #cleanlist = ["dev-libs/C-1"]), - cleanlist = ["dev-libs/B-1"]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed, world=world) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - -class DepcleanInstalledKeywordMaskedSlotTestCase(TestCase): - - def testDepcleanInstalledKeywordMaskedSlot(self): - """ - Verify that depclean removes newer slot - masked by KEYWORDS (see bug #350285). - """ - ebuilds = { - "dev-libs/A-1": { "RDEPEND": "|| ( =dev-libs/B-2.7* =dev-libs/B-2.6* )" }, - "dev-libs/B-2.6": { "SLOT":"2.6", "KEYWORDS": "x86" }, - "dev-libs/B-2.7": { "SLOT":"2.7", "KEYWORDS": "~x86" }, - } - installed = { - "dev-libs/A-1": { "EAPI" : "3", "RDEPEND": "|| ( dev-libs/B:2.7 dev-libs/B:2.6 )" }, - "dev-libs/B-2.6": { "SLOT":"2.6", "KEYWORDS": "x86" }, - "dev-libs/B-2.7": { "SLOT":"2.7", "KEYWORDS": "~x86" }, - } - - world = ( - "dev-libs/A", - ) - - test_cases = ( - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/B-2.7"]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed, world=world) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - -class DepcleanWithExcludeTestCase(TestCase): - - def testDepcleanWithExclude(self): - - installed = { - "dev-libs/A-1": {}, - "dev-libs/B-1": { "RDEPEND": "dev-libs/A" }, - } - - test_cases = ( - #Without --exclude. - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/B-1", "dev-libs/A-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/A"], - options = {"--depclean": True}, - success = True, - cleanlist = []), - ResolverPlaygroundTestCase( - ["dev-libs/B"], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/B-1"]), - - #With --exclude - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True, "--exclude": ["dev-libs/A"]}, - success = True, - cleanlist = ["dev-libs/B-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/B"], - options = {"--depclean": True, "--exclude": ["dev-libs/B"]}, - success = True, - cleanlist = []), - ) - - playground = ResolverPlayground(installed=installed) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - -class DepcleanWithExcludeAndSlotsTestCase(TestCase): - - def testDepcleanWithExcludeAndSlots(self): - - installed = { - "dev-libs/Z-1": { "SLOT": 1}, - "dev-libs/Z-2": { "SLOT": 2}, - "dev-libs/Y-1": { "RDEPEND": "=dev-libs/Z-1", "SLOT": 1 }, - "dev-libs/Y-2": { "RDEPEND": "=dev-libs/Z-2", "SLOT": 2 }, - } - - world = [ "dev-libs/Y" ] - - test_cases = ( - #Without --exclude. - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/Y-1", "dev-libs/Z-1"]), - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True, "--exclude": ["dev-libs/Z"]}, - success = True, - cleanlist = ["dev-libs/Y-1"]), - ResolverPlaygroundTestCase( - [], - options = {"--depclean": True, "--exclude": ["dev-libs/Y"]}, - success = True, - cleanlist = []), - ) - - playground = ResolverPlayground(installed=installed, world=world) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - -class DepcleanAndWildcardsTestCase(TestCase): - - def testDepcleanAndWildcards(self): - - installed = { - "dev-libs/A-1": { "RDEPEND": "dev-libs/B" }, - "dev-libs/B-1": {}, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["*/*"], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/A-1", "dev-libs/B-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/*"], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/A-1", "dev-libs/B-1"]), - ResolverPlaygroundTestCase( - ["*/A"], - options = {"--depclean": True}, - success = True, - cleanlist = ["dev-libs/A-1"]), - ResolverPlaygroundTestCase( - ["*/B"], - options = {"--depclean": True}, - success = True, - cleanlist = []), - ) - - playground = ResolverPlayground(installed=installed) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_depth.py b/portage_with_autodep/pym/portage/tests/resolver/test_depth.py deleted file mode 100644 index cb1e2dd..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_depth.py +++ /dev/null @@ -1,252 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, - ResolverPlaygroundTestCase) - -class ResolverDepthTestCase(TestCase): - - def testResolverDepth(self): - - ebuilds = { - "dev-libs/A-1": {"RDEPEND" : "dev-libs/B"}, - "dev-libs/A-2": {"RDEPEND" : "dev-libs/B"}, - "dev-libs/B-1": {"RDEPEND" : "dev-libs/C"}, - "dev-libs/B-2": {"RDEPEND" : "dev-libs/C"}, - "dev-libs/C-1": {}, - "dev-libs/C-2": {}, - - "virtual/libusb-0" : {"EAPI" :"2", "SLOT" : "0", "RDEPEND" : "|| ( >=dev-libs/libusb-0.1.12-r1:0 dev-libs/libusb-compat >=sys-freebsd/freebsd-lib-8.0[usb] )"}, - "virtual/libusb-1" : {"EAPI" :"2", "SLOT" : "1", "RDEPEND" : ">=dev-libs/libusb-1.0.4:1"}, - "dev-libs/libusb-0.1.13" : {}, - "dev-libs/libusb-1.0.5" : {"SLOT":"1"}, - "dev-libs/libusb-compat-1" : {}, - "sys-freebsd/freebsd-lib-8": {"IUSE" : "+usb"}, - - "sys-fs/udev-164" : {"EAPI" : "1", "RDEPEND" : "virtual/libusb:0"}, - - "virtual/jre-1.5.0" : {"SLOT" : "1.5", "RDEPEND" : "|| ( =dev-java/sun-jre-bin-1.5.0* =virtual/jdk-1.5.0* )"}, - "virtual/jre-1.5.0-r1" : {"SLOT" : "1.5", "RDEPEND" : "|| ( =dev-java/sun-jre-bin-1.5.0* =virtual/jdk-1.5.0* )"}, - "virtual/jre-1.6.0" : {"SLOT" : "1.6", "RDEPEND" : "|| ( =dev-java/sun-jre-bin-1.6.0* =virtual/jdk-1.6.0* )"}, - "virtual/jre-1.6.0-r1" : {"SLOT" : "1.6", "RDEPEND" : "|| ( =dev-java/sun-jre-bin-1.6.0* =virtual/jdk-1.6.0* )"}, - "virtual/jdk-1.5.0" : {"SLOT" : "1.5", "RDEPEND" : "|| ( =dev-java/sun-jdk-1.5.0* dev-java/gcj-jdk )"}, - "virtual/jdk-1.5.0-r1" : {"SLOT" : "1.5", "RDEPEND" : "|| ( =dev-java/sun-jdk-1.5.0* dev-java/gcj-jdk )"}, - "virtual/jdk-1.6.0" : {"SLOT" : "1.6", "RDEPEND" : "|| ( =dev-java/icedtea-6* =dev-java/sun-jdk-1.6.0* )"}, - "virtual/jdk-1.6.0-r1" : {"SLOT" : "1.6", "RDEPEND" : "|| ( =dev-java/icedtea-6* =dev-java/sun-jdk-1.6.0* )"}, - "dev-java/gcj-jdk-4.5" : {}, - "dev-java/gcj-jdk-4.5-r1" : {}, - "dev-java/icedtea-6.1" : {}, - "dev-java/icedtea-6.1-r1" : {}, - "dev-java/sun-jdk-1.5" : {"SLOT" : "1.5"}, - "dev-java/sun-jdk-1.6" : {"SLOT" : "1.6"}, - "dev-java/sun-jre-bin-1.5" : {"SLOT" : "1.5"}, - "dev-java/sun-jre-bin-1.6" : {"SLOT" : "1.6"}, - - "dev-java/ant-core-1.8" : {"DEPEND" : ">=virtual/jdk-1.4"}, - "dev-db/hsqldb-1.8" : {"RDEPEND" : ">=virtual/jre-1.6"}, - } - - installed = { - "dev-libs/A-1": {"RDEPEND" : "dev-libs/B"}, - "dev-libs/B-1": {"RDEPEND" : "dev-libs/C"}, - "dev-libs/C-1": {}, - - "virtual/jre-1.5.0" : {"SLOT" : "1.5", "RDEPEND" : "|| ( =virtual/jdk-1.5.0* =dev-java/sun-jre-bin-1.5.0* )"}, - "virtual/jre-1.6.0" : {"SLOT" : "1.6", "RDEPEND" : "|| ( =virtual/jdk-1.6.0* =dev-java/sun-jre-bin-1.6.0* )"}, - "virtual/jdk-1.5.0" : {"SLOT" : "1.5", "RDEPEND" : "|| ( =dev-java/sun-jdk-1.5.0* dev-java/gcj-jdk )"}, - "virtual/jdk-1.6.0" : {"SLOT" : "1.6", "RDEPEND" : "|| ( =dev-java/icedtea-6* =dev-java/sun-jdk-1.6.0* )"}, - "dev-java/gcj-jdk-4.5" : {}, - "dev-java/icedtea-6.1" : {}, - - "virtual/libusb-0" : {"EAPI" :"2", "SLOT" : "0", "RDEPEND" : "|| ( >=dev-libs/libusb-0.1.12-r1:0 dev-libs/libusb-compat >=sys-freebsd/freebsd-lib-8.0[usb] )"}, - } - - world = ["dev-libs/A"] - - test_cases = ( - ResolverPlaygroundTestCase( - ["dev-libs/A"], - options = {"--update": True, "--deep": 0}, - success = True, - mergelist = ["dev-libs/A-2"]), - - ResolverPlaygroundTestCase( - ["dev-libs/A"], - options = {"--update": True, "--deep": 1}, - success = True, - mergelist = ["dev-libs/B-2", "dev-libs/A-2"]), - - ResolverPlaygroundTestCase( - ["dev-libs/A"], - options = {"--update": True, "--deep": 2}, - success = True, - mergelist = ["dev-libs/C-2", "dev-libs/B-2", "dev-libs/A-2"]), - - ResolverPlaygroundTestCase( - ["@world"], - options = {"--update": True, "--deep": True}, - success = True, - mergelist = ["dev-libs/C-2", "dev-libs/B-2", "dev-libs/A-2"]), - - ResolverPlaygroundTestCase( - ["@world"], - options = {"--emptytree": True}, - success = True, - mergelist = ["dev-libs/C-2", "dev-libs/B-2", "dev-libs/A-2"]), - - ResolverPlaygroundTestCase( - ["@world"], - options = {"--selective": True, "--deep": True}, - success = True, - mergelist = []), - - ResolverPlaygroundTestCase( - ["dev-libs/A"], - options = {"--deep": 2}, - success = True, - mergelist = ["dev-libs/A-2"]), - - ResolverPlaygroundTestCase( - ["virtual/jre"], - options = {}, - success = True, - mergelist = ['virtual/jre-1.6.0-r1']), - - ResolverPlaygroundTestCase( - ["virtual/jre"], - options = {"--deep" : True}, - success = True, - mergelist = ['virtual/jre-1.6.0-r1']), - - # Test bug #141118, where we avoid pulling in - # redundant deps, satisfying nested virtuals - # as efficiently as possible. - ResolverPlaygroundTestCase( - ["virtual/jre"], - options = {"--selective" : True, "--deep" : True}, - success = True, - mergelist = []), - - # Test bug #150361, where depgraph._greedy_slots() - # is triggered by --update with AtomArg. - ResolverPlaygroundTestCase( - ["virtual/jre"], - options = {"--update" : True}, - success = True, - ambiguous_merge_order = True, - mergelist = [('virtual/jre-1.6.0-r1', 'virtual/jre-1.5.0-r1')]), - - # Recursively traversed virtual dependencies, and their - # direct dependencies, are considered to have the same - # depth as direct dependencies. - ResolverPlaygroundTestCase( - ["virtual/jre"], - options = {"--update" : True, "--deep" : 1}, - success = True, - ambiguous_merge_order = True, - merge_order_assertions=(('dev-java/icedtea-6.1-r1', 'virtual/jdk-1.6.0-r1'), ('virtual/jdk-1.6.0-r1', 'virtual/jre-1.6.0-r1'), - ('dev-java/gcj-jdk-4.5-r1', 'virtual/jdk-1.5.0-r1'), ('virtual/jdk-1.5.0-r1', 'virtual/jre-1.5.0-r1')), - mergelist = [('dev-java/icedtea-6.1-r1', 'dev-java/gcj-jdk-4.5-r1', 'virtual/jdk-1.6.0-r1', 'virtual/jdk-1.5.0-r1', 'virtual/jre-1.6.0-r1', 'virtual/jre-1.5.0-r1')]), - - ResolverPlaygroundTestCase( - ["virtual/jre:1.5"], - options = {"--update" : True}, - success = True, - mergelist = ['virtual/jre-1.5.0-r1']), - - ResolverPlaygroundTestCase( - ["virtual/jre:1.6"], - options = {"--update" : True}, - success = True, - mergelist = ['virtual/jre-1.6.0-r1']), - - # Test that we don't pull in any unnecessary updates - # when --update is not specified, even though we - # specified --deep. - ResolverPlaygroundTestCase( - ["dev-java/ant-core"], - options = {"--deep" : True}, - success = True, - mergelist = ["dev-java/ant-core-1.8"]), - - ResolverPlaygroundTestCase( - ["dev-java/ant-core"], - options = {"--update" : True}, - success = True, - mergelist = ["dev-java/ant-core-1.8"]), - - # Recursively traversed virtual dependencies, and their - # direct dependencies, are considered to have the same - # depth as direct dependencies. - ResolverPlaygroundTestCase( - ["dev-java/ant-core"], - options = {"--update" : True, "--deep" : 1}, - success = True, - mergelist = ['dev-java/icedtea-6.1-r1', 'virtual/jdk-1.6.0-r1', 'dev-java/ant-core-1.8']), - - ResolverPlaygroundTestCase( - ["dev-db/hsqldb"], - options = {"--deep" : True}, - success = True, - mergelist = ["dev-db/hsqldb-1.8"]), - - # Don't traverse deps of an installed package with --deep=0, - # even if it's a virtual. - ResolverPlaygroundTestCase( - ["virtual/libusb:0"], - options = {"--selective" : True, "--deep" : 0}, - success = True, - mergelist = []), - - # Satisfy unsatisfied dep of installed package with --deep=1. - ResolverPlaygroundTestCase( - ["virtual/libusb:0"], - options = {"--selective" : True, "--deep" : 1}, - success = True, - mergelist = ['dev-libs/libusb-0.1.13']), - - # Pull in direct dep of virtual, even with --deep=0. - ResolverPlaygroundTestCase( - ["sys-fs/udev"], - options = {"--deep" : 0}, - success = True, - mergelist = ['dev-libs/libusb-0.1.13', 'sys-fs/udev-164']), - - # Test --nodeps with direct virtual deps. - ResolverPlaygroundTestCase( - ["sys-fs/udev"], - options = {"--nodeps" : True}, - success = True, - mergelist = ["sys-fs/udev-164"]), - - # Test that --nodeps overrides --deep. - ResolverPlaygroundTestCase( - ["sys-fs/udev"], - options = {"--nodeps" : True, "--deep" : True}, - success = True, - mergelist = ["sys-fs/udev-164"]), - - # Test that --nodeps overrides --emptytree. - ResolverPlaygroundTestCase( - ["sys-fs/udev"], - options = {"--nodeps" : True, "--emptytree" : True}, - success = True, - mergelist = ["sys-fs/udev-164"]), - - # Test --emptytree with virtuals. - ResolverPlaygroundTestCase( - ["sys-fs/udev"], - options = {"--emptytree" : True}, - success = True, - mergelist = ['dev-libs/libusb-0.1.13', 'virtual/libusb-0', 'sys-fs/udev-164']), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed, - world=world) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_eapi.py b/portage_with_autodep/pym/portage/tests/resolver/test_eapi.py deleted file mode 100644 index 525b585..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_eapi.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class EAPITestCase(TestCase): - - def testEAPI(self): - - ebuilds = { - #EAPI-1: IUSE-defaults - "dev-libs/A-1.0": { "EAPI": 0, "IUSE": "+foo" }, - "dev-libs/A-1.1": { "EAPI": 1, "IUSE": "+foo" }, - "dev-libs/A-1.2": { "EAPI": 2, "IUSE": "+foo" }, - "dev-libs/A-1.3": { "EAPI": 3, "IUSE": "+foo" }, - "dev-libs/A-1.4": { "EAPI": "4", "IUSE": "+foo" }, - - #EAPI-1: slot deps - "dev-libs/A-2.0": { "EAPI": 0, "DEPEND": "dev-libs/B:0" }, - "dev-libs/A-2.1": { "EAPI": 1, "DEPEND": "dev-libs/B:0" }, - "dev-libs/A-2.2": { "EAPI": 2, "DEPEND": "dev-libs/B:0" }, - "dev-libs/A-2.3": { "EAPI": 3, "DEPEND": "dev-libs/B:0" }, - "dev-libs/A-2.4": { "EAPI": "4", "DEPEND": "dev-libs/B:0" }, - - #EAPI-2: use deps - "dev-libs/A-3.0": { "EAPI": 0, "DEPEND": "dev-libs/B[foo]" }, - "dev-libs/A-3.1": { "EAPI": 1, "DEPEND": "dev-libs/B[foo]" }, - "dev-libs/A-3.2": { "EAPI": 2, "DEPEND": "dev-libs/B[foo]" }, - "dev-libs/A-3.3": { "EAPI": 3, "DEPEND": "dev-libs/B[foo]" }, - "dev-libs/A-3.4": { "EAPI": "4", "DEPEND": "dev-libs/B[foo]" }, - - #EAPI-2: strong blocks - "dev-libs/A-4.0": { "EAPI": 0, "DEPEND": "!!dev-libs/B" }, - "dev-libs/A-4.1": { "EAPI": 1, "DEPEND": "!!dev-libs/B" }, - "dev-libs/A-4.2": { "EAPI": 2, "DEPEND": "!!dev-libs/B" }, - "dev-libs/A-4.3": { "EAPI": 3, "DEPEND": "!!dev-libs/B" }, - "dev-libs/A-4.4": { "EAPI": "4", "DEPEND": "!!dev-libs/B" }, - - #EAPI-4: slot operator deps - #~ "dev-libs/A-5.0": { "EAPI": 0, "DEPEND": "dev-libs/B:*" }, - #~ "dev-libs/A-5.1": { "EAPI": 1, "DEPEND": "dev-libs/B:*" }, - #~ "dev-libs/A-5.2": { "EAPI": 2, "DEPEND": "dev-libs/B:*" }, - #~ "dev-libs/A-5.3": { "EAPI": 3, "DEPEND": "dev-libs/B:*" }, - #~ "dev-libs/A-5.4": { "EAPI": "4", "DEPEND": "dev-libs/B:*" }, - - #EAPI-4: use dep defaults - "dev-libs/A-6.0": { "EAPI": 0, "DEPEND": "dev-libs/B[bar(+)]" }, - "dev-libs/A-6.1": { "EAPI": 1, "DEPEND": "dev-libs/B[bar(+)]" }, - "dev-libs/A-6.2": { "EAPI": 2, "DEPEND": "dev-libs/B[bar(+)]" }, - "dev-libs/A-6.3": { "EAPI": 3, "DEPEND": "dev-libs/B[bar(+)]" }, - "dev-libs/A-6.4": { "EAPI": "4", "DEPEND": "dev-libs/B[bar(+)]" }, - - #EAPI-4: REQUIRED_USE - "dev-libs/A-7.0": { "EAPI": 0, "IUSE": "foo bar", "REQUIRED_USE": "|| ( foo bar )" }, - "dev-libs/A-7.1": { "EAPI": 1, "IUSE": "foo +bar", "REQUIRED_USE": "|| ( foo bar )" }, - "dev-libs/A-7.2": { "EAPI": 2, "IUSE": "foo +bar", "REQUIRED_USE": "|| ( foo bar )" }, - "dev-libs/A-7.3": { "EAPI": 3, "IUSE": "foo +bar", "REQUIRED_USE": "|| ( foo bar )" }, - "dev-libs/A-7.4": { "EAPI": "4", "IUSE": "foo +bar", "REQUIRED_USE": "|| ( foo bar )" }, - - "dev-libs/B-1": {"EAPI": 1, "IUSE": "+foo"}, - } - - test_cases = ( - ResolverPlaygroundTestCase(["=dev-libs/A-1.0"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-1.1"], success = True, mergelist = ["dev-libs/A-1.1"]), - ResolverPlaygroundTestCase(["=dev-libs/A-1.2"], success = True, mergelist = ["dev-libs/A-1.2"]), - ResolverPlaygroundTestCase(["=dev-libs/A-1.3"], success = True, mergelist = ["dev-libs/A-1.3"]), - ResolverPlaygroundTestCase(["=dev-libs/A-1.4"], success = True, mergelist = ["dev-libs/A-1.4"]), - - ResolverPlaygroundTestCase(["=dev-libs/A-2.0"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-2.1"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-2.1"]), - ResolverPlaygroundTestCase(["=dev-libs/A-2.2"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-2.2"]), - ResolverPlaygroundTestCase(["=dev-libs/A-2.3"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-2.3"]), - ResolverPlaygroundTestCase(["=dev-libs/A-2.4"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-2.4"]), - - ResolverPlaygroundTestCase(["=dev-libs/A-3.0"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-3.1"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-3.2"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-3.2"]), - ResolverPlaygroundTestCase(["=dev-libs/A-3.3"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-3.3"]), - ResolverPlaygroundTestCase(["=dev-libs/A-3.4"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-3.4"]), - - ResolverPlaygroundTestCase(["=dev-libs/A-4.0"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-4.1"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-4.2"], success = True, mergelist = ["dev-libs/A-4.2"]), - ResolverPlaygroundTestCase(["=dev-libs/A-4.3"], success = True, mergelist = ["dev-libs/A-4.3"]), - ResolverPlaygroundTestCase(["=dev-libs/A-4.4"], success = True, mergelist = ["dev-libs/A-4.4"]), - - ResolverPlaygroundTestCase(["=dev-libs/A-5.0"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-5.1"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-5.2"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-5.3"], success = False), - # not implemented: EAPI-4: slot operator deps - #~ ResolverPlaygroundTestCase(["=dev-libs/A-5.4"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-5.4"]), - - ResolverPlaygroundTestCase(["=dev-libs/A-6.0"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-6.1"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-6.2"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-6.3"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-6.4"], success = True, mergelist = ["dev-libs/B-1", "dev-libs/A-6.4"]), - - ResolverPlaygroundTestCase(["=dev-libs/A-7.0"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-7.1"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-7.2"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-7.3"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-7.4"], success = True, mergelist = ["dev-libs/A-7.4"]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_merge_order.py b/portage_with_autodep/pym/portage/tests/resolver/test_merge_order.py deleted file mode 100644 index 0a52c81..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_merge_order.py +++ /dev/null @@ -1,453 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import portage -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, - ResolverPlaygroundTestCase) - -class MergeOrderTestCase(TestCase): - - def testMergeOrder(self): - ebuilds = { - "app-misc/blocker-buildtime-a-1" : {}, - "app-misc/blocker-buildtime-unbuilt-a-1" : { - "DEPEND" : "!app-misc/installed-blocker-a", - }, - "app-misc/blocker-buildtime-unbuilt-hard-a-1" : { - "EAPI" : "2", - "DEPEND" : "!!app-misc/installed-blocker-a", - }, - "app-misc/blocker-update-order-a-1" : {}, - "app-misc/blocker-update-order-hard-a-1" : {}, - "app-misc/blocker-update-order-hard-unsolvable-a-1" : {}, - "app-misc/blocker-runtime-a-1" : {}, - "app-misc/blocker-runtime-b-1" : {}, - "app-misc/blocker-runtime-hard-a-1" : {}, - "app-misc/circ-buildtime-a-0": {}, - "app-misc/circ-buildtime-a-1": { - "RDEPEND": "app-misc/circ-buildtime-b", - }, - "app-misc/circ-buildtime-b-1": { - "RDEPEND": "app-misc/circ-buildtime-c", - }, - "app-misc/circ-buildtime-c-1": { - "DEPEND": "app-misc/circ-buildtime-a", - }, - "app-misc/circ-buildtime-unsolvable-a-1": { - "RDEPEND": "app-misc/circ-buildtime-unsolvable-b", - }, - "app-misc/circ-buildtime-unsolvable-b-1": { - "RDEPEND": "app-misc/circ-buildtime-unsolvable-c", - }, - "app-misc/circ-buildtime-unsolvable-c-1": { - "DEPEND": "app-misc/circ-buildtime-unsolvable-a", - }, - "app-misc/circ-post-runtime-a-1": { - "PDEPEND": "app-misc/circ-post-runtime-b", - }, - "app-misc/circ-post-runtime-b-1": { - "RDEPEND": "app-misc/circ-post-runtime-c", - }, - "app-misc/circ-post-runtime-c-1": { - "RDEPEND": "app-misc/circ-post-runtime-a", - }, - "app-misc/circ-runtime-a-1": { - "RDEPEND": "app-misc/circ-runtime-b", - }, - "app-misc/circ-runtime-b-1": { - "RDEPEND": "app-misc/circ-runtime-c", - }, - "app-misc/circ-runtime-c-1": { - "RDEPEND": "app-misc/circ-runtime-a", - }, - "app-misc/circ-satisfied-a-0": { - "RDEPEND": "app-misc/circ-satisfied-b", - }, - "app-misc/circ-satisfied-a-1": { - "RDEPEND": "app-misc/circ-satisfied-b", - }, - "app-misc/circ-satisfied-b-0": { - "RDEPEND": "app-misc/circ-satisfied-c", - }, - "app-misc/circ-satisfied-b-1": { - "RDEPEND": "app-misc/circ-satisfied-c", - }, - "app-misc/circ-satisfied-c-0": { - "DEPEND": "app-misc/circ-satisfied-a", - "RDEPEND": "app-misc/circ-satisfied-a", - }, - "app-misc/circ-satisfied-c-1": { - "DEPEND": "app-misc/circ-satisfied-a", - "RDEPEND": "app-misc/circ-satisfied-a", - }, - "app-misc/circ-smallest-a-1": { - "RDEPEND": "app-misc/circ-smallest-b", - }, - "app-misc/circ-smallest-b-1": { - "RDEPEND": "app-misc/circ-smallest-a", - }, - "app-misc/circ-smallest-c-1": { - "RDEPEND": "app-misc/circ-smallest-d", - }, - "app-misc/circ-smallest-d-1": { - "RDEPEND": "app-misc/circ-smallest-e", - }, - "app-misc/circ-smallest-e-1": { - "RDEPEND": "app-misc/circ-smallest-c", - }, - "app-misc/circ-smallest-f-1": { - "RDEPEND": "app-misc/circ-smallest-g app-misc/circ-smallest-a app-misc/circ-smallest-c", - }, - "app-misc/circ-smallest-g-1": { - "RDEPEND": "app-misc/circ-smallest-f", - }, - "app-misc/installed-blocker-a-1" : { - "EAPI" : "2", - "DEPEND" : "!app-misc/blocker-buildtime-a", - "RDEPEND" : "!app-misc/blocker-runtime-a !app-misc/blocker-runtime-b !!app-misc/blocker-runtime-hard-a", - }, - "app-misc/installed-old-version-blocks-a-1" : { - "RDEPEND" : "!app-misc/blocker-update-order-a", - }, - "app-misc/installed-old-version-blocks-a-2" : {}, - "app-misc/installed-old-version-blocks-hard-a-1" : { - "EAPI" : "2", - "RDEPEND" : "!!app-misc/blocker-update-order-hard-a", - }, - "app-misc/installed-old-version-blocks-hard-a-2" : {}, - "app-misc/installed-old-version-blocks-hard-unsolvable-a-1" : { - "EAPI" : "2", - "RDEPEND" : "!!app-misc/blocker-update-order-hard-unsolvable-a", - }, - "app-misc/installed-old-version-blocks-hard-unsolvable-a-2" : { - "DEPEND" : "app-misc/blocker-update-order-hard-unsolvable-a", - "RDEPEND" : "", - }, - "app-misc/some-app-a-1": { - "RDEPEND": "app-misc/circ-runtime-a app-misc/circ-runtime-b", - }, - "app-misc/some-app-b-1": { - "RDEPEND": "app-misc/circ-post-runtime-a app-misc/circ-post-runtime-b", - }, - "app-misc/some-app-c-1": { - "RDEPEND": "app-misc/circ-buildtime-a app-misc/circ-buildtime-b", - }, - "app-admin/eselect-python-20100321" : {}, - "sys-apps/portage-2.1.9.42" : { - "DEPEND" : "dev-lang/python", - "RDEPEND" : "dev-lang/python", - }, - "sys-apps/portage-2.1.9.49" : { - "DEPEND" : "dev-lang/python >=app-admin/eselect-python-20091230", - "RDEPEND" : "dev-lang/python", - }, - "dev-lang/python-3.1" : {}, - "dev-lang/python-3.2" : {}, - "virtual/libc-0" : { - "RDEPEND" : "sys-libs/glibc", - }, - "sys-devel/gcc-4.5.2" : {}, - "sys-devel/binutils-2.18" : {}, - "sys-devel/binutils-2.20.1" : {}, - "sys-libs/glibc-2.11" : { - "DEPEND" : "virtual/os-headers sys-devel/gcc sys-devel/binutils", - "RDEPEND": "", - }, - "sys-libs/glibc-2.13" : { - "DEPEND" : "virtual/os-headers sys-devel/gcc sys-devel/binutils", - "RDEPEND": "", - }, - "virtual/os-headers-0" : { - "RDEPEND" : "sys-kernel/linux-headers", - }, - "sys-kernel/linux-headers-2.6.38": { - "DEPEND" : "app-arch/xz-utils", - "RDEPEND": "", - }, - "sys-kernel/linux-headers-2.6.39": { - "DEPEND" : "app-arch/xz-utils", - "RDEPEND": "", - }, - "app-arch/xz-utils-5.0.1" : {}, - "app-arch/xz-utils-5.0.2" : {}, - "dev-util/pkgconfig-0.25-r2" : {}, - "kde-base/kdelibs-3.5.7" : { - "PDEPEND" : "kde-misc/kdnssd-avahi", - }, - "kde-misc/kdnssd-avahi-0.1.2" : { - "DEPEND" : "kde-base/kdelibs app-arch/xz-utils dev-util/pkgconfig", - "RDEPEND" : "kde-base/kdelibs", - }, - "kde-base/kdnssd-3.5.7" : { - "DEPEND" : "kde-base/kdelibs", - "RDEPEND" : "kde-base/kdelibs", - }, - "kde-base/libkdegames-3.5.7" : { - "DEPEND" : "kde-base/kdelibs", - "RDEPEND" : "kde-base/kdelibs", - }, - "kde-base/kmines-3.5.7" : { - "DEPEND" : "kde-base/libkdegames", - "RDEPEND" : "kde-base/libkdegames", - }, - "media-video/libav-0.7_pre20110327" : { - "EAPI" : "2", - "IUSE" : "X +encode", - "RDEPEND" : "!media-video/ffmpeg", - }, - "media-video/ffmpeg-0.7_rc1" : { - "EAPI" : "2", - "IUSE" : "X +encode", - }, - "virtual/ffmpeg-0.6.90" : { - "EAPI" : "2", - "IUSE" : "X +encode", - "RDEPEND" : "|| ( >=media-video/ffmpeg-0.6.90_rc0-r2[X=,encode=] >=media-video/libav-0.6.90_rc[X=,encode=] )", - }, - } - - installed = { - "app-misc/circ-buildtime-a-0": {}, - "app-misc/circ-satisfied-a-0": { - "RDEPEND": "app-misc/circ-satisfied-b", - }, - "app-misc/circ-satisfied-b-0": { - "RDEPEND": "app-misc/circ-satisfied-c", - }, - "app-misc/circ-satisfied-c-0": { - "DEPEND": "app-misc/circ-satisfied-a", - "RDEPEND": "app-misc/circ-satisfied-a", - }, - "app-misc/installed-blocker-a-1" : { - "EAPI" : "2", - "DEPEND" : "!app-misc/blocker-buildtime-a", - "RDEPEND" : "!app-misc/blocker-runtime-a !app-misc/blocker-runtime-b !!app-misc/blocker-runtime-hard-a", - }, - "app-misc/installed-old-version-blocks-a-1" : { - "RDEPEND" : "!app-misc/blocker-update-order-a", - }, - "app-misc/installed-old-version-blocks-hard-a-1" : { - "EAPI" : "2", - "RDEPEND" : "!!app-misc/blocker-update-order-hard-a", - }, - "app-misc/installed-old-version-blocks-hard-unsolvable-a-1" : { - "EAPI" : "2", - "RDEPEND" : "!!app-misc/blocker-update-order-hard-unsolvable-a", - }, - "sys-apps/portage-2.1.9.42" : { - "DEPEND" : "dev-lang/python", - "RDEPEND" : "dev-lang/python", - }, - "dev-lang/python-3.1" : {}, - "virtual/libc-0" : { - "RDEPEND" : "sys-libs/glibc", - }, - "sys-devel/binutils-2.18" : {}, - "sys-libs/glibc-2.11" : { - "DEPEND" : "virtual/os-headers sys-devel/gcc sys-devel/binutils", - "RDEPEND": "", - }, - "virtual/os-headers-0" : { - "RDEPEND" : "sys-kernel/linux-headers", - }, - "sys-kernel/linux-headers-2.6.38": { - "DEPEND" : "app-arch/xz-utils", - "RDEPEND": "", - }, - "app-arch/xz-utils-5.0.1" : {}, - "media-video/ffmpeg-0.7_rc1" : { - "EAPI" : "2", - "IUSE" : "X +encode", - "USE" : "encode", - }, - "virtual/ffmpeg-0.6.90" : { - "EAPI" : "2", - "IUSE" : "X +encode", - "USE" : "encode", - "RDEPEND" : "|| ( >=media-video/ffmpeg-0.6.90_rc0-r2[X=,encode=] >=media-video/libav-0.6.90_rc[X=,encode=] )", - }, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["app-misc/some-app-a"], - success = True, - ambiguous_merge_order = True, - mergelist = [("app-misc/circ-runtime-a-1", "app-misc/circ-runtime-b-1", "app-misc/circ-runtime-c-1"), "app-misc/some-app-a-1"]), - ResolverPlaygroundTestCase( - ["app-misc/some-app-a"], - success = True, - ambiguous_merge_order = True, - mergelist = [("app-misc/circ-runtime-c-1", "app-misc/circ-runtime-b-1", "app-misc/circ-runtime-a-1"), "app-misc/some-app-a-1"]), - # Test unsolvable circular dep that is RDEPEND in one - # direction and DEPEND in the other. - ResolverPlaygroundTestCase( - ["app-misc/circ-buildtime-unsolvable-a"], - success = False, - circular_dependency_solutions = {}), - # Test optimal merge order for a circular dep that is - # RDEPEND in one direction and DEPEND in the other. - # This requires an installed instance of the DEPEND - # package in order to be solvable. - ResolverPlaygroundTestCase( - ["app-misc/some-app-c", "app-misc/circ-buildtime-a"], - success = True, - ambiguous_merge_order = True, - mergelist = [("app-misc/circ-buildtime-b-1", "app-misc/circ-buildtime-c-1"), "app-misc/circ-buildtime-a-1", "app-misc/some-app-c-1"]), - # Test optimal merge order for a circular dep that is - # RDEPEND in one direction and PDEPEND in the other. - ResolverPlaygroundTestCase( - ["app-misc/some-app-b"], - success = True, - ambiguous_merge_order = True, - mergelist = ["app-misc/circ-post-runtime-a-1", ("app-misc/circ-post-runtime-b-1", "app-misc/circ-post-runtime-c-1"), "app-misc/some-app-b-1"]), - # Test optimal merge order for a circular dep that is - # RDEPEND in one direction and DEPEND in the other, - # with all dependencies initially satisfied. Optimally, - # the DEPEND/buildtime dep should be updated before the - # package that depends on it, even though it's feasible - # to update it later since it is already satisfied. - ResolverPlaygroundTestCase( - ["app-misc/circ-satisfied-a", "app-misc/circ-satisfied-b", "app-misc/circ-satisfied-c"], - success = True, - all_permutations = True, - ambiguous_merge_order = True, - merge_order_assertions = (("app-misc/circ-satisfied-a-1", "app-misc/circ-satisfied-c-1"),), - mergelist = [("app-misc/circ-satisfied-a-1", "app-misc/circ-satisfied-b-1", "app-misc/circ-satisfied-c-1")]), - # In the case of multiple runtime cycles, where some cycles - # may depend on smaller independent cycles, it's optimal - # to merge smaller independent cycles before other cycles - # that depend on them. - ResolverPlaygroundTestCase( - ["app-misc/circ-smallest-a", "app-misc/circ-smallest-c", "app-misc/circ-smallest-f"], - success = True, - ambiguous_merge_order = True, - all_permutations = True, - mergelist = [('app-misc/circ-smallest-a-1', 'app-misc/circ-smallest-b-1'), - ('app-misc/circ-smallest-c-1', 'app-misc/circ-smallest-d-1', 'app-misc/circ-smallest-e-1'), - ('app-misc/circ-smallest-f-1', 'app-misc/circ-smallest-g-1')]), - # installed package has buildtime-only blocker - # that should be ignored - ResolverPlaygroundTestCase( - ["app-misc/blocker-buildtime-a"], - success = True, - mergelist = ["app-misc/blocker-buildtime-a-1"]), - # We're installing a package that an old version of - # an installed package blocks. However, an update is - # available to the old package. The old package should - # be updated first, in order to solve the blocker without - # any need for blocking packages to temporarily overlap. - ResolverPlaygroundTestCase( - ["app-misc/blocker-update-order-a", "app-misc/installed-old-version-blocks-a"], - success = True, - all_permutations = True, - mergelist = ["app-misc/installed-old-version-blocks-a-2", "app-misc/blocker-update-order-a-1"]), - # This is the same as above but with a hard blocker. The hard - # blocker is solved automatically since the update makes it - # irrelevant. - ResolverPlaygroundTestCase( - ["app-misc/blocker-update-order-hard-a", "app-misc/installed-old-version-blocks-hard-a"], - success = True, - all_permutations = True, - mergelist = ["app-misc/installed-old-version-blocks-hard-a-2", "app-misc/blocker-update-order-hard-a-1"]), - # This is similar to the above case except that it's unsolvable - # due to merge order, unless bug 250286 is implemented so that - # the installed blocker will be unmerged before installation - # of the package it blocks (rather than after like a soft blocker - # would be handled). The "unmerge before" behavior requested - # in bug 250286 must be optional since essential programs or - # libraries may be temporarily unavailable during a - # non-overlapping update like this. - ResolverPlaygroundTestCase( - ["app-misc/blocker-update-order-hard-unsolvable-a", "app-misc/installed-old-version-blocks-hard-unsolvable-a"], - success = False, - all_permutations = True, - ambiguous_merge_order = True, - merge_order_assertions = (('app-misc/blocker-update-order-hard-unsolvable-a-1', 'app-misc/installed-old-version-blocks-hard-unsolvable-a-2'),), - mergelist = [('app-misc/blocker-update-order-hard-unsolvable-a-1', 'app-misc/installed-old-version-blocks-hard-unsolvable-a-2', '!!app-misc/blocker-update-order-hard-unsolvable-a')]), - # The installed package has runtime blockers that - # should cause it to be uninstalled. The uninstall - # task is executed only after blocking packages have - # been merged. - # TODO: distinguish between install/uninstall tasks in mergelist - ResolverPlaygroundTestCase( - ["app-misc/blocker-runtime-a", "app-misc/blocker-runtime-b"], - success = True, - all_permutations = True, - ambiguous_merge_order = True, - mergelist = [("app-misc/blocker-runtime-a-1", "app-misc/blocker-runtime-b-1"), "app-misc/installed-blocker-a-1", ("!app-misc/blocker-runtime-a", "!app-misc/blocker-runtime-b")]), - # We have a soft buildtime blocker against an installed - # package that should cause it to be uninstalled. Note that with - # soft blockers, the blocking packages are allowed to temporarily - # overlap. This allows any essential programs/libraries provided - # by both packages to be available at all times. - # TODO: distinguish between install/uninstall tasks in mergelist - ResolverPlaygroundTestCase( - ["app-misc/blocker-buildtime-unbuilt-a"], - success = True, - mergelist = ["app-misc/blocker-buildtime-unbuilt-a-1", "app-misc/installed-blocker-a-1", "!app-misc/installed-blocker-a"]), - # We have a hard buildtime blocker against an installed - # package that will not resolve automatically (unless - # the option requested in bug 250286 is implemented). - ResolverPlaygroundTestCase( - ["app-misc/blocker-buildtime-unbuilt-hard-a"], - success = False, - mergelist = ['app-misc/blocker-buildtime-unbuilt-hard-a-1', '!!app-misc/installed-blocker-a']), - # An installed package has a hard runtime blocker that - # will not resolve automatically (unless the option - # requested in bug 250286 is implemented). - ResolverPlaygroundTestCase( - ["app-misc/blocker-runtime-hard-a"], - success = False, - mergelist = ['app-misc/blocker-runtime-hard-a-1', '!!app-misc/blocker-runtime-hard-a']), - # Test swapping of providers for a new-style virtual package, - # which relies on delayed evaluation of disjunctive (virtual - # and ||) deps as required to solve bug #264434. Note that - # this behavior is not supported for old-style PROVIDE virtuals, - # as reported in bug #339164. - ResolverPlaygroundTestCase( - ["media-video/libav"], - success=True, - mergelist = ['media-video/libav-0.7_pre20110327', 'media-video/ffmpeg-0.7_rc1', '!media-video/ffmpeg']), - # Test that PORTAGE_PACKAGE_ATOM is merged asap. Optimally, - # satisfied deps are always merged after the asap nodes that - # depend on them. - ResolverPlaygroundTestCase( - ["dev-lang/python", portage.const.PORTAGE_PACKAGE_ATOM], - success = True, - all_permutations = True, - mergelist = ['app-admin/eselect-python-20100321', 'sys-apps/portage-2.1.9.49', 'dev-lang/python-3.2']), - # Test that OS_HEADERS_PACKAGE_ATOM and LIBC_PACKAGE_ATOM - # are merged asap, in order to account for implicit - # dependencies. See bug #303567. Optimally, satisfied deps - # are always merged after the asap nodes that depend on them. - ResolverPlaygroundTestCase( - ["app-arch/xz-utils", "sys-kernel/linux-headers", "sys-devel/binutils", "sys-libs/glibc"], - options = {"--complete-graph" : True}, - success = True, - all_permutations = True, - ambiguous_merge_order = True, - mergelist = ['sys-kernel/linux-headers-2.6.39', 'sys-devel/gcc-4.5.2', 'sys-libs/glibc-2.13', ('app-arch/xz-utils-5.0.2', 'sys-devel/binutils-2.20.1')]), - # Test asap install of PDEPEND for bug #180045. - ResolverPlaygroundTestCase( - ["kde-base/kmines", "kde-base/kdnssd", "kde-base/kdelibs", "app-arch/xz-utils"], - success = True, - all_permutations = True, - ambiguous_merge_order = True, - merge_order_assertions = ( - ('dev-util/pkgconfig-0.25-r2', 'kde-misc/kdnssd-avahi-0.1.2'), - ('kde-misc/kdnssd-avahi-0.1.2', 'kde-base/libkdegames-3.5.7'), - ('kde-misc/kdnssd-avahi-0.1.2', 'kde-base/kdnssd-3.5.7'), - ('kde-base/libkdegames-3.5.7', 'kde-base/kmines-3.5.7'), - ), - mergelist = [('kde-base/kdelibs-3.5.7', 'dev-util/pkgconfig-0.25-r2', 'kde-misc/kdnssd-avahi-0.1.2', 'app-arch/xz-utils-5.0.2', 'kde-base/libkdegames-3.5.7', 'kde-base/kdnssd-3.5.7', 'kde-base/kmines-3.5.7')]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_missing_iuse_and_evaluated_atoms.py b/portage_with_autodep/pym/portage/tests/resolver/test_missing_iuse_and_evaluated_atoms.py deleted file mode 100644 index a860e7b..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_missing_iuse_and_evaluated_atoms.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class MissingIUSEandEvaluatedAtomsTestCase(TestCase): - - def testMissingIUSEandEvaluatedAtoms(self): - ebuilds = { - "dev-libs/A-1": { "DEPEND": "dev-libs/B[foo?]", "IUSE": "foo bar", "EAPI": 2 }, - "dev-libs/A-2": { "DEPEND": "dev-libs/B[foo?,bar]", "IUSE": "foo bar", "EAPI": 2 }, - "dev-libs/B-1": { "IUSE": "bar" }, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["=dev-libs/A-1"], - success = False), - ResolverPlaygroundTestCase( - ["=dev-libs/A-2"], - success = False), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, debug=False) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_multirepo.py b/portage_with_autodep/pym/portage/tests/resolver/test_multirepo.py deleted file mode 100644 index 34c6d45..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_multirepo.py +++ /dev/null @@ -1,318 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class MultirepoTestCase(TestCase): - - def testMultirepo(self): - ebuilds = { - #Simple repo selection - "dev-libs/A-1": { }, - "dev-libs/A-1::repo1": { }, - "dev-libs/A-2::repo1": { }, - "dev-libs/A-1::repo2": { }, - - #Packages in exactly one repo - "dev-libs/B-1": { }, - "dev-libs/C-1::repo1": { }, - - #Package in repository 1 and 2, but 1 must be used - "dev-libs/D-1::repo1": { }, - "dev-libs/D-1::repo2": { }, - - "dev-libs/E-1": { }, - "dev-libs/E-1::repo1": { }, - "dev-libs/E-1::repo2": { "SLOT": "1" }, - - "dev-libs/F-1::repo1": { "SLOT": "1" }, - "dev-libs/F-1::repo2": { "SLOT": "1" }, - - "dev-libs/G-1::repo1": { "EAPI" : "4", "IUSE":"+x +y", "REQUIRED_USE" : "" }, - "dev-libs/G-1::repo2": { "EAPI" : "4", "IUSE":"+x +y", "REQUIRED_USE" : "^^ ( x y )" }, - - "dev-libs/H-1": { "KEYWORDS": "x86", "EAPI" : "3", - "RDEPEND" : "|| ( dev-libs/I:2 dev-libs/I:1 )" }, - - "dev-libs/I-1::repo2": { "SLOT" : "1"}, - "dev-libs/I-2::repo2": { "SLOT" : "2"}, - } - - installed = { - "dev-libs/H-1": { "RDEPEND" : "|| ( dev-libs/I:2 dev-libs/I:1 )"}, - "dev-libs/I-2::repo1": {"SLOT" : "2"}, - } - - sets = { - "multirepotest": - ( "dev-libs/A::test_repo", ) - } - - test_cases = ( - #Simple repo selection - ResolverPlaygroundTestCase( - ["dev-libs/A"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/A-2::repo1"]), - ResolverPlaygroundTestCase( - ["dev-libs/A::test_repo"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/A-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/A::repo2"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/A-1::repo2"]), - ResolverPlaygroundTestCase( - ["=dev-libs/A-1::repo1"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/A-1::repo1"]), - ResolverPlaygroundTestCase( - ["@multirepotest"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/A-1"]), - - #Packages in exactly one repo - ResolverPlaygroundTestCase( - ["dev-libs/B"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/B-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/C"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/C-1::repo1"]), - - #Package in repository 1 and 2, but 2 must be used - ResolverPlaygroundTestCase( - ["dev-libs/D"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/D-1::repo2"]), - - #Atoms with slots - ResolverPlaygroundTestCase( - ["dev-libs/E"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/E-1::repo2"]), - ResolverPlaygroundTestCase( - ["dev-libs/E:1::repo2"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/E-1::repo2"]), - ResolverPlaygroundTestCase( - ["dev-libs/E:1"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/E-1::repo2"]), - ResolverPlaygroundTestCase( - ["dev-libs/F:1"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/F-1::repo2"]), - ResolverPlaygroundTestCase( - ["=dev-libs/F-1:1"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/F-1::repo2"]), - ResolverPlaygroundTestCase( - ["=dev-libs/F-1:1::repo1"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/F-1::repo1"]), - - # Dependency on installed dev-libs/C-2 ebuild for which ebuild is - # not available from the same repo should not unnecessarily - # reinstall the same version from a different repo. - ResolverPlaygroundTestCase( - ["dev-libs/H"], - options = {"--update": True, "--deep": True}, - success = True, - mergelist = []), - - # Check interaction between repo priority and unsatisfied - # REQUIRED_USE, for bug #350254. - ResolverPlaygroundTestCase( - ["=dev-libs/G-1"], - check_repo_names = True, - success = False), - - ) - - playground = ResolverPlayground(ebuilds=ebuilds, - installed=installed, sets=sets) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - - - def testMultirepoUserConfig(self): - ebuilds = { - #package.use test - "dev-libs/A-1": { "IUSE": "foo" }, - "dev-libs/A-2::repo1": { "IUSE": "foo" }, - "dev-libs/A-3::repo2": { }, - "dev-libs/B-1": { "DEPEND": "dev-libs/A", "EAPI": 2 }, - "dev-libs/B-2": { "DEPEND": "dev-libs/A[foo]", "EAPI": 2 }, - "dev-libs/B-3": { "DEPEND": "dev-libs/A[-foo]", "EAPI": 2 }, - - #package.keywords test - "dev-libs/C-1": { "KEYWORDS": "~x86" }, - "dev-libs/C-1::repo1": { "KEYWORDS": "~x86" }, - - #package.license - "dev-libs/D-1": { "LICENSE": "TEST" }, - "dev-libs/D-1::repo1": { "LICENSE": "TEST" }, - - #package.mask - "dev-libs/E-1": { }, - "dev-libs/E-1::repo1": { }, - "dev-libs/H-1": { }, - "dev-libs/H-1::repo1": { }, - "dev-libs/I-1::repo2": { "SLOT" : "1"}, - "dev-libs/I-2::repo2": { "SLOT" : "2"}, - "dev-libs/J-1": { "KEYWORDS": "x86", "EAPI" : "3", - "RDEPEND" : "|| ( dev-libs/I:2 dev-libs/I:1 )" }, - - #package.properties - "dev-libs/F-1": { "PROPERTIES": "bar"}, - "dev-libs/F-1::repo1": { "PROPERTIES": "bar"}, - - #package.unmask - "dev-libs/G-1": { }, - "dev-libs/G-1::repo1": { }, - - #package.mask with wildcards - "dev-libs/Z-1::repo3": { }, - } - - installed = { - "dev-libs/J-1": { "RDEPEND" : "|| ( dev-libs/I:2 dev-libs/I:1 )"}, - "dev-libs/I-2::repo1": {"SLOT" : "2"}, - } - - user_config = { - "package.use": - ( - "dev-libs/A::repo1 foo", - ), - "package.keywords": - ( - "=dev-libs/C-1::test_repo", - ), - "package.license": - ( - "=dev-libs/D-1::test_repo TEST", - ), - "package.mask": - ( - "dev-libs/E::repo1", - "dev-libs/H", - "dev-libs/I::repo1", - #needed for package.unmask test - "dev-libs/G", - #wildcard test - "*/*::repo3", - ), - "package.properties": - ( - "dev-libs/F::repo1 -bar", - ), - "package.unmask": - ( - "dev-libs/G::test_repo", - ), - } - - test_cases = ( - #package.use test - ResolverPlaygroundTestCase( - ["=dev-libs/B-1"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/A-3::repo2", "dev-libs/B-1"]), - ResolverPlaygroundTestCase( - ["=dev-libs/B-2"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/A-2::repo1", "dev-libs/B-2"]), - ResolverPlaygroundTestCase( - ["=dev-libs/B-3"], - options = { "--autounmask": 'n' }, - success = False, - check_repo_names = True), - - #package.keywords test - ResolverPlaygroundTestCase( - ["dev-libs/C"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/C-1"]), - - #package.license test - ResolverPlaygroundTestCase( - ["dev-libs/D"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/D-1"]), - - #package.mask test - ResolverPlaygroundTestCase( - ["dev-libs/E"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/E-1"]), - - # Dependency on installed dev-libs/C-2 ebuild for which ebuild is - # masked from the same repo should not unnecessarily pull - # in a different slot. It should just pull in the same slot from - # a different repo (bug #351828). - ResolverPlaygroundTestCase( - ["dev-libs/J"], - options = {"--update": True, "--deep": True}, - success = True, - mergelist = ["dev-libs/I-2"]), - - #package.properties test - ResolverPlaygroundTestCase( - ["dev-libs/F"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/F-1"]), - - #package.mask test - ResolverPlaygroundTestCase( - ["dev-libs/G"], - success = True, - check_repo_names = True, - mergelist = ["dev-libs/G-1"]), - ResolverPlaygroundTestCase( - ["dev-libs/H"], - options = { "--autounmask": 'n' }, - success = False), - - #package.mask with wildcards - ResolverPlaygroundTestCase( - ["dev-libs/Z"], - options = { "--autounmask": 'n' }, - success = False), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, - installed=installed, user_config=user_config) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_multislot.py b/portage_with_autodep/pym/portage/tests/resolver/test_multislot.py deleted file mode 100644 index 8615419..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_multislot.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class MultSlotTestCase(TestCase): - - def testMultiSlotSelective(self): - """ - Test that a package isn't reinstalled due to SLOT dependency - interaction with USE=multislot (bug #220341). - """ - - ebuilds = { - "sys-devel/gcc-4.4.4": { "SLOT": "4.4" }, - } - - installed = { - "sys-devel/gcc-4.4.4": { "SLOT": "i686-pc-linux-gnu-4.4.4" }, - } - - options = {'--update' : True, '--deep' : True, '--selective' : True} - - test_cases = ( - ResolverPlaygroundTestCase( - ["sys-devel/gcc:4.4"], - options = options, - mergelist = [], - success = True), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed) - - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_old_dep_chain_display.py b/portage_with_autodep/pym/portage/tests/resolver/test_old_dep_chain_display.py deleted file mode 100644 index 8aedf59..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_old_dep_chain_display.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class OldDepChainDisplayTestCase(TestCase): - - def testOldDepChainDisplay(self): - ebuilds = { - "dev-libs/A-1": { "DEPEND": "foo? ( dev-libs/B[-bar] )", "IUSE": "+foo", "EAPI": "2" }, - "dev-libs/A-2": { "DEPEND": "foo? ( dev-libs/C )", "IUSE": "+foo", "EAPI": "1" }, - "dev-libs/B-1": { "IUSE": "bar", "DEPEND": "!bar? ( dev-libs/D[-baz] )", "EAPI": "2" }, - "dev-libs/C-1": { "KEYWORDS": "~x86" }, - "dev-libs/D-1": { "IUSE": "+baz", "EAPI": "1" }, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["=dev-libs/A-1"], - options = { "--autounmask": 'n' }, - success = False), - ResolverPlaygroundTestCase( - ["=dev-libs/A-2"], - options = { "--autounmask": 'n' }, - success = False), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_output.py b/portage_with_autodep/pym/portage/tests/resolver/test_output.py deleted file mode 100644 index 34efe9c..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_output.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class MergelistOutputTestCase(TestCase): - - def testMergelistOutput(self): - """ - This test doesn't check if the output is correct, but makes sure - that we don't backtrace somewhere in the output code. - """ - ebuilds = { - "dev-libs/A-1": { "DEPEND": "dev-libs/B dev-libs/C", "IUSE": "+foo", "EAPI": 1 }, - "dev-libs/B-1": { "DEPEND": "dev-libs/D", "IUSE": "foo +bar", "EAPI": 1 }, - "dev-libs/C-1": { "DEPEND": "dev-libs/E", "IUSE": "foo bar" }, - "dev-libs/D-1": { "IUSE": "" }, - "dev-libs/E-1": {}, - - #reinstall for flags - "dev-libs/Z-1": { "IUSE": "+foo", "EAPI": 1 }, - "dev-libs/Y-1": { "IUSE": "foo", "EAPI": 1 }, - "dev-libs/X-1": {}, - "dev-libs/W-1": { "IUSE": "+foo", "EAPI": 1 }, - } - - installed = { - "dev-libs/Z-1": { "USE": "", "IUSE": "foo" }, - "dev-libs/Y-1": { "USE": "foo", "IUSE": "+foo", "EAPI": 1 }, - "dev-libs/X-1": { "USE": "foo", "IUSE": "+foo", "EAPI": 1 }, - "dev-libs/W-1": { }, - } - - option_cobos = ( - (), - ("verbose",), - ("tree",), - ("tree", "unordered-display",), - ("verbose",), - ("verbose", "tree",), - ("verbose", "tree", "unordered-display",), - ) - - test_cases = [] - for options in option_cobos: - testcase_opts = {} - for opt in options: - testcase_opts["--" + opt] = True - - test_cases.append(ResolverPlaygroundTestCase( - ["dev-libs/A"], - options = testcase_opts, - success = True, - ignore_mergelist_order=True, - mergelist = ["dev-libs/D-1", "dev-libs/E-1", "dev-libs/C-1", "dev-libs/B-1", "dev-libs/A-1"])) - - test_cases.append(ResolverPlaygroundTestCase( - ["dev-libs/Z"], - options = testcase_opts, - success = True, - mergelist = ["dev-libs/Z-1"])) - - test_cases.append(ResolverPlaygroundTestCase( - ["dev-libs/Y"], - options = testcase_opts, - success = True, - mergelist = ["dev-libs/Y-1"])) - - test_cases.append(ResolverPlaygroundTestCase( - ["dev-libs/X"], - options = testcase_opts, - success = True, - mergelist = ["dev-libs/X-1"])) - - test_cases.append(ResolverPlaygroundTestCase( - ["dev-libs/W"], - options = testcase_opts, - success = True, - mergelist = ["dev-libs/W-1"])) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_rebuild.py b/portage_with_autodep/pym/portage/tests/resolver/test_rebuild.py deleted file mode 100644 index b9c4d6d..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_rebuild.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import (ResolverPlayground, - ResolverPlaygroundTestCase) - -class RebuildTestCase(TestCase): - - def testRebuild(self): - """ - Rebuild packages when dependencies that are used at both build-time and - run-time are upgraded. - """ - - ebuilds = { - "sys-libs/x-1": { }, - "sys-libs/x-1-r1": { }, - "sys-libs/x-2": { }, - "sys-apps/a-1": { "DEPEND" : "sys-libs/x", "RDEPEND" : "sys-libs/x"}, - "sys-apps/a-2": { "DEPEND" : "sys-libs/x", "RDEPEND" : "sys-libs/x"}, - "sys-apps/b-1": { "DEPEND" : "sys-libs/x", "RDEPEND" : "sys-libs/x"}, - "sys-apps/b-2": { "DEPEND" : "sys-libs/x", "RDEPEND" : "sys-libs/x"}, - "sys-apps/c-1": { "DEPEND" : "sys-libs/x", "RDEPEND" : ""}, - "sys-apps/c-2": { "DEPEND" : "sys-libs/x", "RDEPEND" : ""}, - "sys-apps/d-1": { "RDEPEND" : "sys-libs/x"}, - "sys-apps/d-2": { "RDEPEND" : "sys-libs/x"}, - "sys-apps/e-2": { "DEPEND" : "sys-libs/x", "RDEPEND" : "sys-libs/x"}, - "sys-apps/f-2": { "DEPEND" : "sys-apps/a", "RDEPEND" : "sys-apps/a"}, - "sys-apps/g-2": { "DEPEND" : "sys-apps/b sys-libs/x", - "RDEPEND" : "sys-apps/b"}, - } - - installed = { - "sys-libs/x-1": { }, - "sys-apps/a-1": { "DEPEND" : "sys-libs/x", "RDEPEND" : "sys-libs/x"}, - "sys-apps/b-1": { "DEPEND" : "sys-libs/x", "RDEPEND" : "sys-libs/x"}, - "sys-apps/c-1": { "DEPEND" : "sys-libs/x", "RDEPEND" : ""}, - "sys-apps/d-1": { "RDEPEND" : "sys-libs/x"}, - "sys-apps/e-1": { "DEPEND" : "sys-libs/x", "RDEPEND" : "sys-libs/x"}, - "sys-apps/f-1": { "DEPEND" : "sys-apps/a", "RDEPEND" : "sys-apps/a"}, - "sys-apps/g-1": { "DEPEND" : "sys-apps/b sys-libs/x", - "RDEPEND" : "sys-apps/b"}, - } - - world = ["sys-apps/a", "sys-apps/b", "sys-apps/c", "sys-apps/d", - "sys-apps/e", "sys-apps/f", "sys-apps/g"] - - test_cases = ( - ResolverPlaygroundTestCase( - ["sys-libs/x"], - options = {"--rebuild-if-unbuilt" : True, - "--rebuild-exclude" : ["sys-apps/b"]}, - mergelist = ['sys-libs/x-2', 'sys-apps/a-2', 'sys-apps/e-2'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["sys-libs/x"], - options = {"--rebuild-if-unbuilt" : True}, - mergelist = ['sys-libs/x-2', 'sys-apps/a-2', 'sys-apps/b-2', - 'sys-apps/e-2', 'sys-apps/g-2'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["sys-libs/x"], - options = {"--rebuild-if-unbuilt" : True, - "--rebuild-ignore" : ["sys-libs/x"]}, - mergelist = ['sys-libs/x-2'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["sys-libs/x"], - options = {"--rebuild-if-unbuilt" : True, - "--rebuild-ignore" : ["sys-apps/b"]}, - mergelist = ['sys-libs/x-2', 'sys-apps/a-2', 'sys-apps/b-2', - 'sys-apps/e-2'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["=sys-libs/x-1-r1"], - options = {"--rebuild-if-unbuilt" : True}, - mergelist = ['sys-libs/x-1-r1', 'sys-apps/a-2', - 'sys-apps/b-2', 'sys-apps/e-2', 'sys-apps/g-2'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["=sys-libs/x-1-r1"], - options = {"--rebuild-if-new-rev" : True}, - mergelist = ['sys-libs/x-1-r1', 'sys-apps/a-2', - 'sys-apps/b-2', 'sys-apps/e-2', 'sys-apps/g-2'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["=sys-libs/x-1-r1"], - options = {"--rebuild-if-new-ver" : True}, - mergelist = ['sys-libs/x-1-r1'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["sys-libs/x"], - options = {"--rebuild-if-new-ver" : True}, - mergelist = ['sys-libs/x-2', 'sys-apps/a-2', - 'sys-apps/b-2', 'sys-apps/e-2', 'sys-apps/g-2'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["=sys-libs/x-1"], - options = {"--rebuild-if-new-rev" : True}, - mergelist = ['sys-libs/x-1'], - ignore_mergelist_order = True, - success = True), - - ResolverPlaygroundTestCase( - ["=sys-libs/x-1"], - options = {"--rebuild-if-unbuilt" : True}, - mergelist = ['sys-libs/x-1', 'sys-apps/a-2', - 'sys-apps/b-2', 'sys-apps/e-2', 'sys-apps/g-2'], - ignore_mergelist_order = True, - success = True), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, - installed=installed, world=world) - - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_required_use.py b/portage_with_autodep/pym/portage/tests/resolver/test_required_use.py deleted file mode 100644 index c8810fa..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_required_use.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class RequiredUSETestCase(TestCase): - - def testRequiredUSE(self): - """ - Only simple REQUIRED_USE values here. The parser is tested under in dep/testCheckRequiredUse - """ - - ebuilds = { - "dev-libs/A-1" : {"EAPI": "4", "IUSE": "foo bar", "REQUIRED_USE": "|| ( foo bar )"}, - "dev-libs/A-2" : {"EAPI": "4", "IUSE": "foo +bar", "REQUIRED_USE": "|| ( foo bar )"}, - "dev-libs/A-3" : {"EAPI": "4", "IUSE": "+foo bar", "REQUIRED_USE": "|| ( foo bar )"}, - "dev-libs/A-4" : {"EAPI": "4", "IUSE": "+foo +bar", "REQUIRED_USE": "|| ( foo bar )"}, - "dev-libs/A-5" : {"EAPI": "4", "IUSE": "+foo +bar", "REQUIRED_USE": "|| ( )"}, - - "dev-libs/B-1" : {"EAPI": "4", "IUSE": "foo bar", "REQUIRED_USE": "^^ ( foo bar )"}, - "dev-libs/B-2" : {"EAPI": "4", "IUSE": "foo +bar", "REQUIRED_USE": "^^ ( foo bar )"}, - "dev-libs/B-3" : {"EAPI": "4", "IUSE": "+foo bar", "REQUIRED_USE": "^^ ( foo bar )"}, - "dev-libs/B-4" : {"EAPI": "4", "IUSE": "+foo +bar", "REQUIRED_USE": "^^ ( foo bar )"}, - "dev-libs/B-5" : {"EAPI": "4", "IUSE": "+foo +bar", "REQUIRED_USE": "^^ ( )"}, - - "dev-libs/C-1" : {"EAPI": "4", "IUSE": "+foo bar", "REQUIRED_USE": "foo? ( !bar )"}, - "dev-libs/C-2" : {"EAPI": "4", "IUSE": "+foo +bar", "REQUIRED_USE": "foo? ( !bar )"}, - "dev-libs/C-3" : {"EAPI": "4", "IUSE": "+foo +bar", "REQUIRED_USE": "foo? ( bar )"}, - "dev-libs/C-4" : {"EAPI": "4", "IUSE": "+foo bar", "REQUIRED_USE": "foo? ( bar )"}, - "dev-libs/C-5" : {"EAPI": "4", "IUSE": "foo bar", "REQUIRED_USE": "foo? ( bar )"}, - "dev-libs/C-6" : {"EAPI": "4", "IUSE": "foo +bar", "REQUIRED_USE": "foo? ( bar )"}, - "dev-libs/C-7" : {"EAPI": "4", "IUSE": "foo +bar", "REQUIRED_USE": "!foo? ( bar )"}, - "dev-libs/C-8" : {"EAPI": "4", "IUSE": "+foo +bar", "REQUIRED_USE": "!foo? ( bar )"}, - "dev-libs/C-9" : {"EAPI": "4", "IUSE": "+foo bar", "REQUIRED_USE": "!foo? ( bar )"}, - "dev-libs/C-10": {"EAPI": "4", "IUSE": "foo bar", "REQUIRED_USE": "!foo? ( bar )"}, - "dev-libs/C-11": {"EAPI": "4", "IUSE": "foo bar", "REQUIRED_USE": "!foo? ( !bar )"}, - "dev-libs/C-12": {"EAPI": "4", "IUSE": "foo +bar", "REQUIRED_USE": "!foo? ( !bar )"}, - "dev-libs/C-13": {"EAPI": "4", "IUSE": "+foo +bar", "REQUIRED_USE": "!foo? ( !bar )"}, - "dev-libs/C-14": {"EAPI": "4", "IUSE": "+foo bar", "REQUIRED_USE": "!foo? ( !bar )"}, - - "dev-libs/D-1" : {"EAPI": "4", "IUSE": "+w +x +y z", "REQUIRED_USE": "w? ( x || ( y z ) )"}, - "dev-libs/D-2" : {"EAPI": "4", "IUSE": "+w +x +y +z", "REQUIRED_USE": "w? ( x || ( y z ) )"}, - "dev-libs/D-3" : {"EAPI": "4", "IUSE": "+w +x y z", "REQUIRED_USE": "w? ( x || ( y z ) )"}, - "dev-libs/D-4" : {"EAPI": "4", "IUSE": "+w x +y +z", "REQUIRED_USE": "w? ( x || ( y z ) )"}, - "dev-libs/D-5" : {"EAPI": "4", "IUSE": "w x y z", "REQUIRED_USE": "w? ( x || ( y z ) )"}, - } - - test_cases = ( - ResolverPlaygroundTestCase(["=dev-libs/A-1"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/A-2"], success = True, mergelist=["dev-libs/A-2"]), - ResolverPlaygroundTestCase(["=dev-libs/A-3"], success = True, mergelist=["dev-libs/A-3"]), - ResolverPlaygroundTestCase(["=dev-libs/A-4"], success = True, mergelist=["dev-libs/A-4"]), - ResolverPlaygroundTestCase(["=dev-libs/A-5"], success = True, mergelist=["dev-libs/A-5"]), - - ResolverPlaygroundTestCase(["=dev-libs/B-1"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/B-2"], success = True, mergelist=["dev-libs/B-2"]), - ResolverPlaygroundTestCase(["=dev-libs/B-3"], success = True, mergelist=["dev-libs/B-3"]), - ResolverPlaygroundTestCase(["=dev-libs/B-4"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/B-5"], success = True, mergelist=["dev-libs/B-5"]), - - ResolverPlaygroundTestCase(["=dev-libs/C-1"], success = True, mergelist=["dev-libs/C-1"]), - ResolverPlaygroundTestCase(["=dev-libs/C-2"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/C-3"], success = True, mergelist=["dev-libs/C-3"]), - ResolverPlaygroundTestCase(["=dev-libs/C-4"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/C-5"], success = True, mergelist=["dev-libs/C-5"]), - ResolverPlaygroundTestCase(["=dev-libs/C-6"], success = True, mergelist=["dev-libs/C-6"]), - ResolverPlaygroundTestCase(["=dev-libs/C-7"], success = True, mergelist=["dev-libs/C-7"]), - ResolverPlaygroundTestCase(["=dev-libs/C-8"], success = True, mergelist=["dev-libs/C-8"]), - ResolverPlaygroundTestCase(["=dev-libs/C-9"], success = True, mergelist=["dev-libs/C-9"]), - ResolverPlaygroundTestCase(["=dev-libs/C-10"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/C-11"], success = True, mergelist=["dev-libs/C-11"]), - ResolverPlaygroundTestCase(["=dev-libs/C-12"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/C-13"], success = True, mergelist=["dev-libs/C-13"]), - ResolverPlaygroundTestCase(["=dev-libs/C-14"], success = True, mergelist=["dev-libs/C-14"]), - - ResolverPlaygroundTestCase(["=dev-libs/D-1"], success = True, mergelist=["dev-libs/D-1"]), - ResolverPlaygroundTestCase(["=dev-libs/D-2"], success = True, mergelist=["dev-libs/D-2"]), - ResolverPlaygroundTestCase(["=dev-libs/D-3"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/D-4"], success = False), - ResolverPlaygroundTestCase(["=dev-libs/D-5"], success = True, mergelist=["dev-libs/D-5"]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() - - def testRequiredUseOrDeps(self): - - ebuilds = { - "dev-libs/A-1": { "IUSE": "+x +y", "REQUIRED_USE": "^^ ( x y )", "EAPI": "4" }, - "dev-libs/B-1": { "IUSE": "+x +y", "REQUIRED_USE": "", "EAPI": "4" }, - "app-misc/p-1": { "RDEPEND": "|| ( =dev-libs/A-1 =dev-libs/B-1 )" }, - } - - test_cases = ( - # This should fail and show a REQUIRED_USE error for - # dev-libs/A-1, since this choice it preferred. - ResolverPlaygroundTestCase( - ["=app-misc/p-1"], - success = False), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_simple.py b/portage_with_autodep/pym/portage/tests/resolver/test_simple.py deleted file mode 100644 index 0bcfc4b..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_simple.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class SimpleResolverTestCase(TestCase): - - def testSimple(self): - ebuilds = { - "dev-libs/A-1": { "KEYWORDS": "x86" }, - "dev-libs/A-2": { "KEYWORDS": "~x86" }, - "dev-libs/B-1.2": {}, - - "app-misc/Z-1": { "DEPEND": "|| ( app-misc/Y ( app-misc/X app-misc/W ) )", "RDEPEND": "" }, - "app-misc/Y-1": { "KEYWORDS": "~x86" }, - "app-misc/X-1": {}, - "app-misc/W-1": {}, - } - installed = { - "dev-libs/A-1": {}, - "dev-libs/B-1.1": {}, - } - - test_cases = ( - ResolverPlaygroundTestCase(["dev-libs/A"], success = True, mergelist = ["dev-libs/A-1"]), - ResolverPlaygroundTestCase(["=dev-libs/A-2"], options = { "--autounmask": 'n' }, success = False), - - ResolverPlaygroundTestCase( - ["dev-libs/A"], - options = {"--noreplace": True}, - success = True, - mergelist = []), - ResolverPlaygroundTestCase( - ["dev-libs/B"], - options = {"--noreplace": True}, - success = True, - mergelist = []), - ResolverPlaygroundTestCase( - ["dev-libs/B"], - options = {"--update": True}, - success = True, - mergelist = ["dev-libs/B-1.2"]), - - ResolverPlaygroundTestCase( - ["app-misc/Z"], - success = True, - mergelist = ["app-misc/W-1", "app-misc/X-1", "app-misc/Z-1"]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_slot_collisions.py b/portage_with_autodep/pym/portage/tests/resolver/test_slot_collisions.py deleted file mode 100644 index 4867cea..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_slot_collisions.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class SlotCollisionTestCase(TestCase): - - def testSlotCollision(self): - - ebuilds = { - "dev-libs/A-1": { "PDEPEND": "foo? ( dev-libs/B )", "IUSE": "foo" }, - "dev-libs/B-1": { "IUSE": "foo" }, - "dev-libs/C-1": { "DEPEND": "dev-libs/A[foo]", "EAPI": 2 }, - "dev-libs/D-1": { "DEPEND": "dev-libs/A[foo=] dev-libs/B[foo=]", "IUSE": "foo", "EAPI": 2 }, - "dev-libs/E-1": { }, - "dev-libs/E-2": { "IUSE": "foo" }, - - "app-misc/Z-1": { }, - "app-misc/Z-2": { }, - "app-misc/Y-1": { "DEPEND": "=app-misc/Z-1" }, - "app-misc/Y-2": { "DEPEND": ">app-misc/Z-1" }, - "app-misc/X-1": { "DEPEND": "=app-misc/Z-2" }, - "app-misc/X-2": { "DEPEND": "<app-misc/Z-2" }, - - "sci-libs/K-1": { "IUSE": "+foo", "EAPI": 1 }, - "sci-libs/L-1": { "DEPEND": "sci-libs/K[-foo]", "EAPI": 2 }, - "sci-libs/M-1": { "DEPEND": "sci-libs/K[foo=]", "IUSE": "+foo", "EAPI": 2 }, - - "sci-libs/Q-1": { "SLOT": "1", "IUSE": "+bar foo", "EAPI": 1 }, - "sci-libs/Q-2": { "SLOT": "2", "IUSE": "+bar +foo", "EAPI": 2, "PDEPEND": "sci-libs/Q:1[bar?,foo?]" }, - "sci-libs/P-1": { "DEPEND": "sci-libs/Q:1[foo=]", "IUSE": "foo", "EAPI": 2 }, - - "sys-libs/A-1": { "RDEPEND": "foo? ( sys-libs/J[foo=] )", "IUSE": "+foo", "EAPI": "4" }, - "sys-libs/B-1": { "RDEPEND": "bar? ( sys-libs/J[bar=] )", "IUSE": "+bar", "EAPI": "4" }, - "sys-libs/C-1": { "RDEPEND": "sys-libs/J[bar]", "EAPI": "4" }, - "sys-libs/D-1": { "RDEPEND": "sys-libs/J[bar?]", "IUSE": "bar", "EAPI": "4" }, - "sys-libs/E-1": { "RDEPEND": "sys-libs/J[foo(+)?]", "IUSE": "+foo", "EAPI": "4" }, - "sys-libs/F-1": { "RDEPEND": "sys-libs/J[foo(+)]", "EAPI": "4" }, - "sys-libs/J-1": { "IUSE": "+foo", "EAPI": "4" }, - "sys-libs/J-2": { "IUSE": "+bar", "EAPI": "4" }, - - "app-misc/A-1": { "IUSE": "foo +bar", "REQUIRED_USE": "^^ ( foo bar )", "EAPI": "4" }, - "app-misc/B-1": { "DEPEND": "=app-misc/A-1[foo=]", "IUSE": "foo", "EAPI": 2 }, - "app-misc/C-1": { "DEPEND": "=app-misc/A-1[foo]", "EAPI": 2 }, - "app-misc/E-1": { "RDEPEND": "dev-libs/E[foo?]", "IUSE": "foo", "EAPI": "2" }, - "app-misc/F-1": { "RDEPEND": "=dev-libs/E-1", "IUSE": "foo", "EAPI": "2" }, - } - installed = { - "dev-libs/A-1": { "PDEPEND": "foo? ( dev-libs/B )", "IUSE": "foo", "USE": "foo" }, - "dev-libs/B-1": { "IUSE": "foo", "USE": "foo" }, - "dev-libs/C-1": { "DEPEND": "dev-libs/A[foo]", "EAPI": 2 }, - "dev-libs/D-1": { "DEPEND": "dev-libs/A[foo=] dev-libs/B[foo=]", "IUSE": "foo", "USE": "foo", "EAPI": 2 }, - - "sci-libs/K-1": { "IUSE": "foo", "USE": "" }, - "sci-libs/L-1": { "DEPEND": "sci-libs/K[-foo]" }, - - "sci-libs/Q-1": { "SLOT": "1", "IUSE": "+bar +foo", "USE": "bar foo", "EAPI": 1 }, - "sci-libs/Q-2": { "SLOT": "2", "IUSE": "+bar +foo", "USE": "bar foo", "EAPI": 2, "PDEPEND": "sci-libs/Q:1[bar?,foo?]" }, - - "app-misc/A-1": { "IUSE": "+foo bar", "USE": "foo", "REQUIRED_USE": "^^ ( foo bar )", "EAPI": "4" }, - } - - test_cases = ( - #A qt-*[qt3support] like mess. - ResolverPlaygroundTestCase( - ["dev-libs/A", "dev-libs/B", "dev-libs/C", "dev-libs/D"], - options = { "--autounmask": 'n' }, - success = False, - mergelist = ["dev-libs/A-1", "dev-libs/B-1", "dev-libs/C-1", "dev-libs/D-1"], - ignore_mergelist_order = True, - slot_collision_solutions = [ {"dev-libs/A-1": {"foo": True}, "dev-libs/D-1": {"foo": True}} ]), - - ResolverPlaygroundTestCase( - ["sys-libs/A", "sys-libs/B", "sys-libs/C", "sys-libs/D", "sys-libs/E", "sys-libs/F"], - options = { "--autounmask": 'n' }, - success = False, - ignore_mergelist_order = True, - slot_collision_solutions = [], - mergelist = ['sys-libs/J-2', 'sys-libs/J-1', 'sys-libs/A-1', 'sys-libs/B-1', 'sys-libs/C-1', 'sys-libs/D-1', 'sys-libs/E-1', 'sys-libs/F-1'], - ), - - #A version based conflicts, nothing we can do. - ResolverPlaygroundTestCase( - ["=app-misc/X-1", "=app-misc/Y-1"], - success = False, - mergelist = ["app-misc/Z-1", "app-misc/Z-2", "app-misc/X-1", "app-misc/Y-1"], - ignore_mergelist_order = True, - slot_collision_solutions = [] - ), - ResolverPlaygroundTestCase( - ["=app-misc/X-2", "=app-misc/Y-2"], - success = False, - mergelist = ["app-misc/Z-1", "app-misc/Z-2", "app-misc/X-2", "app-misc/Y-2"], - ignore_mergelist_order = True, - slot_collision_solutions = [] - ), - - ResolverPlaygroundTestCase( - ["=app-misc/E-1", "=app-misc/F-1"], - success = False, - mergelist = ["dev-libs/E-1", "dev-libs/E-2", "app-misc/E-1", "app-misc/F-1"], - ignore_mergelist_order = True, - slot_collision_solutions = [] - ), - - #Simple cases. - ResolverPlaygroundTestCase( - ["sci-libs/L", "sci-libs/M"], - success = False, - mergelist = ["sci-libs/L-1", "sci-libs/M-1", "sci-libs/K-1"], - ignore_mergelist_order = True, - slot_collision_solutions = [{"sci-libs/K-1": {"foo": False}, "sci-libs/M-1": {"foo": False}}] - ), - - #Avoid duplicates. - ResolverPlaygroundTestCase( - ["sci-libs/P", "sci-libs/Q:2"], - success = False, - options = { "--update": True, "--complete-graph": True, "--autounmask": 'n' }, - mergelist = ["sci-libs/P-1", "sci-libs/Q-1"], - ignore_mergelist_order = True, - all_permutations=True, - slot_collision_solutions = [{"sci-libs/Q-1": {"foo": True}, "sci-libs/P-1": {"foo": True}}] - ), - - #Conflict with REQUIRED_USE - ResolverPlaygroundTestCase( - ["=app-misc/C-1", "=app-misc/B-1"], - all_permutations = True, - slot_collision_solutions = [], - mergelist = ["app-misc/A-1", "app-misc/C-1", "app-misc/B-1"], - ignore_mergelist_order = True, - success = False), - ) - - playground = ResolverPlayground(ebuilds=ebuilds, installed=installed) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/resolver/test_use_dep_defaults.py b/portage_with_autodep/pym/portage/tests/resolver/test_use_dep_defaults.py deleted file mode 100644 index 7d17106..0000000 --- a/portage_with_autodep/pym/portage/tests/resolver/test_use_dep_defaults.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.tests.resolver.ResolverPlayground import ResolverPlayground, ResolverPlaygroundTestCase - -class UseDepDefaultsTestCase(TestCase): - - def testUseDepDefaultse(self): - - ebuilds = { - "dev-libs/A-1": { "DEPEND": "dev-libs/B[foo]", "RDEPEND": "dev-libs/B[foo]", "EAPI": "2" }, - "dev-libs/A-2": { "DEPEND": "dev-libs/B[foo(+)]", "RDEPEND": "dev-libs/B[foo(+)]", "EAPI": "4" }, - "dev-libs/A-3": { "DEPEND": "dev-libs/B[foo(-)]", "RDEPEND": "dev-libs/B[foo(-)]", "EAPI": "4" }, - "dev-libs/B-1": { "IUSE": "+foo", "EAPI": "1" }, - "dev-libs/B-2": {}, - } - - test_cases = ( - ResolverPlaygroundTestCase( - ["=dev-libs/A-1"], - success = True, - mergelist = ["dev-libs/B-1", "dev-libs/A-1"]), - ResolverPlaygroundTestCase( - ["=dev-libs/A-2"], - success = True, - mergelist = ["dev-libs/B-2", "dev-libs/A-2"]), - ResolverPlaygroundTestCase( - ["=dev-libs/A-3"], - success = True, - mergelist = ["dev-libs/B-1", "dev-libs/A-3"]), - ) - - playground = ResolverPlayground(ebuilds=ebuilds) - try: - for test_case in test_cases: - playground.run_TestCase(test_case) - self.assertEqual(test_case.test_success, True, test_case.fail_msg) - finally: - playground.cleanup() diff --git a/portage_with_autodep/pym/portage/tests/runTests b/portage_with_autodep/pym/portage/tests/runTests index 6b3311d..4c10087 100755 --- a/portage_with_autodep/pym/portage/tests/runTests +++ b/portage_with_autodep/pym/portage/tests/runTests @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python -Wd # runTests.py -- Portage Unit Test Functionality # Copyright 2006 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -41,6 +41,4 @@ del path if __name__ == "__main__": - result = tests.main() - if not result.wasSuccessful(): - sys.exit(1) + sys.exit(tests.main()) diff --git a/portage_with_autodep/pym/portage/tests/sets/__init__.py b/portage_with_autodep/pym/portage/tests/sets/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/__init__.py +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/sets/base/__init__.py b/portage_with_autodep/pym/portage/tests/sets/base/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/base/__init__.py +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/sets/base/__test__ b/portage_with_autodep/pym/portage/tests/sets/base/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/base/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/sets/base/testInternalPackageSet.py b/portage_with_autodep/pym/portage/tests/sets/base/testInternalPackageSet.py deleted file mode 100644 index e0a3478..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/base/testInternalPackageSet.py +++ /dev/null @@ -1,61 +0,0 @@ -# testConfigFileSet.py -- Portage Unit Testing Functionality -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.dep import Atom -from portage.exception import InvalidAtom -from portage.tests import TestCase -from portage._sets.base import InternalPackageSet - -class InternalPackageSetTestCase(TestCase): - """Simple Test Case for InternalPackageSet""" - - def testInternalPackageSet(self): - i1_atoms = set(("dev-libs/A", ">=dev-libs/A-1", "dev-libs/B")) - i2_atoms = set(("dev-libs/A", "dev-libs/*", "dev-libs/C")) - - i1 = InternalPackageSet(initial_atoms=i1_atoms) - i2 = InternalPackageSet(initial_atoms=i2_atoms, allow_wildcard=True) - self.assertRaises(InvalidAtom, InternalPackageSet, initial_atoms=i2_atoms) - - self.assertEqual(i1.getAtoms(), i1_atoms) - self.assertEqual(i2.getAtoms(), i2_atoms) - - new_atom = Atom("*/*", allow_wildcard=True) - self.assertRaises(InvalidAtom, i1.add, new_atom) - i2.add(new_atom) - - i2_atoms.add(new_atom) - - self.assertEqual(i1.getAtoms(), i1_atoms) - self.assertEqual(i2.getAtoms(), i2_atoms) - - removed_atom = Atom("dev-libs/A") - - i1.remove(removed_atom) - i2.remove(removed_atom) - - i1_atoms.remove(removed_atom) - i2_atoms.remove(removed_atom) - - self.assertEqual(i1.getAtoms(), i1_atoms) - self.assertEqual(i2.getAtoms(), i2_atoms) - - update_atoms = [Atom("dev-libs/C"), Atom("dev-*/C", allow_wildcard=True)] - - self.assertRaises(InvalidAtom, i1.update, update_atoms) - i2.update(update_atoms) - - i2_atoms.update(update_atoms) - - self.assertEqual(i1.getAtoms(), i1_atoms) - self.assertEqual(i2.getAtoms(), i2_atoms) - - replace_atoms = [Atom("dev-libs/D"), Atom("*-libs/C", allow_wildcard=True)] - - self.assertRaises(InvalidAtom, i1.replace, replace_atoms) - i2.replace(replace_atoms) - - i2_atoms = set(replace_atoms) - - self.assertEqual(i2.getAtoms(), i2_atoms) diff --git a/portage_with_autodep/pym/portage/tests/sets/files/__init__.py b/portage_with_autodep/pym/portage/tests/sets/files/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/files/__init__.py +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/sets/files/__test__ b/portage_with_autodep/pym/portage/tests/sets/files/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/files/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/sets/files/testConfigFileSet.py b/portage_with_autodep/pym/portage/tests/sets/files/testConfigFileSet.py deleted file mode 100644 index 3ec26a0..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/files/testConfigFileSet.py +++ /dev/null @@ -1,32 +0,0 @@ -# testConfigFileSet.py -- Portage Unit Testing Functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import tempfile - -from portage import os -from portage.tests import TestCase, test_cps -from portage._sets.files import ConfigFileSet - -class ConfigFileSetTestCase(TestCase): - """Simple Test Case for ConfigFileSet""" - - def setUp(self): - fd, self.testfile = tempfile.mkstemp(suffix=".testdata", prefix=self.__class__.__name__, text=True) - f = os.fdopen(fd, 'w') - for i in range(0, len(test_cps)): - atom = test_cps[i] - if i % 2 == 0: - f.write(atom + ' abc def\n') - else: - f.write(atom + '\n') - f.close() - - def tearDown(self): - os.unlink(self.testfile) - - def testConfigStaticFileSet(self): - s = ConfigFileSet(self.testfile) - s.load() - self.assertEqual(set(test_cps), s.getAtoms()) - diff --git a/portage_with_autodep/pym/portage/tests/sets/files/testStaticFileSet.py b/portage_with_autodep/pym/portage/tests/sets/files/testStaticFileSet.py deleted file mode 100644 index d515a67..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/files/testStaticFileSet.py +++ /dev/null @@ -1,27 +0,0 @@ -# testStaticFileSet.py -- Portage Unit Testing Functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import tempfile - -from portage import os -from portage.tests import TestCase, test_cps -from portage._sets.files import StaticFileSet - -class StaticFileSetTestCase(TestCase): - """Simple Test Case for StaticFileSet""" - - def setUp(self): - fd, self.testfile = tempfile.mkstemp(suffix=".testdata", prefix=self.__class__.__name__, text=True) - f = os.fdopen(fd, 'w') - f.write("\n".join(test_cps)) - f.close() - - def tearDown(self): - os.unlink(self.testfile) - - def testSampleStaticFileSet(self): - s = StaticFileSet(self.testfile) - s.load() - self.assertEqual(set(test_cps), s.getAtoms()) - diff --git a/portage_with_autodep/pym/portage/tests/sets/shell/__init__.py b/portage_with_autodep/pym/portage/tests/sets/shell/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/shell/__init__.py +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/sets/shell/__test__ b/portage_with_autodep/pym/portage/tests/sets/shell/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/shell/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/sets/shell/testShell.py b/portage_with_autodep/pym/portage/tests/sets/shell/testShell.py deleted file mode 100644 index 2cdd833..0000000 --- a/portage_with_autodep/pym/portage/tests/sets/shell/testShell.py +++ /dev/null @@ -1,28 +0,0 @@ -# testCommandOututSet.py -- Portage Unit Testing Functionality -# Copyright 2007 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.process import find_binary -from portage.tests import TestCase, test_cps -from portage._sets.shell import CommandOutputSet - -class CommandOutputSetTestCase(TestCase): - """Simple Test Case for CommandOutputSet""" - - def setUp(self): - pass - - def tearDown(self): - pass - - def testCommand(self): - - input = set(test_cps) - command = find_binary("bash") - command += " -c '" - for a in input: - command += " echo -e \"%s\" ; " % a - command += "'" - s = CommandOutputSet(command) - atoms = s.getAtoms() - self.assertEqual(atoms, input) diff --git a/portage_with_autodep/pym/portage/tests/unicode/__test__ b/portage_with_autodep/pym/portage/tests/unicode/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/unicode/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/unicode/test_string_format.py b/portage_with_autodep/pym/portage/tests/unicode/test_string_format.py deleted file mode 100644 index fb6e8e0..0000000 --- a/portage_with_autodep/pym/portage/tests/unicode/test_string_format.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import sys - -from portage import _encodings, _unicode_decode -from portage.exception import PortageException -from portage.tests import TestCase -from _emerge.DependencyArg import DependencyArg -from _emerge.UseFlagDisplay import UseFlagDisplay - -if sys.hexversion >= 0x3000000: - basestring = str - -STR_IS_UNICODE = sys.hexversion >= 0x3000000 - -class StringFormatTestCase(TestCase): - """ - Test that string formatting works correctly in the current interpretter, - which may be either python2 or python3. - """ - - # In order to get some unicode test strings in a way that works in - # both python2 and python3, write them here as byte strings and - # decode them before use. This assumes _encodings['content'] is - # utf_8. - - unicode_strings = ( - b'\xE2\x80\x98', - b'\xE2\x80\x99', - ) - - def testDependencyArg(self): - - self.assertEqual(_encodings['content'], 'utf_8') - - for arg_bytes in self.unicode_strings: - arg_unicode = _unicode_decode(arg_bytes, encoding=_encodings['content']) - dependency_arg = DependencyArg(arg=arg_unicode) - - # Force unicode format string so that __unicode__() is - # called in python2. - formatted_str = _unicode_decode("%s") % (dependency_arg,) - self.assertEqual(formatted_str, arg_unicode) - - if STR_IS_UNICODE: - - # Test the __str__ method which returns unicode in python3 - formatted_str = "%s" % (dependency_arg,) - self.assertEqual(formatted_str, arg_unicode) - - else: - - # Test the __str__ method which returns encoded bytes in python2 - formatted_bytes = "%s" % (dependency_arg,) - self.assertEqual(formatted_bytes, arg_bytes) - - def testPortageException(self): - - self.assertEqual(_encodings['content'], 'utf_8') - - for arg_bytes in self.unicode_strings: - arg_unicode = _unicode_decode(arg_bytes, encoding=_encodings['content']) - e = PortageException(arg_unicode) - - # Force unicode format string so that __unicode__() is - # called in python2. - formatted_str = _unicode_decode("%s") % (e,) - self.assertEqual(formatted_str, arg_unicode) - - if STR_IS_UNICODE: - - # Test the __str__ method which returns unicode in python3 - formatted_str = "%s" % (e,) - self.assertEqual(formatted_str, arg_unicode) - - else: - - # Test the __str__ method which returns encoded bytes in python2 - formatted_bytes = "%s" % (e,) - self.assertEqual(formatted_bytes, arg_bytes) - - def testUseFlagDisplay(self): - - self.assertEqual(_encodings['content'], 'utf_8') - - for enabled in (True, False): - for forced in (True, False): - for arg_bytes in self.unicode_strings: - arg_unicode = _unicode_decode(arg_bytes, encoding=_encodings['content']) - e = UseFlagDisplay(arg_unicode, enabled, forced) - - # Force unicode format string so that __unicode__() is - # called in python2. - formatted_str = _unicode_decode("%s") % (e,) - self.assertEqual(isinstance(formatted_str, basestring), True) - - if STR_IS_UNICODE: - - # Test the __str__ method which returns unicode in python3 - formatted_str = "%s" % (e,) - self.assertEqual(isinstance(formatted_str, str), True) - - else: - - # Test the __str__ method which returns encoded bytes in python2 - formatted_bytes = "%s" % (e,) - self.assertEqual(isinstance(formatted_bytes, bytes), True) diff --git a/portage_with_autodep/pym/portage/tests/util/__init__.py b/portage_with_autodep/pym/portage/tests/util/__init__.py deleted file mode 100644 index 69ce189..0000000 --- a/portage_with_autodep/pym/portage/tests/util/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# tests/portage.util/__init__.py -- Portage Unit Test functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - diff --git a/portage_with_autodep/pym/portage/tests/util/__test__ b/portage_with_autodep/pym/portage/tests/util/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/util/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/util/test_digraph.py b/portage_with_autodep/pym/portage/tests/util/test_digraph.py deleted file mode 100644 index b65c0b1..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_digraph.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright 2010-2011 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.util.digraph import digraph -#~ from portage.util import noiselimit -import portage.util - -class DigraphTest(TestCase): - - def testBackwardCompatibility(self): - g = digraph() - f = g.copy() - g.addnode("A", None) - self.assertEqual("A" in g, True) - self.assertEqual(bool(g), True) - self.assertEqual(g.allnodes(), ["A"]) - self.assertEqual(g.allzeros(), ["A"]) - self.assertEqual(g.hasnode("A"), True) - - def testDigraphEmptyGraph(self): - g = digraph() - f = g.clone() - for x in g, f: - self.assertEqual(bool(x), False) - self.assertEqual(x.contains("A"), False) - self.assertEqual(x.firstzero(), None) - self.assertRaises(KeyError, x.remove, "A") - x.delnode("A") - self.assertEqual(list(x), []) - self.assertEqual(x.get("A"), None) - self.assertEqual(x.get("A", "default"), "default") - self.assertEqual(x.all_nodes(), []) - self.assertEqual(x.leaf_nodes(), []) - self.assertEqual(x.root_nodes(), []) - self.assertRaises(KeyError, x.child_nodes, "A") - self.assertRaises(KeyError, x.parent_nodes, "A") - self.assertEqual(x.hasallzeros(), True) - self.assertRaises(KeyError, list, x.bfs("A")) - self.assertRaises(KeyError, x.shortest_path, "A", "B") - self.assertRaises(KeyError, x.remove_edge, "A", "B") - self.assertEqual(x.get_cycles(), []) - x.difference_update("A") - portage.util.noiselimit = -2 - x.debug_print() - portage.util.noiselimit = 0 - - def testDigraphCircle(self): - g = digraph() - g.add("A", "B", -1) - g.add("B", "C", 0) - g.add("C", "D", 1) - g.add("D", "A", 2) - - f = g.clone() - for x in g, f: - self.assertEqual(bool(x), True) - self.assertEqual(x.contains("A"), True) - self.assertEqual(x.firstzero(), None) - self.assertRaises(KeyError, x.remove, "Z") - x.delnode("Z") - self.assertEqual(list(x), ["A", "B", "C", "D"]) - self.assertEqual(x.get("A"), "A") - self.assertEqual(x.get("A", "default"), "A") - self.assertEqual(x.all_nodes(), ["A", "B", "C", "D"]) - self.assertEqual(x.leaf_nodes(), []) - self.assertEqual(x.root_nodes(), []) - self.assertEqual(x.child_nodes("A"), ["D"]) - self.assertEqual(x.child_nodes("A", ignore_priority=2), []) - self.assertEqual(x.parent_nodes("A"), ["B"]) - self.assertEqual(x.parent_nodes("A", ignore_priority=-2), ["B"]) - self.assertEqual(x.parent_nodes("A", ignore_priority=-1), []) - self.assertEqual(x.hasallzeros(), False) - self.assertEqual(list(x.bfs("A")), [(None, "A"), ("A", "D"), ("D", "C"), ("C", "B")]) - self.assertEqual(x.shortest_path("A", "D"), ["A", "D"]) - self.assertEqual(x.shortest_path("D", "A"), ["D", "C", "B", "A"]) - self.assertEqual(x.shortest_path("A", "D", ignore_priority=2), None) - self.assertEqual(x.shortest_path("D", "A", ignore_priority=-2), ["D", "C", "B", "A"]) - cycles = set(tuple(y) for y in x.get_cycles()) - self.assertEqual(cycles, set([("D", "C", "B", "A"), ("C", "B", "A", "D"), ("B", "A", "D", "C"), \ - ("A", "D", "C", "B")])) - x.remove_edge("A", "B") - self.assertEqual(x.get_cycles(), []) - x.difference_update(["D"]) - self.assertEqual(x.all_nodes(), ["A", "B", "C"]) - portage.util.noiselimit = -2 - x.debug_print() - portage.util.noiselimit = 0 - - def testDigraphTree(self): - g = digraph() - g.add("B", "A", -1) - g.add("C", "A", 0) - g.add("D", "C", 1) - g.add("E", "C", 2) - - f = g.clone() - for x in g, f: - self.assertEqual(bool(x), True) - self.assertEqual(x.contains("A"), True) - self.assertEqual(x.firstzero(), "B") - self.assertRaises(KeyError, x.remove, "Z") - x.delnode("Z") - self.assertEqual(set(x), set(["A", "B", "C", "D", "E"])) - self.assertEqual(x.get("A"), "A") - self.assertEqual(x.get("A", "default"), "A") - self.assertEqual(set(x.all_nodes()), set(["A", "B", "C", "D", "E"])) - self.assertEqual(set(x.leaf_nodes()), set(["B", "D", "E"])) - self.assertEqual(set(x.leaf_nodes(ignore_priority=0)), set(["A", "B", "D", "E"])) - self.assertEqual(x.root_nodes(), ["A"]) - self.assertEqual(set(x.root_nodes(ignore_priority=0)), set(["A", "B", "C"])) - self.assertEqual(set(x.child_nodes("A")), set(["B", "C"])) - self.assertEqual(x.child_nodes("A", ignore_priority=2), []) - self.assertEqual(x.parent_nodes("B"), ["A"]) - self.assertEqual(x.parent_nodes("B", ignore_priority=-2), ["A"]) - self.assertEqual(x.parent_nodes("B", ignore_priority=-1), []) - self.assertEqual(x.hasallzeros(), False) - self.assertEqual(list(x.bfs("A")), [(None, "A"), ("A", "C"), ("A", "B"), ("C", "E"), ("C", "D")]) - self.assertEqual(x.shortest_path("A", "D"), ["A", "C", "D"]) - self.assertEqual(x.shortest_path("D", "A"), None) - self.assertEqual(x.shortest_path("A", "D", ignore_priority=2), None) - cycles = set(tuple(y) for y in x.get_cycles()) - self.assertEqual(cycles, set()) - x.remove("D") - self.assertEqual(set(x.all_nodes()), set(["A", "B", "C", "E"])) - x.remove("C") - self.assertEqual(set(x.all_nodes()), set(["A", "B", "E"])) - portage.util.noiselimit = -2 - x.debug_print() - portage.util.noiselimit = 0 - self.assertRaises(KeyError, x.remove_edge, "A", "E") - - def testDigraphCompleteGraph(self): - g = digraph() - g.add("A", "B", -1) - g.add("B", "A", 1) - g.add("A", "C", 1) - g.add("C", "A", -1) - g.add("C", "B", 1) - g.add("B", "C", 1) - - f = g.clone() - for x in g, f: - self.assertEqual(bool(x), True) - self.assertEqual(x.contains("A"), True) - self.assertEqual(x.firstzero(), None) - self.assertRaises(KeyError, x.remove, "Z") - x.delnode("Z") - self.assertEqual(list(x), ["A", "B", "C"]) - self.assertEqual(x.get("A"), "A") - self.assertEqual(x.get("A", "default"), "A") - self.assertEqual(x.all_nodes(), ["A", "B", "C"]) - self.assertEqual(x.leaf_nodes(), []) - self.assertEqual(x.root_nodes(), []) - self.assertEqual(set(x.child_nodes("A")), set(["B", "C"])) - self.assertEqual(x.child_nodes("A", ignore_priority=0), ["B"]) - self.assertEqual(set(x.parent_nodes("A")), set(["B", "C"])) - self.assertEqual(x.parent_nodes("A", ignore_priority=0), ["C"]) - self.assertEqual(x.parent_nodes("A", ignore_priority=1), []) - self.assertEqual(x.hasallzeros(), False) - self.assertEqual(list(x.bfs("A")), [(None, "A"), ("A", "C"), ("A", "B")]) - self.assertEqual(x.shortest_path("A", "C"), ["A", "C"]) - self.assertEqual(x.shortest_path("C", "A"), ["C", "A"]) - self.assertEqual(x.shortest_path("A", "C", ignore_priority=0), ["A", "B", "C"]) - self.assertEqual(x.shortest_path("C", "A", ignore_priority=0), ["C", "A"]) - cycles = set(tuple(y) for y in x.get_cycles()) - self.assertEqual(cycles, set([("C", "A"), ("A", "B"), ("A", "C")])) - x.remove_edge("A", "B") - self.assertEqual(x.get_cycles(), [["C", "A"], ["A", "C"], ["C", "B"]]) - x.difference_update(["C"]) - self.assertEqual(x.all_nodes(), ["A", "B"]) - portage.util.noiselimit = -2 - x.debug_print() - portage.util.noiselimit = 0 - - def testDigraphIgnorePriority(self): - - def always_true(dummy): - return True - - def always_false(dummy): - return False - - g = digraph() - g.add("A", "B") - - self.assertEqual(g.parent_nodes("A"), ["B"]) - self.assertEqual(g.parent_nodes("A", ignore_priority=always_false), ["B"]) - self.assertEqual(g.parent_nodes("A", ignore_priority=always_true), []) - - self.assertEqual(g.child_nodes("B"), ["A"]) - self.assertEqual(g.child_nodes("B", ignore_priority=always_false), ["A"]) - self.assertEqual(g.child_nodes("B", ignore_priority=always_true), []) - - self.assertEqual(g.leaf_nodes(), ["A"]) - self.assertEqual(g.leaf_nodes(ignore_priority=always_false), ["A"]) - self.assertEqual(g.leaf_nodes(ignore_priority=always_true), ["A", "B"]) - - self.assertEqual(g.root_nodes(), ["B"]) - self.assertEqual(g.root_nodes(ignore_priority=always_false), ["B"]) - self.assertEqual(g.root_nodes(ignore_priority=always_true), ["A", "B"]) diff --git a/portage_with_autodep/pym/portage/tests/util/test_getconfig.py b/portage_with_autodep/pym/portage/tests/util/test_getconfig.py deleted file mode 100644 index 22e0bfc..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_getconfig.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage.const import PORTAGE_BASE_PATH -from portage.tests import TestCase -from portage.util import getconfig - -class GetConfigTestCase(TestCase): - """ - Test that getconfig() produces that same result as bash would when - sourcing the same input. - """ - - _cases = { - 'FETCHCOMMAND' : '/usr/bin/wget -t 3 -T 60 --passive-ftp -O "${DISTDIR}/${FILE}" "${URI}"', - 'FETCHCOMMAND_RSYNC' : 'rsync -avP "${URI}" "${DISTDIR}/${FILE}"', - 'FETCHCOMMAND_SFTP' : 'bash -c "x=\\${2#sftp://} ; host=\\${x%%/*} ; port=\\${host##*:} ; host=\\${host%:*} ; [[ \\${host} = \\${port} ]] && port=22 ; exec sftp -P \\${port} \\"\\${host}:/\\${x#*/}\\" \\"\\$1\\"" sftp "${DISTDIR}/${FILE}" "${URI}"', - 'FETCHCOMMAND_SSH' : 'bash -c "x=\\${2#ssh://} ; host=\\${x%%/*} ; port=\\${host##*:} ; host=\\${host%:*} ; [[ \\${host} = \\${port} ]] && port=22 ; exec rsync --rsh=\\"ssh -p\\${port}\\" -avP \\"\\${host}:/\\${x#*/}\\" \\"\\$1\\"" rsync "${DISTDIR}/${FILE}" "${URI}"', - 'PORTAGE_ELOG_MAILSUBJECT' : '[portage] ebuild log for ${PACKAGE} on ${HOST}' - } - - def testGetConfig(self): - - make_globals_file = os.path.join(PORTAGE_BASE_PATH, - 'cnf', 'make.globals') - d = getconfig(make_globals_file) - for k, v in self._cases.items(): - self.assertEqual(d[k], v) diff --git a/portage_with_autodep/pym/portage/tests/util/test_grabdict.py b/portage_with_autodep/pym/portage/tests/util/test_grabdict.py deleted file mode 100644 index e62a75d..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_grabdict.py +++ /dev/null @@ -1,11 +0,0 @@ -# test_grabDict.py -- Portage Unit Testing Functionality -# Copyright 2006-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -#from portage.util import grabdict - -class GrabDictTestCase(TestCase): - - def testGrabDictPass(self): - pass diff --git a/portage_with_autodep/pym/portage/tests/util/test_normalizedPath.py b/portage_with_autodep/pym/portage/tests/util/test_normalizedPath.py deleted file mode 100644 index f993886..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_normalizedPath.py +++ /dev/null @@ -1,14 +0,0 @@ -# test_normalizePath.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase - -class NormalizePathTestCase(TestCase): - - def testNormalizePath(self): - - from portage.util import normalize_path - path = "///foo/bar/baz" - good = "/foo/bar/baz" - self.assertEqual(normalize_path(path), good) diff --git a/portage_with_autodep/pym/portage/tests/util/test_stackDictList.py b/portage_with_autodep/pym/portage/tests/util/test_stackDictList.py deleted file mode 100644 index 678001c..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_stackDictList.py +++ /dev/null @@ -1,17 +0,0 @@ -# test_stackDictList.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase - -class StackDictListTestCase(TestCase): - - def testStackDictList(self): - from portage.util import stack_dictlist - - tests = [ ({'a':'b'},{'x':'y'},False,{'a':['b'],'x':['y']}) ] - tests.append(( {'KEYWORDS':['alpha','x86']},{'KEYWORDS':['-*']},True,{} )) - tests.append(( {'KEYWORDS':['alpha','x86']},{'KEYWORDS':['-x86']},True,{'KEYWORDS':['alpha']} )) - for test in tests: - self.assertEqual( - stack_dictlist([test[0],test[1]],incremental=test[2]), test[3] ) diff --git a/portage_with_autodep/pym/portage/tests/util/test_stackDicts.py b/portage_with_autodep/pym/portage/tests/util/test_stackDicts.py deleted file mode 100644 index 0d2cadd..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_stackDicts.py +++ /dev/null @@ -1,36 +0,0 @@ -# test_stackDicts.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.util import stack_dicts - - -class StackDictsTestCase(TestCase): - - def testStackDictsPass(self): - - tests = [ ( [ { "a":"b" }, { "b":"c" } ], { "a":"b", "b":"c" }, - False, [], False ), - ( [ { "a":"b" }, { "a":"c" } ], { "a":"b c" }, - True, [], False ), - ( [ { "a":"b" }, { "a":"c" } ], { "a":"b c" }, - False, ["a"], False ), - ( [ { "a":"b" }, None ], { "a":"b" }, - False, [], True ), - ( [ None ], {}, False, [], False ), - ( [ None, {}], {}, False, [], True ) ] - - - for test in tests: - result = stack_dicts( test[0], test[2], test[3], test[4] ) - self.assertEqual( result, test[1] ) - - def testStackDictsFail(self): - - tests = [ ( [ None, {} ], None, False, [], True ), - ( [ { "a":"b"}, {"a":"c" } ], { "a":"b c" }, - False, [], False ) ] - for test in tests: - result = stack_dicts( test[0], test[2], test[3], test[4] ) - self.assertNotEqual( result , test[1] ) diff --git a/portage_with_autodep/pym/portage/tests/util/test_stackLists.py b/portage_with_autodep/pym/portage/tests/util/test_stackLists.py deleted file mode 100644 index 8d01ea5..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_stackLists.py +++ /dev/null @@ -1,19 +0,0 @@ -# test_stackLists.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.util import stack_lists - -class StackListsTestCase(TestCase): - - def testStackLists(self): - - tests = [ ( [ ['a','b','c'], ['d','e','f'] ], ['a','c','b','e','d','f'], False ), - ( [ ['a','x'], ['b','x'] ], ['a','x','b'], False ), - ( [ ['a','b','c'], ['-*'] ], [], True ), - ( [ ['a'], ['-a'] ], [], True ) ] - - for test in tests: - result = stack_lists( test[0], test[2] ) - self.assertEqual( result , test[1] ) diff --git a/portage_with_autodep/pym/portage/tests/util/test_uniqueArray.py b/portage_with_autodep/pym/portage/tests/util/test_uniqueArray.py deleted file mode 100644 index 2a1a209..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_uniqueArray.py +++ /dev/null @@ -1,24 +0,0 @@ -# test_uniqueArray.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage import os -from portage.tests import TestCase -from portage.util import unique_array - -class UniqueArrayTestCase(TestCase): - - def testUniqueArrayPass(self): - """ - test portage.util.uniqueArray() - """ - - tests = [ ( ["a","a","a",os,os,[],[],[]], ['a',os,[]] ), - ( [1,1,1,2,3,4,4] , [1,2,3,4]) ] - - for test in tests: - result = unique_array( test[0] ) - for item in test[1]: - number = result.count(item) - self.assertFalse( number is not 1, msg="%s contains %s of %s, \ - should be only 1" % (result, number, item) ) diff --git a/portage_with_autodep/pym/portage/tests/util/test_varExpand.py b/portage_with_autodep/pym/portage/tests/util/test_varExpand.py deleted file mode 100644 index 7b528d6..0000000 --- a/portage_with_autodep/pym/portage/tests/util/test_varExpand.py +++ /dev/null @@ -1,92 +0,0 @@ -# test_varExpand.py -- Portage Unit Testing Functionality -# Copyright 2006-2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.util import varexpand - -class VarExpandTestCase(TestCase): - - def testVarExpandPass(self): - - varDict = { "a":"5", "b":"7", "c":"-5" } - for key in varDict: - result = varexpand( "$%s" % key, varDict ) - - self.assertFalse( result != varDict[key], - msg="Got %s != %s, from varexpand( %s, %s )" % \ - ( result, varDict[key], "$%s" % key, varDict ) ) - result = varexpand( "${%s}" % key, varDict ) - self.assertFalse( result != varDict[key], - msg="Got %s != %s, from varexpand( %s, %s )" % \ - ( result, varDict[key], "${%s}" % key, varDict ) ) - - def testVarExpandBackslashes(self): - """ - We want to behave like bash does when expanding a variable - assignment in a sourced file, in which case it performs - backslash removal for \\ and \$ but nothing more. It also - removes escaped newline characters. Note that we don't - handle escaped quotes here, since getconfig() uses shlex - to handle that earlier. - """ - - varDict = {} - tests = [ - ("\\", "\\"), - ("\\\\", "\\"), - ("\\\\\\", "\\\\"), - ("\\\\\\\\", "\\\\"), - ("\\$", "$"), - ("\\\\$", "\\$"), - ("\\a", "\\a"), - ("\\b", "\\b"), - ("\\n", "\\n"), - ("\\r", "\\r"), - ("\\t", "\\t"), - ("\\\n", ""), - ("\\\"", "\\\""), - ("\\'", "\\'"), - ] - for test in tests: - result = varexpand( test[0], varDict ) - self.assertFalse( result != test[1], - msg="Got %s != %s from varexpand( %s, %s )" \ - % ( result, test[1], test[0], varDict ) ) - - def testVarExpandDoubleQuotes(self): - - varDict = { "a":"5" } - tests = [ ("\"${a}\"", "\"5\"") ] - for test in tests: - result = varexpand( test[0], varDict ) - self.assertFalse( result != test[1], - msg="Got %s != %s from varexpand( %s, %s )" \ - % ( result, test[1], test[0], varDict ) ) - - def testVarExpandSingleQuotes(self): - - varDict = { "a":"5" } - tests = [ ("\'${a}\'", "\'${a}\'") ] - for test in tests: - result = varexpand( test[0], varDict ) - self.assertFalse( result != test[1], - msg="Got %s != %s from varexpand( %s, %s )" \ - % ( result, test[1], test[0], varDict ) ) - - def testVarExpandFail(self): - - varDict = { "a":"5", "b":"7", "c":"15" } - - testVars = [ "fail" ] - - for var in testVars: - result = varexpand( "$%s" % var, varDict ) - self.assertFalse( len(result), - msg="Got %s == %s, from varexpand( %s, %s )" \ - % ( result, var, "$%s" % var, varDict ) ) - - result = varexpand( "${%s}" % var, varDict ) - self.assertFalse( len(result), - msg="Got %s == %s, from varexpand( %s, %s )" \ - % ( result, var, "${%s}" % var, varDict ) ) diff --git a/portage_with_autodep/pym/portage/tests/versions/__init__.py b/portage_with_autodep/pym/portage/tests/versions/__init__.py deleted file mode 100644 index 2b14180..0000000 --- a/portage_with_autodep/pym/portage/tests/versions/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# tests/portage.versions/__init__.py -- Portage Unit Test functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/tests/versions/__test__ b/portage_with_autodep/pym/portage/tests/versions/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/versions/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/versions/test_cpv_sort_key.py b/portage_with_autodep/pym/portage/tests/versions/test_cpv_sort_key.py deleted file mode 100644 index a223d78..0000000 --- a/portage_with_autodep/pym/portage/tests/versions/test_cpv_sort_key.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2010 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.versions import cpv_sort_key - -class CpvSortKeyTestCase(TestCase): - - def testCpvSortKey(self): - - tests = [ (("a/b-2_alpha", "a", "b", "a/b-2", "a/a-1", "a/b-1"), - ( "a", "a/a-1", "a/b-1", "a/b-2_alpha", "a/b-2", "b")), - ] - - for test in tests: - self.assertEqual( tuple(sorted(test[0], key=cpv_sort_key())), test[1] ) diff --git a/portage_with_autodep/pym/portage/tests/versions/test_vercmp.py b/portage_with_autodep/pym/portage/tests/versions/test_vercmp.py deleted file mode 100644 index aa7969c..0000000 --- a/portage_with_autodep/pym/portage/tests/versions/test_vercmp.py +++ /dev/null @@ -1,80 +0,0 @@ -# test_vercmp.py -- Portage Unit Testing Functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from portage.tests import TestCase -from portage.versions import vercmp - -class VerCmpTestCase(TestCase): - """ A simple testCase for portage.versions.vercmp() - """ - - def testVerCmpGreater(self): - - tests = [ ( "6.0", "5.0"), ("5.0","5"), - ("1.0-r1", "1.0-r0"), - ("1.0-r1", "1.0"), - ("cvs.9999", "9999"), - ("999999999999999999999999999999", "999999999999999999999999999998"), - ("1.0.0", "1.0"), - ("1.0.0", "1.0b"), - ("1b", "1"), - ("1b_p1", "1_p1"), - ("1.1b", "1.1"), - ("12.2.5", "12.2b"), - ] - for test in tests: - self.assertFalse( vercmp( test[0], test[1] ) <= 0, msg="%s < %s? Wrong!" % (test[0],test[1]) ) - - def testVerCmpLess(self): - """ - pre < alpha < beta < rc < p -> test each of these, they are inductive (or should be..) - """ - tests = [ ( "4.0", "5.0"), ("5", "5.0"), ("1.0_pre2","1.0_p2"), - ("1.0_alpha2", "1.0_p2"),("1.0_alpha1", "1.0_beta1"),("1.0_beta3","1.0_rc3"), - ("1.001000000000000000001", "1.001000000000000000002"), - ("1.00100000000", "1.0010000000000000001"), - ("9999", "cvs.9999"), - ("999999999999999999999999999998", "999999999999999999999999999999"), - ("1.01", "1.1"), - ("1.0-r0", "1.0-r1"), - ("1.0", "1.0-r1"), - ("1.0", "1.0.0"), - ("1.0b", "1.0.0"), - ("1_p1", "1b_p1"), - ("1", "1b"), - ("1.1", "1.1b"), - ("12.2b", "12.2.5"), - ] - for test in tests: - self.assertFalse( vercmp( test[0], test[1]) >= 0, msg="%s > %s? Wrong!" % (test[0],test[1])) - - - def testVerCmpEqual(self): - - tests = [ ("4.0", "4.0"), - ("1.0", "1.0"), - ("1.0-r0", "1.0"), - ("1.0", "1.0-r0"), - ("1.0-r0", "1.0-r0"), - ("1.0-r1", "1.0-r1")] - for test in tests: - self.assertFalse( vercmp( test[0], test[1]) != 0, msg="%s != %s? Wrong!" % (test[0],test[1])) - - def testVerNotEqual(self): - - tests = [ ("1","2"),("1.0_alpha","1.0_pre"),("1.0_beta","1.0_alpha"), - ("0", "0.0"), - ("cvs.9999", "9999"), - ("1.0-r0", "1.0-r1"), - ("1.0-r1", "1.0-r0"), - ("1.0", "1.0-r1"), - ("1.0-r1", "1.0"), - ("1.0", "1.0.0"), - ("1_p1", "1b_p1"), - ("1b", "1"), - ("1.1b", "1.1"), - ("12.2b", "12.2"), - ] - for test in tests: - self.assertFalse( vercmp( test[0], test[1]) == 0, msg="%s == %s? Wrong!" % (test[0],test[1])) diff --git a/portage_with_autodep/pym/portage/tests/xpak/__init__.py b/portage_with_autodep/pym/portage/tests/xpak/__init__.py deleted file mode 100644 index 9c3f524..0000000 --- a/portage_with_autodep/pym/portage/tests/xpak/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# tests/portage.dep/__init__.py -- Portage Unit Test functionality -# Copyright 2006 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/tests/xpak/__test__ b/portage_with_autodep/pym/portage/tests/xpak/__test__ deleted file mode 100644 index e69de29..0000000 --- a/portage_with_autodep/pym/portage/tests/xpak/__test__ +++ /dev/null diff --git a/portage_with_autodep/pym/portage/tests/xpak/test_decodeint.py b/portage_with_autodep/pym/portage/tests/xpak/test_decodeint.py deleted file mode 100644 index 2da5735..0000000 --- a/portage_with_autodep/pym/portage/tests/xpak/test_decodeint.py +++ /dev/null @@ -1,16 +0,0 @@ -# xpak/test_decodeint.py -# Copright Gentoo Foundation 2006 -# Portage Unit Testing Functionality - -from portage.tests import TestCase -from portage.xpak import decodeint, encodeint - -class testDecodeIntTestCase(TestCase): - - def testDecodeInt(self): - - for n in range(1000): - self.assertEqual(decodeint(encodeint(n)), n) - - for n in (2 ** 32 - 1,): - self.assertEqual(decodeint(encodeint(n)), n) diff --git a/portage_with_autodep/pym/portage/update.py b/portage_with_autodep/pym/portage/update.py index 52ab506..34e4663 100644 --- a/portage_with_autodep/pym/portage/update.py +++ b/portage_with_autodep/pym/portage/update.py @@ -86,10 +86,11 @@ def fixdbentries(update_iter, dbdir): mydata = {} for myfile in [f for f in os.listdir(dbdir) if f not in ignored_dbentries]: file_path = os.path.join(dbdir, myfile) - mydata[myfile] = io.open(_unicode_encode(file_path, + with io.open(_unicode_encode(file_path, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], - errors='replace').read() + errors='replace') as f: + mydata[myfile] = f.read() updated_items = update_dbentries(update_iter, mydata) for myfile, mycontent in updated_items.items(): file_path = os.path.join(dbdir, myfile) @@ -132,10 +133,11 @@ def grab_updates(updpath, prev_mtimes=None): if update_data or \ file_path not in prev_mtimes or \ long(prev_mtimes[file_path]) != mystat[stat.ST_MTIME]: - content = io.open(_unicode_encode(file_path, + f = io.open(_unicode_encode(file_path, encoding=_encodings['fs'], errors='strict'), - mode='r', encoding=_encodings['repo.content'], errors='replace' - ).read() + mode='r', encoding=_encodings['repo.content'], errors='replace') + content = f.read() + f.close() update_data.append((file_path, mystat, content)) return update_data @@ -155,6 +157,7 @@ def parse_updates(mycontent): if len(mysplit) != 3: errors.append(_("ERROR: Update command invalid '%s'") % myline) continue + valid = True for i in (1, 2): try: atom = Atom(mysplit[i]) @@ -168,7 +171,11 @@ def parse_updates(mycontent): else: errors.append( _("ERROR: Malformed update entry '%s'") % myline) + valid = False break + if not valid: + continue + if mysplit[0] == "slotmove": if len(mysplit)!=4: errors.append(_("ERROR: Update command invalid '%s'") % myline) @@ -252,14 +259,19 @@ def update_config_files(config_root, protect, protect_mask, update_iter, match_c recursivefiles.append(x) myxfiles = recursivefiles for x in myxfiles: + f = None try: - file_contents[x] = io.open( + f = io.open( _unicode_encode(os.path.join(abs_user_config, x), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], - errors='replace').readlines() + errors='replace') + file_contents[x] = f.readlines() except IOError: continue + finally: + if f is not None: + f.close() # update /etc/portage/packages.* ignore_line_re = re.compile(r'^#|^\s*$') diff --git a/portage_with_autodep/pym/portage/update.pyo b/portage_with_autodep/pym/portage/update.pyo Binary files differnew file mode 100644 index 0000000..9628ea9 --- /dev/null +++ b/portage_with_autodep/pym/portage/update.pyo diff --git a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py index 5cb9747..69bd58a 100644 --- a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py +++ b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.py @@ -14,7 +14,7 @@ def ExtractKernelVersion(base_dir): @param base_dir: Path to sources (usually /usr/src/linux) @type base_dir: string @rtype: tuple( version[string], error[string]) - @returns: + @return: 1. tuple( version[string], error[string]) Either version or error is populated (but never both) @@ -37,6 +37,8 @@ def ExtractKernelVersion(base_dir): return (None, str(details)) except IOError as details: return (None, str(details)) + finally: + f.close() lines = [l.strip() for l in lines] diff --git a/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo Binary files differnew file mode 100644 index 0000000..d0302fd --- /dev/null +++ b/portage_with_autodep/pym/portage/util/ExtractKernelVersion.pyo diff --git a/portage_with_autodep/pym/_emerge/SlotObject.py b/portage_with_autodep/pym/portage/util/SlotObject.py index fdc6f35..a59dfc1 100644 --- a/portage_with_autodep/pym/_emerge/SlotObject.py +++ b/portage_with_autodep/pym/portage/util/SlotObject.py @@ -1,4 +1,4 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 class SlotObject(object): @@ -15,9 +15,18 @@ class SlotObject(object): if not slots: continue for myattr in slots: - myvalue = kwargs.get(myattr, None) + myvalue = kwargs.pop(myattr, None) + if myvalue is None and getattr(self, myattr, None) is not None: + raise AssertionError( + "class '%s' duplicates '%s' value in __slots__ of base class '%s'" % + (self.__class__.__name__, myattr, c.__name__)) setattr(self, myattr, myvalue) + if kwargs: + raise TypeError( + "'%s' is an invalid keyword argument for this constructor" % + (next(iter(kwargs)),)) + def copy(self): """ Create a new instance and copy all attributes diff --git a/portage_with_autodep/pym/portage/util/SlotObject.pyo b/portage_with_autodep/pym/portage/util/SlotObject.pyo Binary files differnew file mode 100644 index 0000000..11d0ec7 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/SlotObject.pyo diff --git a/portage_with_autodep/pym/portage/util/_ShelveUnicodeWrapper.py b/portage_with_autodep/pym/portage/util/_ShelveUnicodeWrapper.py new file mode 100644 index 0000000..adbd519 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_ShelveUnicodeWrapper.py @@ -0,0 +1,45 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +class ShelveUnicodeWrapper(object): + """ + Convert unicode to str and back again, since python-2.x shelve + module doesn't support unicode. + """ + def __init__(self, shelve_instance): + self._shelve = shelve_instance + + def _encode(self, s): + if isinstance(s, unicode): + s = s.encode('utf_8') + return s + + def __len__(self): + return len(self._shelve) + + def __contains__(self, k): + return self._encode(k) in self._shelve + + def __iter__(self): + return self._shelve.__iter__() + + def items(self): + return self._shelve.iteritems() + + def __setitem__(self, k, v): + self._shelve[self._encode(k)] = self._encode(v) + + def __getitem__(self, k): + return self._shelve[self._encode(k)] + + def __delitem__(self, k): + del self._shelve[self._encode(k)] + + def get(self, k, *args): + return self._shelve.get(self._encode(k), *args) + + def close(self): + self._shelve.close() + + def clear(self): + self._shelve.clear() diff --git a/portage_with_autodep/pym/portage/util/__init__.py b/portage_with_autodep/pym/portage/util/__init__.py index 4aa63d5..2e0a32b 100644 --- a/portage_with_autodep/pym/portage/util/__init__.py +++ b/portage_with_autodep/pym/portage/util/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2004-2011 Gentoo Foundation +# Copyright 2004-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['apply_permissions', 'apply_recursive_permissions', @@ -25,6 +25,7 @@ import stat import string import sys import traceback +import glob import portage portage.proxy.lazyimport.lazyimport(globals(), @@ -40,7 +41,7 @@ from portage import _os_merge from portage import _unicode_encode from portage import _unicode_decode from portage.exception import InvalidAtom, PortageException, FileNotFound, \ - OperationNotPermitted, PermissionDenied, ReadOnlyFileSystem + OperationNotPermitted, ParseError, PermissionDenied, ReadOnlyFileSystem from portage.localization import _ from portage.proxy.objectproxy import ObjectProxy from portage.cache.mappings import UserDict @@ -318,11 +319,11 @@ def stack_lists(lists, incremental=1, remember_source_file=False, for source_file, tokens in unmatched_removals.items(): if len(tokens) > 3: selected = [tokens.pop(), tokens.pop(), tokens.pop()] - writemsg(_("--- Unmatch removal atoms in %s: %s and %s more\n") % \ + writemsg(_("--- Unmatched removal atoms in %s: %s and %s more\n") % \ (source_file, ", ".join(selected), len(tokens)), noiselevel=-1) else: - writemsg(_("--- Unmatch removal atom(s) in %s: %s\n") % (source_file, ", ".join(tokens)), + writemsg(_("--- Unmatched removal atom(s) in %s: %s\n") % (source_file, ", ".join(tokens)), noiselevel=-1) if remember_source_file: @@ -345,12 +346,11 @@ def grabdict(myfilename, juststrings=0, empty=0, recursive=0, incremental=1): @param incremental: Append to the return list, don't overwrite @type incremental: Boolean (integer) @rtype: Dictionary - @returns: + @return: 1. Returns the lines in a file in a dictionary, for example: 'sys-apps/portage x86 amd64 ppc' would return { "sys-apps/portage" : [ 'x86', 'amd64', 'ppc' ] - the line syntax is key : [list of values] """ newdict={} for x in grablines(myfilename, recursive): @@ -387,7 +387,9 @@ def read_corresponding_eapi_file(filename): default = "0" eapi_file = os.path.join(os.path.dirname(filename), "eapi") try: - f = open(eapi_file, "r") + f = io.open(_unicode_encode(eapi_file, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace') lines = f.readlines() if len(lines) == 1: eapi = lines[0].rstrip("\n") @@ -503,14 +505,15 @@ def writedict(mydict,myfilename,writekey=True): def shlex_split(s): """ - This is equivalent to shlex.split but it temporarily encodes unicode - strings to bytes since shlex.split() doesn't handle unicode strings. + This is equivalent to shlex.split, but if the current interpreter is + python2, it temporarily encodes unicode strings to bytes since python2's + shlex.split() doesn't handle unicode strings. """ - is_unicode = sys.hexversion < 0x3000000 and isinstance(s, unicode) - if is_unicode: + convert_to_bytes = sys.hexversion < 0x3000000 and not isinstance(s, bytes) + if convert_to_bytes: s = _unicode_encode(s) rval = shlex.split(s) - if is_unicode: + if convert_to_bytes: rval = [_unicode_decode(x) for x in rval] return rval @@ -534,16 +537,18 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): else: expand_map = {} mykeys = {} + f = None try: # NOTE: shlex doesn't support unicode objects with Python 2 # (produces spurious \0 characters). if sys.hexversion < 0x3000000: - content = open(_unicode_encode(mycfg, - encoding=_encodings['fs'], errors='strict'), 'rb').read() + f = open(_unicode_encode(mycfg, + encoding=_encodings['fs'], errors='strict'), 'rb') else: - content = open(_unicode_encode(mycfg, + f = open(_unicode_encode(mycfg, encoding=_encodings['fs'], errors='strict'), mode='r', - encoding=_encodings['content'], errors='replace').read() + encoding=_encodings['content'], errors='replace') + content = f.read() except IOError as e: if e.errno == PermissionDenied.errno: raise PermissionDenied(mycfg) @@ -552,6 +557,9 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): if e.errno not in (errno.EISDIR,): raise return None + finally: + if f is not None: + f.close() # Workaround for avoiding a silent error in shlex that is # triggered by a source statement at the end of the file @@ -565,6 +573,7 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): writemsg(("!!! " + _("Please use dos2unix to convert line endings " + \ "in config file: '%s'") + "\n") % mycfg, noiselevel=-1) + lex = None try: if tolerant: shlex_class = _tolerant_shlex @@ -588,64 +597,63 @@ def getconfig(mycfg, tolerant=0, allow_sourcing=False, expand=True): break; equ=lex.get_token() if (equ==''): - #unexpected end of file - #lex.error_leader(self.filename,lex.lineno) + msg = lex.error_leader() + _("Unexpected EOF") if not tolerant: - writemsg(_("!!! Unexpected end of config file: variable %s\n") % key, - noiselevel=-1) - raise Exception(_("ParseError: Unexpected EOF: %s: on/before line %s") % (mycfg, lex.lineno)) + raise ParseError(msg) else: + writemsg("%s\n" % msg, noiselevel=-1) return mykeys elif (equ!='='): - #invalid token - #lex.error_leader(self.filename,lex.lineno) + msg = lex.error_leader() + \ + _("Invalid token '%s' (not '=')") % (equ,) if not tolerant: - raise Exception(_("ParseError: Invalid token " - "'%s' (not '='): %s: line %s") % \ - (equ, mycfg, lex.lineno)) + raise ParseError(msg) else: + writemsg("%s\n" % msg, noiselevel=-1) return mykeys val=lex.get_token() if val is None: - #unexpected end of file - #lex.error_leader(self.filename,lex.lineno) + msg = lex.error_leader() + \ + _("Unexpected end of config file: variable '%s'") % (key,) if not tolerant: - writemsg(_("!!! Unexpected end of config file: variable %s\n") % key, - noiselevel=-1) - raise portage.exception.CorruptionError(_("ParseError: Unexpected EOF: %s: line %s") % (mycfg, lex.lineno)) + raise ParseError(msg) else: + writemsg("%s\n" % msg, noiselevel=-1) return mykeys key = _unicode_decode(key) val = _unicode_decode(val) if _invalid_var_name_re.search(key) is not None: + msg = lex.error_leader() + \ + _("Invalid variable name '%s'") % (key,) if not tolerant: - raise Exception(_( - "ParseError: Invalid variable name '%s': line %s") % \ - (key, lex.lineno - 1)) - writemsg(_("!!! Invalid variable name '%s': line %s in %s\n") \ - % (key, lex.lineno - 1, mycfg), noiselevel=-1) + raise ParseError(msg) + writemsg("%s\n" % msg, noiselevel=-1) continue if expand: - mykeys[key] = varexpand(val, expand_map) + mykeys[key] = varexpand(val, mydict=expand_map, + error_leader=lex.error_leader) expand_map[key] = mykeys[key] else: mykeys[key] = val except SystemExit as e: raise except Exception as e: - raise portage.exception.ParseError(str(e)+" in "+mycfg) + if isinstance(e, ParseError) or lex is None: + raise + msg = _unicode_decode("%s%s") % (lex.error_leader(), e) + writemsg("%s\n" % msg, noiselevel=-1) + raise + return mykeys - -#cache expansions of constant strings -cexpand={} -def varexpand(mystring, mydict=None): + +_varexpand_word_chars = frozenset(string.ascii_letters + string.digits + "_") +_varexpand_unexpected_eof_msg = "unexpected EOF while looking for matching `}'" + +def varexpand(mystring, mydict=None, error_leader=None): if mydict is None: mydict = {} - newstring = cexpand.get(" "+mystring, None) - if newstring is not None: - return newstring """ new variable expansion code. Preserves quotes, handles \n, etc. @@ -653,36 +661,37 @@ def varexpand(mystring, mydict=None): This would be a good bunch of code to port to C. """ numvars=0 - mystring=" "+mystring #in single, double quotes insing=0 indoub=0 - pos=1 - newstring=" " - while (pos<len(mystring)): - if (mystring[pos]=="'") and (mystring[pos-1]!="\\"): + pos = 0 + length = len(mystring) + newstring = [] + while pos < length: + current = mystring[pos] + if current == "'": if (indoub): - newstring=newstring+"'" + newstring.append("'") else: - newstring += "'" # Quote removal is handled by shlex. + newstring.append("'") # Quote removal is handled by shlex. insing=not insing pos=pos+1 continue - elif (mystring[pos]=='"') and (mystring[pos-1]!="\\"): + elif current == '"': if (insing): - newstring=newstring+'"' + newstring.append('"') else: - newstring += '"' # Quote removal is handled by shlex. + newstring.append('"') # Quote removal is handled by shlex. indoub=not indoub pos=pos+1 continue if (not insing): #expansion time - if (mystring[pos]=="\n"): + if current == "\n": #convert newlines to spaces - newstring=newstring+" " - pos=pos+1 - elif (mystring[pos]=="\\"): + newstring.append(" ") + pos += 1 + elif current == "\\": # For backslash expansion, this function used to behave like # echo -e, but that's not needed for our purposes. We want to # behave like bash does when expanding a variable assignment @@ -692,19 +701,27 @@ def varexpand(mystring, mydict=None): # escaped quotes here, since getconfig() uses shlex # to handle that earlier. if (pos+1>=len(mystring)): - newstring=newstring+mystring[pos] + newstring.append(current) break else: - a = mystring[pos + 1] - pos = pos + 2 - if a in ("\\", "$"): - newstring = newstring + a - elif a == "\n": + current = mystring[pos + 1] + pos += 2 + if current == "$": + newstring.append(current) + elif current == "\\": + newstring.append(current) + # BUG: This spot appears buggy, but it's intended to + # be bug-for-bug compatible with existing behavior. + if pos < length and \ + mystring[pos] in ("'", '"', "$"): + newstring.append(mystring[pos]) + pos += 1 + elif current == "\n": pass else: - newstring = newstring + mystring[pos-2:pos] + newstring.append(mystring[pos - 2:pos]) continue - elif (mystring[pos]=="$") and (mystring[pos-1]!="\\"): + elif current == "$": pos=pos+1 if mystring[pos]=="{": pos=pos+1 @@ -712,11 +729,13 @@ def varexpand(mystring, mydict=None): else: braced=False myvstart=pos - validchars=string.ascii_letters+string.digits+"_" - while mystring[pos] in validchars: + while mystring[pos] in _varexpand_word_chars: if (pos+1)>=len(mystring): if braced: - cexpand[mystring]="" + msg = _varexpand_unexpected_eof_msg + if error_leader is not None: + msg = error_leader() + msg + writemsg(msg + "\n", noiselevel=-1) return "" else: pos=pos+1 @@ -725,25 +744,33 @@ def varexpand(mystring, mydict=None): myvarname=mystring[myvstart:pos] if braced: if mystring[pos]!="}": - cexpand[mystring]="" + msg = _varexpand_unexpected_eof_msg + if error_leader is not None: + msg = error_leader() + msg + writemsg(msg + "\n", noiselevel=-1) return "" else: pos=pos+1 if len(myvarname)==0: - cexpand[mystring]="" + msg = "$" + if braced: + msg += "{}" + msg += ": bad substitution" + if error_leader is not None: + msg = error_leader() + msg + writemsg(msg + "\n", noiselevel=-1) return "" numvars=numvars+1 if myvarname in mydict: - newstring=newstring+mydict[myvarname] + newstring.append(mydict[myvarname]) else: - newstring=newstring+mystring[pos] - pos=pos+1 + newstring.append(current) + pos += 1 else: - newstring=newstring+mystring[pos] - pos=pos+1 - if numvars==0: - cexpand[mystring]=newstring[1:] - return newstring[1:] + newstring.append(current) + pos += 1 + + return "".join(newstring) # broken and removed, but can still be imported pickle_write = None @@ -1589,13 +1616,27 @@ def find_updated_config_files(target_root, config_protect): else: yield (x, None) +_ld_so_include_re = re.compile(r'^include\s+(\S.*)') + def getlibpaths(root, env=None): + def read_ld_so_conf(path): + for l in grabfile(path): + include_match = _ld_so_include_re.match(l) + if include_match is not None: + subpath = os.path.join(os.path.dirname(path), + include_match.group(1)) + for p in glob.glob(subpath): + for r in read_ld_so_conf(p): + yield r + else: + yield l + """ Return a list of paths that are used for library lookups """ if env is None: env = os.environ # the following is based on the information from ld.so(8) rval = env.get("LD_LIBRARY_PATH", "").split(":") - rval.extend(grabfile(os.path.join(root, "etc", "ld.so.conf"))) + rval.extend(read_ld_so_conf(os.path.join(root, "etc", "ld.so.conf"))) rval.append("/usr/lib") rval.append("/lib") diff --git a/portage_with_autodep/pym/portage/util/__init__.pyo b/portage_with_autodep/pym/portage/util/__init__.pyo Binary files differnew file mode 100644 index 0000000..941b286 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_argparse.py b/portage_with_autodep/pym/portage/util/_argparse.py new file mode 100644 index 0000000..6ca7852 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_argparse.py @@ -0,0 +1,42 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +__all__ = ['ArgumentParser'] + +try: + from argparse import ArgumentParser +except ImportError: + # Compatibility with Python 2.6 and 3.1 + from optparse import OptionGroup, OptionParser + + from portage.localization import _ + + class ArgumentParser(object): + def __init__(self, **kwargs): + add_help = kwargs.pop("add_help", None) + if add_help is not None: + kwargs["add_help_option"] = add_help + parser = OptionParser(**kwargs) + self._parser = parser + self.add_argument = parser.add_option + self.print_help = parser.print_help + self.error = parser.error + + def add_argument_group(self, title=None, **kwargs): + optiongroup = OptionGroup(self._parser, title, **kwargs) + self._parser.add_option_group(optiongroup) + return _ArgumentGroup(optiongroup) + + def parse_known_args(self, args=None, namespace=None): + return self._parser.parse_args(args, namespace) + + def parse_args(self, args=None, namespace=None): + args, argv = self.parse_known_args(args, namespace) + if argv: + msg = _('unrecognized arguments: %s') + self.error(msg % ' '.join(argv)) + return args + + class _ArgumentGroup(object): + def __init__(self, optiongroup): + self.add_argument = optiongroup.add_option diff --git a/portage_with_autodep/pym/portage/util/_async/AsyncScheduler.py b/portage_with_autodep/pym/portage/util/_async/AsyncScheduler.py new file mode 100644 index 0000000..9b96c6f --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/AsyncScheduler.py @@ -0,0 +1,102 @@ +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from _emerge.AsynchronousTask import AsynchronousTask +from _emerge.PollScheduler import PollScheduler + +class AsyncScheduler(AsynchronousTask, PollScheduler): + + def __init__(self, max_jobs=None, max_load=None, **kwargs): + AsynchronousTask.__init__(self) + PollScheduler.__init__(self, **kwargs) + + if max_jobs is None: + max_jobs = 1 + self._max_jobs = max_jobs + self._max_load = max_load + self._error_count = 0 + self._running_tasks = set() + self._remaining_tasks = True + self._term_check_id = None + self._loadavg_check_id = None + + def _poll(self): + if not (self._is_work_scheduled() or self._keep_scheduling()): + self.wait() + return self.returncode + + def _cancel(self): + self._terminated.set() + self._termination_check() + + def _terminate_tasks(self): + for task in list(self._running_tasks): + task.cancel() + + def _next_task(self): + raise NotImplementedError(self) + + def _keep_scheduling(self): + return self._remaining_tasks and not self._terminated.is_set() + + def _running_job_count(self): + return len(self._running_tasks) + + def _schedule_tasks(self): + while self._keep_scheduling() and self._can_add_job(): + try: + task = self._next_task() + except StopIteration: + self._remaining_tasks = False + else: + self._running_tasks.add(task) + task.scheduler = self._sched_iface + task.addExitListener(self._task_exit) + task.start() + + # Triggers cleanup and exit listeners if there's nothing left to do. + self.poll() + + def _task_exit(self, task): + self._running_tasks.discard(task) + if task.returncode != os.EX_OK: + self._error_count += 1 + self._schedule() + + def _start(self): + self._term_check_id = self._event_loop.idle_add(self._termination_check) + if self._max_load is not None and \ + self._loadavg_latency is not None and \ + (self._max_jobs is True or self._max_jobs > 1): + # We have to schedule periodically, in case the load + # average has changed since the last call. + self._loadavg_check_id = self._event_loop.timeout_add( + self._loadavg_latency, self._schedule) + self._schedule() + + def _wait(self): + # Loop while there are jobs to be scheduled. + while self._keep_scheduling(): + self._event_loop.iteration() + + # Clean shutdown of previously scheduled jobs. In the + # case of termination, this allows for basic cleanup + # such as flushing of buffered output to logs. + while self._is_work_scheduled(): + self._event_loop.iteration() + + if self._term_check_id is not None: + self._event_loop.source_remove(self._term_check_id) + self._term_check_id = None + + if self._loadavg_check_id is not None: + self._event_loop.source_remove(self._loadavg_check_id) + self._loadavg_check_id = None + + if self._error_count > 0: + self.returncode = 1 + else: + self.returncode = os.EX_OK + + return self.returncode diff --git a/portage_with_autodep/pym/portage/util/_async/FileCopier.py b/portage_with_autodep/pym/portage/util/_async/FileCopier.py new file mode 100644 index 0000000..27e5ab4 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/FileCopier.py @@ -0,0 +1,17 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from portage import shutil +from portage.util._async.ForkProcess import ForkProcess + +class FileCopier(ForkProcess): + """ + Asynchronously copy a file. + """ + + __slots__ = ('src_path', 'dest_path') + + def _run(self): + shutil.copy(self.src_path, self.dest_path) + return os.EX_OK diff --git a/portage_with_autodep/pym/portage/util/_async/FileDigester.py b/portage_with_autodep/pym/portage/util/_async/FileDigester.py new file mode 100644 index 0000000..881c692 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/FileDigester.py @@ -0,0 +1,73 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage import os +from portage.checksum import perform_multiple_checksums +from portage.util._async.ForkProcess import ForkProcess +from _emerge.PipeReader import PipeReader + +class FileDigester(ForkProcess): + """ + Asynchronously generate file digests. Pass in file_path and + hash_names, and after successful execution, the digests + attribute will be a dict containing all of the requested + digests. + """ + + __slots__ = ('file_path', 'digests', 'hash_names', + '_digest_pipe_reader', '_digest_pw') + + def _start(self): + pr, pw = os.pipe() + self.fd_pipes = {} + self.fd_pipes[pw] = pw + self._digest_pw = pw + self._digest_pipe_reader = PipeReader( + input_files={"input":pr}, + scheduler=self.scheduler) + self._digest_pipe_reader.addExitListener(self._digest_pipe_reader_exit) + self._digest_pipe_reader.start() + ForkProcess._start(self) + os.close(pw) + + def _run(self): + digests = perform_multiple_checksums(self.file_path, + hashes=self.hash_names) + + buf = "".join("%s=%s\n" % item + for item in digests.items()).encode('utf_8') + + while buf: + buf = buf[os.write(self._digest_pw, buf):] + + return os.EX_OK + + def _parse_digests(self, data): + + digests = {} + for line in data.decode('utf_8').splitlines(): + parts = line.split('=', 1) + if len(parts) == 2: + digests[parts[0]] = parts[1] + + self.digests = digests + + def _pipe_logger_exit(self, pipe_logger): + # Ignore this event, since we want to ensure that we + # exit only after _digest_pipe_reader has reached EOF. + self._pipe_logger = None + + def _digest_pipe_reader_exit(self, pipe_reader): + self._parse_digests(pipe_reader.getvalue()) + self._digest_pipe_reader = None + self._unregister() + self.wait() + + def _unregister(self): + ForkProcess._unregister(self) + + pipe_reader = self._digest_pipe_reader + if pipe_reader is not None: + self._digest_pipe_reader = None + pipe_reader.removeExitListener(self._digest_pipe_reader_exit) + pipe_reader.cancel() diff --git a/portage_with_autodep/pym/portage/util/_async/ForkProcess.py b/portage_with_autodep/pym/portage/util/_async/ForkProcess.py new file mode 100644 index 0000000..25f72d3 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/ForkProcess.py @@ -0,0 +1,65 @@ +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import signal +import sys +import traceback + +import portage +from portage import os +from _emerge.SpawnProcess import SpawnProcess + +class ForkProcess(SpawnProcess): + + __slots__ = () + + def _spawn(self, args, fd_pipes=None, **kwargs): + """ + Fork a subprocess, apply local settings, and call fetch(). + """ + + parent_pid = os.getpid() + pid = None + try: + pid = os.fork() + + if pid != 0: + if not isinstance(pid, int): + raise AssertionError( + "fork returned non-integer: %s" % (repr(pid),)) + return [pid] + + rval = 1 + try: + + # Use default signal handlers in order to avoid problems + # killing subprocesses as reported in bug #353239. + signal.signal(signal.SIGINT, signal.SIG_DFL) + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + portage.locks._close_fds() + # We don't exec, so use close_fds=False + # (see _setup_pipes docstring). + portage.process._setup_pipes(fd_pipes, close_fds=False) + + rval = self._run() + except SystemExit: + raise + except: + traceback.print_exc() + # os._exit() skips stderr flush! + sys.stderr.flush() + finally: + os._exit(rval) + + finally: + if pid == 0 or (pid is None and os.getpid() != parent_pid): + # Call os._exit() from a finally block in order + # to suppress any finally blocks from earlier + # in the call stack (see bug #345289). This + # finally block has to be setup before the fork + # in order to avoid a race condition. + os._exit(1) + + def _run(self): + raise NotImplementedError(self) diff --git a/portage_with_autodep/pym/portage/util/_async/PipeLogger.py b/portage_with_autodep/pym/portage/util/_async/PipeLogger.py new file mode 100644 index 0000000..aa605d9 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/PipeLogger.py @@ -0,0 +1,163 @@ +# Copyright 2008-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import fcntl +import errno +import gzip +import sys + +import portage +from portage import os, _encodings, _unicode_encode +from _emerge.AbstractPollTask import AbstractPollTask + +class PipeLogger(AbstractPollTask): + + """ + This can be used for logging output of a child process, + optionally outputing to log_file_path and/or stdout_fd. It can + also monitor for EOF on input_fd, which may be used to detect + termination of a child process. If log_file_path ends with + '.gz' then the log file is written with compression. + """ + + __slots__ = ("input_fd", "log_file_path", "stdout_fd") + \ + ("_log_file", "_log_file_real", "_reg_id") + + def _start(self): + + log_file_path = self.log_file_path + if log_file_path is not None: + + self._log_file = open(_unicode_encode(log_file_path, + encoding=_encodings['fs'], errors='strict'), mode='ab') + if log_file_path.endswith('.gz'): + self._log_file_real = self._log_file + self._log_file = gzip.GzipFile(filename='', mode='ab', + fileobj=self._log_file) + + portage.util.apply_secpass_permissions(log_file_path, + uid=portage.portage_uid, gid=portage.portage_gid, + mode=0o660) + + if isinstance(self.input_fd, int): + fd = self.input_fd + else: + fd = self.input_fd.fileno() + + fcntl.fcntl(fd, fcntl.F_SETFL, + fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) + + # FD_CLOEXEC is enabled by default in Python >=3.4. + if sys.hexversion < 0x3040000: + try: + fcntl.FD_CLOEXEC + except AttributeError: + pass + else: + fcntl.fcntl(fd, fcntl.F_SETFD, + fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC) + + self._reg_id = self.scheduler.io_add_watch(fd, + self._registered_events, self._output_handler) + self._registered = True + + def _cancel(self): + self._unregister() + if self.returncode is None: + self.returncode = self._cancelled_returncode + + def _wait(self): + if self.returncode is not None: + return self.returncode + self._wait_loop() + self.returncode = os.EX_OK + return self.returncode + + def _output_handler(self, fd, event): + + background = self.background + stdout_fd = self.stdout_fd + log_file = self._log_file + + while True: + buf = self._read_buf(fd, event) + + if buf is None: + # not a POLLIN event, EAGAIN, etc... + break + + if not buf: + # EOF + self._unregister() + self.wait() + break + + else: + if not background and stdout_fd is not None: + failures = 0 + stdout_buf = buf + while stdout_buf: + try: + stdout_buf = \ + stdout_buf[os.write(stdout_fd, stdout_buf):] + except OSError as e: + if e.errno != errno.EAGAIN: + raise + del e + failures += 1 + if failures > 50: + # Avoid a potentially infinite loop. In + # most cases, the failure count is zero + # and it's unlikely to exceed 1. + raise + + # This means that a subprocess has put an inherited + # stdio file descriptor (typically stdin) into + # O_NONBLOCK mode. This is not acceptable (see bug + # #264435), so revert it. We need to use a loop + # here since there's a race condition due to + # parallel processes being able to change the + # flags on the inherited file descriptor. + # TODO: When possible, avoid having child processes + # inherit stdio file descriptors from portage + # (maybe it can't be avoided with + # PROPERTIES=interactive). + fcntl.fcntl(stdout_fd, fcntl.F_SETFL, + fcntl.fcntl(stdout_fd, + fcntl.F_GETFL) ^ os.O_NONBLOCK) + + if log_file is not None: + log_file.write(buf) + log_file.flush() + + self._unregister_if_appropriate(event) + + return True + + def _unregister(self): + + if self._reg_id is not None: + self.scheduler.source_remove(self._reg_id) + self._reg_id = None + + if self.input_fd is not None: + if isinstance(self.input_fd, int): + os.close(self.input_fd) + else: + self.input_fd.close() + self.input_fd = None + + if self.stdout_fd is not None: + os.close(self.stdout_fd) + self.stdout_fd = None + + if self._log_file is not None: + self._log_file.close() + self._log_file = None + + if self._log_file_real is not None: + # Avoid "ResourceWarning: unclosed file" since python 3.2. + self._log_file_real.close() + self._log_file_real = None + + self._registered = False diff --git a/portage_with_autodep/pym/portage/util/_async/PipeReaderBlockingIO.py b/portage_with_autodep/pym/portage/util/_async/PipeReaderBlockingIO.py new file mode 100644 index 0000000..b06adf6 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/PipeReaderBlockingIO.py @@ -0,0 +1,91 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +try: + import threading +except ImportError: + # dummy_threading will not suffice + threading = None + +from portage import os +from _emerge.AbstractPollTask import AbstractPollTask + +class PipeReaderBlockingIO(AbstractPollTask): + """ + Reads output from one or more files and saves it in memory, for + retrieval via the getvalue() method. This is driven by a thread + for each input file, in order to support blocking IO. This may + be useful for using threads to handle blocking IO with Jython, + since Jython lacks the fcntl module which is needed for + non-blocking IO (see http://bugs.jython.org/issue1074). + """ + + __slots__ = ("input_files", "_read_data", "_terminate", + "_threads", "_thread_rlock") + + def _start(self): + self._terminate = threading.Event() + self._threads = {} + self._read_data = [] + + self._registered = True + self._thread_rlock = threading.RLock() + with self._thread_rlock: + for f in self.input_files.values(): + t = threading.Thread(target=self._reader_thread, args=(f,)) + t.daemon = True + t.start() + self._threads[f] = t + + def _reader_thread(self, f): + try: + terminated = self._terminate.is_set + except AttributeError: + # Jython 2.7.0a2 + terminated = self._terminate.isSet + bufsize = self._bufsize + while not terminated(): + buf = f.read(bufsize) + with self._thread_rlock: + if terminated(): + break + elif buf: + self._read_data.append(buf) + else: + del self._threads[f] + if not self._threads: + # Thread-safe callback to EventLoop + self.scheduler.idle_add(self._eof) + break + f.close() + + def _eof(self): + self._registered = False + if self.returncode is None: + self.returncode = os.EX_OK + self.wait() + return False + + def _cancel(self): + self._terminate.set() + self._registered = False + if self.returncode is None: + self.returncode = self._cancelled_returncode + self.wait() + + def _wait(self): + if self.returncode is not None: + return self.returncode + self._wait_loop() + self.returncode = os.EX_OK + return self.returncode + + def getvalue(self): + """Retrieve the entire contents""" + with self._thread_rlock: + return b''.join(self._read_data) + + def close(self): + """Free the memory buffer.""" + with self._thread_rlock: + self._read_data = None diff --git a/portage_with_autodep/pym/portage/util/_async/PopenProcess.py b/portage_with_autodep/pym/portage/util/_async/PopenProcess.py new file mode 100644 index 0000000..2fc56d2 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/PopenProcess.py @@ -0,0 +1,33 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from _emerge.SubProcess import SubProcess + +class PopenProcess(SubProcess): + + __slots__ = ("pipe_reader", "proc",) + + def _start(self): + + self.pid = self.proc.pid + self._registered = True + + if self.pipe_reader is None: + self._reg_id = self.scheduler.child_watch_add( + self.pid, self._child_watch_cb) + else: + try: + self.pipe_reader.scheduler = self.scheduler + except AttributeError: + pass + self.pipe_reader.addExitListener(self._pipe_reader_exit) + self.pipe_reader.start() + + def _pipe_reader_exit(self, pipe_reader): + self._reg_id = self.scheduler.child_watch_add( + self.pid, self._child_watch_cb) + + def _child_watch_cb(self, pid, condition, user_data=None): + self._reg_id = None + self._waitpid_cb(pid, condition) + self.wait() diff --git a/portage_with_autodep/pym/portage/util/_async/SchedulerInterface.py b/portage_with_autodep/pym/portage/util/_async/SchedulerInterface.py new file mode 100644 index 0000000..2ab668e --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/SchedulerInterface.py @@ -0,0 +1,79 @@ +# Copyright 2012-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import gzip +import errno + +from portage import _encodings +from portage import _unicode_encode +from portage.util import writemsg_level +from ..SlotObject import SlotObject + +class SchedulerInterface(SlotObject): + + _event_loop_attrs = ("IO_ERR", "IO_HUP", "IO_IN", + "IO_NVAL", "IO_OUT", "IO_PRI", + "child_watch_add", "idle_add", "io_add_watch", + "iteration", "source_remove", "timeout_add") + + __slots__ = _event_loop_attrs + ("_event_loop", "_is_background") + + def __init__(self, event_loop, is_background=None, **kwargs): + SlotObject.__init__(self, **kwargs) + self._event_loop = event_loop + if is_background is None: + is_background = self._return_false + self._is_background = is_background + for k in self._event_loop_attrs: + setattr(self, k, getattr(event_loop, k)) + + @staticmethod + def _return_false(): + return False + + def output(self, msg, log_path=None, background=None, + level=0, noiselevel=-1): + """ + Output msg to stdout if not self._is_background(). If log_path + is not None then append msg to the log (appends with + compression if the filename extension of log_path corresponds + to a supported compression type). + """ + + global_background = self._is_background() + if background is None or global_background: + # Use the global value if the task does not have a local + # background value. For example, parallel-fetch tasks run + # in the background while other tasks concurrently run in + # the foreground. + background = global_background + + msg_shown = False + if not background: + writemsg_level(msg, level=level, noiselevel=noiselevel) + msg_shown = True + + if log_path is not None: + try: + f = open(_unicode_encode(log_path, + encoding=_encodings['fs'], errors='strict'), + mode='ab') + f_real = f + except IOError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + if not msg_shown: + writemsg_level(msg, level=level, noiselevel=noiselevel) + else: + + if log_path.endswith('.gz'): + # NOTE: The empty filename argument prevents us from + # triggering a bug in python3 which causes GzipFile + # to raise AttributeError if fileobj.name is bytes + # instead of unicode. + f = gzip.GzipFile(filename='', mode='ab', fileobj=f) + + f.write(_unicode_encode(msg)) + f.close() + if f_real is not f: + f_real.close() diff --git a/portage_with_autodep/pym/portage/util/_async/TaskScheduler.py b/portage_with_autodep/pym/portage/util/_async/TaskScheduler.py new file mode 100644 index 0000000..35b3875 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/TaskScheduler.py @@ -0,0 +1,20 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from .AsyncScheduler import AsyncScheduler + +class TaskScheduler(AsyncScheduler): + + """ + A simple way to handle scheduling of AbstractPollTask instances. Simply + pass a task iterator into the constructor and call start(). Use the + poll, wait, or addExitListener methods to be notified when all of the + tasks have completed. + """ + + def __init__(self, task_iter, **kwargs): + AsyncScheduler.__init__(self, **kwargs) + self._task_iter = task_iter + + def _next_task(self): + return next(self._task_iter) diff --git a/portage_with_autodep/pym/portage/tests/resolver/__init__.py b/portage_with_autodep/pym/portage/util/_async/__init__.py index 21a391a..418ad86 100644 --- a/portage_with_autodep/pym/portage/tests/resolver/__init__.py +++ b/portage_with_autodep/pym/portage/util/_async/__init__.py @@ -1,2 +1,2 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/util/_async/run_main_scheduler.py b/portage_with_autodep/pym/portage/util/_async/run_main_scheduler.py new file mode 100644 index 0000000..10fed34 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_async/run_main_scheduler.py @@ -0,0 +1,41 @@ + +import signal + +def run_main_scheduler(scheduler): + """ + Start and run an AsyncScheduler (or compatible object), and handle + SIGINT or SIGTERM by calling its terminate() method and waiting + for it to clean up after itself. If SIGINT or SIGTERM is received, + return signum, else return None. Any previous SIGINT or SIGTERM + signal handlers are automatically saved and restored before + returning. + """ + + received_signal = [] + + def sighandler(signum, frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + received_signal.append(signum) + scheduler.terminate() + + earlier_sigint_handler = signal.signal(signal.SIGINT, sighandler) + earlier_sigterm_handler = signal.signal(signal.SIGTERM, sighandler) + + try: + scheduler.start() + scheduler.wait() + finally: + # Restore previous handlers + if earlier_sigint_handler is not None: + signal.signal(signal.SIGINT, earlier_sigint_handler) + else: + signal.signal(signal.SIGINT, signal.SIG_DFL) + if earlier_sigterm_handler is not None: + signal.signal(signal.SIGTERM, earlier_sigterm_handler) + else: + signal.signal(signal.SIGTERM, signal.SIG_DFL) + + if received_signal: + return received_signal[0] + return None diff --git a/portage_with_autodep/pym/portage/util/_ctypes.py b/portage_with_autodep/pym/portage/util/_ctypes.py new file mode 100644 index 0000000..aeceebc --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_ctypes.py @@ -0,0 +1,47 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +try: + import ctypes + import ctypes.util +except ImportError: + ctypes = None +else: + try: + ctypes.cdll + except AttributeError: + ctypes = None + +_library_names = {} + +def find_library(name): + """ + Calls ctype.util.find_library() if the ctypes module is available, + and otherwise returns None. Results are cached for future invocations. + """ + filename = _library_names.get(name) + if filename is None: + if ctypes is not None: + filename = ctypes.util.find_library(name) + if filename is None: + filename = False + _library_names[name] = filename + + if filename is False: + return None + return filename + +_library_handles = {} + +def LoadLibrary(name): + """ + Calls ctypes.cdll.LoadLibrary(name) if the ctypes module is available, + and otherwise returns None. Results are cached for future invocations. + """ + handle = _library_handles.get(name) + + if handle is None and ctypes is not None: + handle = ctypes.CDLL(name, use_errno=True) + _library_handles[name] = handle + + return handle diff --git a/portage_with_autodep/pym/portage/util/_desktop_entry.py b/portage_with_autodep/pym/portage/util/_desktop_entry.py new file mode 100644 index 0000000..7901780 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_desktop_entry.py @@ -0,0 +1,75 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import io +import subprocess +import sys + +try: + from configparser import Error as ConfigParserError, RawConfigParser +except ImportError: + from ConfigParser import Error as ConfigParserError, RawConfigParser + +from portage import _encodings, _unicode_encode, _unicode_decode + +def parse_desktop_entry(path): + """ + Parse the given file with RawConfigParser and return the + result. This may raise an IOError from io.open(), or a + ParsingError from RawConfigParser. + """ + parser = RawConfigParser() + + # use read_file/readfp in order to control decoding of unicode + try: + # Python >=3.2 + read_file = parser.read_file + except AttributeError: + read_file = parser.readfp + + with io.open(_unicode_encode(path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') as f: + read_file(f) + + return parser + +_ignored_service_errors = ( + 'error: required key "Name" in group "Desktop Entry" is not present', + 'error: key "Actions" is present in group "Desktop Entry", but the type is "Service" while this key is only valid for type "Application"', + 'error: key "MimeType" is present in group "Desktop Entry", but the type is "Service" while this key is only valid for type "Application"', +) + +def validate_desktop_entry(path): + args = ["desktop-file-validate", path] + if sys.hexversion < 0x3000000 or sys.hexversion >= 0x3020000: + # Python 3.1 does not support bytes in Popen args. + args = [_unicode_encode(x, errors='strict') for x in args] + proc = subprocess.Popen(args, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + output_lines = _unicode_decode(proc.communicate()[0]).splitlines() + proc.wait() + + if output_lines: + try: + desktop_entry = parse_desktop_entry(path) + except ConfigParserError: + pass + else: + if desktop_entry.has_section("Desktop Entry"): + try: + entry_type = desktop_entry.get("Desktop Entry", "Type") + except ConfigParserError: + pass + else: + if entry_type == "Service": + # Filter false errors for Type=Service (bug #414125). + filtered_output = [] + for line in output_lines: + if line[len(path)+2:] in _ignored_service_errors: + continue + filtered_output.append(line) + output_lines = filtered_output + + return output_lines diff --git a/portage_with_autodep/pym/portage/util/_desktop_entry.pyo b/portage_with_autodep/pym/portage/util/_desktop_entry.pyo Binary files differnew file mode 100644 index 0000000..7dec17a --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_desktop_entry.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py index 52670d9..e71ac73 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.py @@ -267,6 +267,7 @@ class LinkageMapELF(object): owner = plibs.pop(fields[1], None) lines.append((owner, "scanelf", ";".join(fields))) proc.wait() + proc.stdout.close() if plibs: # Preserved libraries that did not appear in the scanelf output. @@ -286,6 +287,13 @@ class LinkageMapELF(object): l = l.rstrip("\n") if not l: continue + if '\0' in l: + # os.stat() will raise "TypeError: must be encoded string + # without NULL bytes, not str" in this case. + writemsg_level(_("\nLine contains null byte(s) " \ + "in %s: %s\n\n") % (location, l), + level=logging.ERROR, noiselevel=-1) + continue fields = l.split(";") if len(fields) < 5: writemsg_level(_("\nWrong number of fields " \ diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo Binary files differnew file mode 100644 index 0000000..c1e5603 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/LinkageMapELF.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py b/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py index 602cf87..4bc64db 100644 --- a/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.py @@ -1,8 +1,10 @@ -# Copyright 1998-2011 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import errno +import json import logging +import stat import sys try: @@ -27,6 +29,19 @@ if sys.hexversion >= 0x3000000: class PreservedLibsRegistry(object): """ This class handles the tracking of preserved library objects """ + + # JSON read support has been available since portage-2.2.0_alpha89. + _json_write = True + + _json_write_opts = { + "ensure_ascii": False, + "indent": "\t", + "sort_keys": True + } + if sys.hexversion < 0x30200F0: + # indent only supports int number of spaces + _json_write_opts["indent"] = 4 + def __init__(self, root, filename): """ @param root: root used to check existence of paths in pruneNonExisting @@ -55,22 +70,51 @@ class PreservedLibsRegistry(object): def load(self): """ Reload the registry data from file """ self._data = None + f = None + content = None try: - self._data = pickle.load( - open(_unicode_encode(self._filename, - encoding=_encodings['fs'], errors='strict'), 'rb')) - except (ValueError, pickle.UnpicklingError) as e: - writemsg_level(_("!!! Error loading '%s': %s\n") % \ - (self._filename, e), level=logging.ERROR, noiselevel=-1) - except (EOFError, IOError) as e: - if isinstance(e, EOFError) or e.errno == errno.ENOENT: + f = open(_unicode_encode(self._filename, + encoding=_encodings['fs'], errors='strict'), 'rb') + content = f.read() + except EnvironmentError as e: + if not hasattr(e, 'errno'): + raise + elif e.errno == errno.ENOENT: pass elif e.errno == PermissionDenied.errno: raise PermissionDenied(self._filename) else: raise + finally: + if f is not None: + f.close() + + # content is empty if it's an empty lock file + if content: + try: + self._data = json.loads(_unicode_decode(content, + encoding=_encodings['repo.content'], errors='strict')) + except SystemExit: + raise + except Exception as e: + try: + self._data = pickle.loads(content) + except SystemExit: + raise + except Exception: + writemsg_level(_("!!! Error loading '%s': %s\n") % + (self._filename, e), level=logging.ERROR, + noiselevel=-1) + if self._data is None: self._data = {} + else: + for k, v in self._data.items(): + if isinstance(v, (list, tuple)) and len(v) == 3 and \ + isinstance(v[2], set): + # convert set to list, for write with JSONEncoder + self._data[k] = (v[0], v[1], list(v[2])) + self._data_orig = self._data.copy() self.pruneNonExisting() @@ -87,7 +131,12 @@ class PreservedLibsRegistry(object): return try: f = atomic_ofstream(self._filename, 'wb') - pickle.dump(self._data, f, protocol=2) + if self._json_write: + f.write(_unicode_encode( + json.dumps(self._data, **self._json_write_opts), + encoding=_encodings['repo.content'], errors='strict')) + else: + pickle.dump(self._data, f, protocol=2) f.close() except EnvironmentError as e: if e.errno != PermissionDenied.errno: @@ -128,6 +177,9 @@ class PreservedLibsRegistry(object): self._normalize_counter(self._data[cps][1]) == counter: del self._data[cps] elif len(paths) > 0: + if isinstance(paths, set): + # convert set to list, for write with JSONEncoder + paths = list(paths) self._data[cps] = (cpv, counter, paths) def unregister(self, cpv, slot, counter): @@ -145,9 +197,38 @@ class PreservedLibsRegistry(object): os = _os_merge for cps in list(self._data): - cpv, counter, paths = self._data[cps] - paths = [f for f in paths \ - if os.path.exists(os.path.join(self._root, f.lstrip(os.sep)))] + cpv, counter, _paths = self._data[cps] + + paths = [] + hardlinks = set() + symlinks = {} + for f in _paths: + f_abs = os.path.join(self._root, f.lstrip(os.sep)) + try: + lst = os.lstat(f_abs) + except OSError: + continue + if stat.S_ISLNK(lst.st_mode): + try: + symlinks[f] = os.readlink(f_abs) + except OSError: + continue + elif stat.S_ISREG(lst.st_mode): + hardlinks.add(f) + paths.append(f) + + # Only count symlinks as preserved if they still point to a hardink + # in the same directory, in order to handle cases where a tool such + # as eselect-opengl has updated the symlink to point to a hardlink + # in a different directory (see bug #406837). The unused hardlink + # is automatically found by _find_unused_preserved_libs, since the + # soname symlink no longer points to it. After the hardlink is + # removed by _remove_preserved_libs, it calls pruneNonExisting + # which eliminates the irrelevant symlink from the registry here. + for f, target in symlinks.items(): + if os.path.join(os.path.dirname(f), target) in hardlinks: + paths.append(f) + if len(paths) > 0: self._data[cps] = (cpv, counter, paths) else: @@ -161,7 +242,7 @@ class PreservedLibsRegistry(object): def getPreservedLibs(self): """ Return a mapping of packages->preserved objects. - @returns mapping of package instances to preserved objects + @return mapping of package instances to preserved objects @rtype Dict cpv->list-of-paths """ if self._data is None: diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo Binary files differnew file mode 100644 index 0000000..8cdd7cb --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/PreservedLibsRegistry.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo b/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo Binary files differnew file mode 100644 index 0000000..960b66e --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_dyn_libs/display_preserved_libs.py b/portage_with_autodep/pym/portage/util/_dyn_libs/display_preserved_libs.py new file mode 100644 index 0000000..b16478d --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_dyn_libs/display_preserved_libs.py @@ -0,0 +1,98 @@ +# Copyright 2007-2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from __future__ import print_function + +import logging + +import portage +from portage.output import colorize + +def display_preserved_libs(vardb): + + MAX_DISPLAY = 3 + + plibdata = vardb._plib_registry.getPreservedLibs() + linkmap = vardb._linkmap + consumer_map = {} + owners = {} + + try: + linkmap.rebuild() + except portage.exception.CommandNotFound as e: + portage.util.writemsg_level("!!! Command Not Found: %s\n" % (e,), + level=logging.ERROR, noiselevel=-1) + else: + search_for_owners = set() + for cpv in plibdata: + internal_plib_keys = set(linkmap._obj_key(f) \ + for f in plibdata[cpv]) + for f in plibdata[cpv]: + if f in consumer_map: + continue + consumers = [] + for c in linkmap.findConsumers(f, greedy=False): + # Filter out any consumers that are also preserved libs + # belonging to the same package as the provider. + if linkmap._obj_key(c) not in internal_plib_keys: + consumers.append(c) + consumers.sort() + consumer_map[f] = consumers + search_for_owners.update(consumers[:MAX_DISPLAY+1]) + + owners = {} + for f in search_for_owners: + owner_set = set() + for owner in linkmap.getOwners(f): + owner_dblink = vardb._dblink(owner) + if owner_dblink.exists(): + owner_set.add(owner_dblink) + if owner_set: + owners[f] = owner_set + + all_preserved = set() + all_preserved.update(*plibdata.values()) + + for cpv in plibdata: + print(colorize("WARN", ">>>") + " package: %s" % cpv) + samefile_map = {} + for f in plibdata[cpv]: + obj_key = linkmap._obj_key(f) + alt_paths = samefile_map.get(obj_key) + if alt_paths is None: + alt_paths = set() + samefile_map[obj_key] = alt_paths + alt_paths.add(f) + + for alt_paths in samefile_map.values(): + alt_paths = sorted(alt_paths) + for p in alt_paths: + print(colorize("WARN", " * ") + " - %s" % (p,)) + f = alt_paths[0] + consumers = consumer_map.get(f, []) + consumers_non_preserved = [c for c in consumers + if c not in all_preserved] + if consumers_non_preserved: + # Filter the consumers that are preserved libraries, since + # they don't need to be rebuilt (see bug #461908). + consumers = consumers_non_preserved + + if len(consumers) == MAX_DISPLAY + 1: + # Display 1 extra consumer, instead of displaying + # "used by 1 other files". + max_display = MAX_DISPLAY + 1 + else: + max_display = MAX_DISPLAY + for c in consumers[:max_display]: + if c in all_preserved: + # The owner is displayed elsewhere due to having + # its libs preserved, so distinguish this special + # case (see bug #461908). + owners_desc = "preserved" + else: + owners_desc = ", ".join(x.mycpv for x in owners.get(c, [])) + print(colorize("WARN", " * ") + " used by %s (%s)" % \ + (c, owners_desc)) + if len(consumers) > max_display: + print(colorize("WARN", " * ") + " used by %d other files" % + (len(consumers) - max_display)) diff --git a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py new file mode 100644 index 0000000..bbbce52 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.py @@ -0,0 +1,490 @@ +# Copyright 1999-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import fcntl +import logging +import os +import select +import signal +import time + +from portage.util import writemsg_level +from ..SlotObject import SlotObject +from .PollConstants import PollConstants +from .PollSelectAdapter import PollSelectAdapter + +class EventLoop(object): + + supports_multiprocessing = True + + # TODO: Find out why SIGCHLD signals aren't delivered during poll + # calls, forcing us to wakeup in order to receive them. + _sigchld_interval = 250 + + class _child_callback_class(SlotObject): + __slots__ = ("callback", "data", "pid", "source_id") + + class _idle_callback_class(SlotObject): + __slots__ = ("args", "callback", "calling", "source_id") + + class _io_handler_class(SlotObject): + __slots__ = ("args", "callback", "f", "source_id") + + class _timeout_handler_class(SlotObject): + __slots__ = ("args", "function", "calling", "interval", "source_id", + "timestamp") + + def __init__(self, main=True): + """ + @param main: If True then this is a singleton instance for use + in the main thread, otherwise it is a local instance which + can safely be use in a non-main thread (default is True, so + that global_event_loop does not need constructor arguments) + @type main: bool + """ + self._use_signal = main + self._poll_event_queue = [] + self._poll_event_handlers = {} + self._poll_event_handler_ids = {} + # Increment id for each new handler. + self._event_handler_id = 0 + self._idle_callbacks = {} + self._timeout_handlers = {} + self._timeout_interval = None + self._poll_obj = create_poll_instance() + + self.IO_ERR = PollConstants.POLLERR + self.IO_HUP = PollConstants.POLLHUP + self.IO_IN = PollConstants.POLLIN + self.IO_NVAL = PollConstants.POLLNVAL + self.IO_OUT = PollConstants.POLLOUT + self.IO_PRI = PollConstants.POLLPRI + + self._child_handlers = {} + self._sigchld_read = None + self._sigchld_write = None + self._sigchld_src_id = None + self._pid = os.getpid() + + def _poll(self, timeout=None): + """ + All poll() calls pass through here. The poll events + are added directly to self._poll_event_queue. + In order to avoid endless blocking, this raises + StopIteration if timeout is None and there are + no file descriptors to poll. + """ + + if timeout is None and \ + not self._poll_event_handlers: + raise StopIteration( + "timeout is None and there are no poll() event handlers") + + while True: + try: + self._poll_event_queue.extend(self._poll_obj.poll(timeout)) + break + except select.error as e: + # Silently handle EINTR, which is normal when we have + # received a signal such as SIGINT. + if not (e.args and e.args[0] == errno.EINTR): + writemsg_level("\n!!! select error: %s\n" % (e,), + level=logging.ERROR, noiselevel=-1) + del e + + # This typically means that we've received a SIGINT, so + # raise StopIteration in order to break out of our current + # iteration and respond appropriately to the signal as soon + # as possible. + raise StopIteration("interrupted") + + def iteration(self, *args): + """ + Like glib.MainContext.iteration(), runs a single iteration. + @type may_block: bool + @param may_block: if True the call may block waiting for an event + (default is True). + @rtype: bool + @return: True if events were dispatched. + """ + + may_block = True + + if args: + if len(args) > 1: + raise TypeError( + "expected at most 1 argument (%s given)" % len(args)) + may_block = args[0] + + event_queue = self._poll_event_queue + event_handlers = self._poll_event_handlers + events_handled = 0 + + if not event_handlers: + if self._run_timeouts(): + events_handled += 1 + if not event_handlers: + if not events_handled and may_block and \ + self._timeout_interval is not None: + # Block so that we don't waste cpu time by looping too + # quickly. This makes EventLoop useful for code that needs + # to wait for timeout callbacks regardless of whether or + # not any IO handlers are currently registered. + try: + self._poll(timeout=self._timeout_interval) + except StopIteration: + pass + if self._run_timeouts(): + events_handled += 1 + + # If any timeouts have executed, then return immediately, + # in order to minimize latency in termination of iteration + # loops that they may control. + if events_handled or not event_handlers: + return bool(events_handled) + + if not event_queue: + + if may_block: + if self._child_handlers: + if self._timeout_interval is None: + timeout = self._sigchld_interval + else: + timeout = min(self._sigchld_interval, + self._timeout_interval) + else: + timeout = self._timeout_interval + else: + timeout = 0 + + try: + self._poll(timeout=timeout) + except StopIteration: + # This can be triggered by EINTR which is caused by signals. + pass + + # NOTE: IO event handlers may be re-entrant, in case something + # like AbstractPollTask._wait_loop() needs to be called inside + # a handler for some reason. + while event_queue: + events_handled += 1 + f, event = event_queue.pop() + x = event_handlers[f] + if not x.callback(f, event, *x.args): + self.source_remove(x.source_id) + + # Run timeouts last, in order to minimize latency in + # termination of iteration loops that they may control. + if self._run_timeouts(): + events_handled += 1 + + return bool(events_handled) + + def child_watch_add(self, pid, callback, data=None): + """ + Like glib.child_watch_add(), sets callback to be called with the + user data specified by data when the child indicated by pid exits. + The signature for the callback is: + + def callback(pid, condition, user_data) + + where pid is is the child process id, condition is the status + information about the child process and user_data is data. + + @type int + @param pid: process id of a child process to watch + @type callback: callable + @param callback: a function to call + @type data: object + @param data: the optional data to pass to function + @rtype: int + @return: an integer ID + """ + self._event_handler_id += 1 + source_id = self._event_handler_id + self._child_handlers[source_id] = self._child_callback_class( + callback=callback, data=data, pid=pid, source_id=source_id) + + if self._use_signal: + if self._sigchld_read is None: + self._sigchld_read, self._sigchld_write = os.pipe() + fcntl.fcntl(self._sigchld_read, fcntl.F_SETFL, + fcntl.fcntl(self._sigchld_read, + fcntl.F_GETFL) | os.O_NONBLOCK) + + # The IO watch is dynamically registered and unregistered as + # needed, since we don't want to consider it as a valid source + # of events when there are no child listeners. It's important + # to distinguish when there are no valid sources of IO events, + # in order to avoid an endless poll call if there's no timeout. + if self._sigchld_src_id is None: + self._sigchld_src_id = self.io_add_watch( + self._sigchld_read, self.IO_IN, self._sigchld_io_cb) + signal.signal(signal.SIGCHLD, self._sigchld_sig_cb) + + # poll now, in case the SIGCHLD has already arrived + self._poll_child_processes() + return source_id + + def _sigchld_sig_cb(self, signum, frame): + # If this signal handler was not installed by the + # current process then the signal doesn't belong to + # this EventLoop instance. + if os.getpid() == self._pid: + os.write(self._sigchld_write, b'\0') + + def _sigchld_io_cb(self, fd, events): + try: + while True: + os.read(self._sigchld_read, 4096) + except OSError: + # read until EAGAIN + pass + self._poll_child_processes() + return True + + def _poll_child_processes(self): + if not self._child_handlers: + return False + + calls = 0 + + for x in list(self._child_handlers.values()): + if x.source_id not in self._child_handlers: + # it's already been called via re-entrance + continue + try: + wait_retval = os.waitpid(x.pid, os.WNOHANG) + except OSError as e: + if e.errno != errno.ECHILD: + raise + del e + self.source_remove(x.source_id) + else: + # With waitpid and WNOHANG, only check the + # first element of the tuple since the second + # element may vary (bug #337465). + if wait_retval[0] != 0: + calls += 1 + self.source_remove(x.source_id) + x.callback(x.pid, wait_retval[1], x.data) + + return bool(calls) + + def idle_add(self, callback, *args): + """ + Like glib.idle_add(), if callback returns False it is + automatically removed from the list of event sources and will + not be called again. + + @type callback: callable + @param callback: a function to call + @rtype: int + @return: an integer ID + """ + self._event_handler_id += 1 + source_id = self._event_handler_id + self._idle_callbacks[source_id] = self._idle_callback_class( + args=args, callback=callback, source_id=source_id) + return source_id + + def _run_idle_callbacks(self): + if not self._idle_callbacks: + return + # Iterate of our local list, since self._idle_callbacks can be + # modified during the exection of these callbacks. + for x in list(self._idle_callbacks.values()): + if x.source_id not in self._idle_callbacks: + # it got cancelled while executing another callback + continue + if x.calling: + # don't call it recursively + continue + x.calling = True + try: + if not x.callback(*x.args): + self.source_remove(x.source_id) + finally: + x.calling = False + + def timeout_add(self, interval, function, *args): + """ + Like glib.timeout_add(), interval argument is the number of + milliseconds between calls to your function, and your function + should return False to stop being called, or True to continue + being called. Any additional positional arguments given here + are passed to your function when it's called. + """ + self._event_handler_id += 1 + source_id = self._event_handler_id + self._timeout_handlers[source_id] = \ + self._timeout_handler_class( + interval=interval, function=function, args=args, + source_id=source_id, timestamp=time.time()) + if self._timeout_interval is None or self._timeout_interval > interval: + self._timeout_interval = interval + return source_id + + def _run_timeouts(self): + + calls = 0 + if not self._use_signal: + if self._poll_child_processes(): + calls += 1 + + self._run_idle_callbacks() + + if not self._timeout_handlers: + return bool(calls) + + ready_timeouts = [] + current_time = time.time() + for x in self._timeout_handlers.values(): + elapsed_seconds = current_time - x.timestamp + # elapsed_seconds < 0 means the system clock has been adjusted + if elapsed_seconds < 0 or \ + (x.interval - 1000 * elapsed_seconds) <= 0: + ready_timeouts.append(x) + + # Iterate of our local list, since self._timeout_handlers can be + # modified during the exection of these callbacks. + for x in ready_timeouts: + if x.source_id not in self._timeout_handlers: + # it got cancelled while executing another timeout + continue + if x.calling: + # don't call it recursively + continue + calls += 1 + x.calling = True + try: + x.timestamp = time.time() + if not x.function(*x.args): + self.source_remove(x.source_id) + finally: + x.calling = False + + return bool(calls) + + def io_add_watch(self, f, condition, callback, *args): + """ + Like glib.io_add_watch(), your function should return False to + stop being called, or True to continue being called. Any + additional positional arguments given here are passed to your + function when it's called. + + @type f: int or object with fileno() method + @param f: a file descriptor to monitor + @type condition: int + @param condition: a condition mask + @type callback: callable + @param callback: a function to call + @rtype: int + @return: an integer ID of the event source + """ + if f in self._poll_event_handlers: + raise AssertionError("fd %d is already registered" % f) + self._event_handler_id += 1 + source_id = self._event_handler_id + self._poll_event_handler_ids[source_id] = f + self._poll_event_handlers[f] = self._io_handler_class( + args=args, callback=callback, f=f, source_id=source_id) + self._poll_obj.register(f, condition) + return source_id + + def source_remove(self, reg_id): + """ + Like glib.source_remove(), this returns True if the given reg_id + is found and removed, and False if the reg_id is invalid or has + already been removed. + """ + x = self._child_handlers.pop(reg_id, None) + if x is not None: + if not self._child_handlers and self._use_signal: + signal.signal(signal.SIGCHLD, signal.SIG_DFL) + self.source_remove(self._sigchld_src_id) + self._sigchld_src_id = None + return True + idle_callback = self._idle_callbacks.pop(reg_id, None) + if idle_callback is not None: + return True + timeout_handler = self._timeout_handlers.pop(reg_id, None) + if timeout_handler is not None: + if timeout_handler.interval == self._timeout_interval: + if self._timeout_handlers: + self._timeout_interval = \ + min(x.interval for x in self._timeout_handlers.values()) + else: + self._timeout_interval = None + return True + f = self._poll_event_handler_ids.pop(reg_id, None) + if f is None: + return False + self._poll_obj.unregister(f) + if self._poll_event_queue: + # Discard any unhandled events that belong to this file, + # in order to prevent these events from being erroneously + # delivered to a future handler that is using a reallocated + # file descriptor of the same numeric value (causing + # extremely confusing bugs). + remaining_events = [] + discarded_events = False + for event in self._poll_event_queue: + if event[0] == f: + discarded_events = True + else: + remaining_events.append(event) + + if discarded_events: + self._poll_event_queue[:] = remaining_events + + del self._poll_event_handlers[f] + return True + +_can_poll_device = None + +def can_poll_device(): + """ + Test if it's possible to use poll() on a device such as a pty. This + is known to fail on Darwin. + @rtype: bool + @return: True if poll() on a device succeeds, False otherwise. + """ + + global _can_poll_device + if _can_poll_device is not None: + return _can_poll_device + + if not hasattr(select, "poll"): + _can_poll_device = False + return _can_poll_device + + try: + dev_null = open('/dev/null', 'rb') + except IOError: + _can_poll_device = False + return _can_poll_device + + p = select.poll() + p.register(dev_null.fileno(), PollConstants.POLLIN) + + invalid_request = False + for f, event in p.poll(): + if event & PollConstants.POLLNVAL: + invalid_request = True + break + dev_null.close() + + _can_poll_device = not invalid_request + return _can_poll_device + +def create_poll_instance(): + """ + Create an instance of select.poll, or an instance of + PollSelectAdapter there is no poll() implementation or + it is broken somehow. + """ + if can_poll_device(): + return select.poll() + return PollSelectAdapter() diff --git a/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo Binary files differnew file mode 100644 index 0000000..6ce2883 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/EventLoop.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.py b/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.py new file mode 100644 index 0000000..f2f5c5e --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.py @@ -0,0 +1,23 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +class GlibEventLoop(object): + + # TODO: Support multiprocessing by using a separate glib.MainContext + # instance for each process. + supports_multiprocessing = False + + def __init__(self): + import gi.repository.GLib as glib + self.IO_ERR = glib.IO_ERR + self.IO_HUP = glib.IO_HUP + self.IO_IN = glib.IO_IN + self.IO_NVAL = glib.IO_NVAL + self.IO_OUT = glib.IO_OUT + self.IO_PRI = glib.IO_PRI + self.iteration = glib.main_context_default().iteration + self.child_watch_add = glib.child_watch_add + self.idle_add = glib.idle_add + self.io_add_watch = glib.io_add_watch + self.timeout_add = glib.timeout_add + self.source_remove = glib.source_remove diff --git a/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo Binary files differnew file mode 100644 index 0000000..d3453a4 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/GlibEventLoop.pyo diff --git a/portage_with_autodep/pym/_emerge/PollConstants.py b/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.py index d0270a9..d0270a9 100644 --- a/portage_with_autodep/pym/_emerge/PollConstants.py +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.py diff --git a/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo b/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo Binary files differnew file mode 100644 index 0000000..6c7c953 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollConstants.pyo diff --git a/portage_with_autodep/pym/_emerge/PollSelectAdapter.py b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py index c11dab8..17e63d9 100644 --- a/portage_with_autodep/pym/_emerge/PollSelectAdapter.py +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.py @@ -1,9 +1,10 @@ -# Copyright 1999-2009 Gentoo Foundation +# Copyright 1999-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -from _emerge.PollConstants import PollConstants +from .PollConstants import PollConstants import select -class PollSelectAdapter(PollConstants): + +class PollSelectAdapter(object): """ Use select to emulate a poll object, for diff --git a/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo Binary files differnew file mode 100644 index 0000000..e9ecc51 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/PollSelectAdapter.pyo diff --git a/portage_with_autodep/pym/portage/tests/unicode/__init__.py b/portage_with_autodep/pym/portage/util/_eventloop/__init__.py index 21a391a..418ad86 100644 --- a/portage_with_autodep/pym/portage/tests/unicode/__init__.py +++ b/portage_with_autodep/pym/portage/util/_eventloop/__init__.py @@ -1,2 +1,2 @@ -# Copyright 2010 Gentoo Foundation +# Copyright 2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 diff --git a/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo b/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo Binary files differnew file mode 100644 index 0000000..69864a6 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/__init__.pyo diff --git a/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.py b/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.py new file mode 100644 index 0000000..502dab8 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.py @@ -0,0 +1,35 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os + +from .EventLoop import EventLoop + +_default_constructor = EventLoop +#from .GlibEventLoop import GlibEventLoop as _default_constructor + +# If _default_constructor doesn't support multiprocessing, +# then _multiprocessing_constructor is used in subprocesses. +_multiprocessing_constructor = EventLoop + +_MAIN_PID = os.getpid() +_instances = {} + +def global_event_loop(): + """ + Get a global EventLoop (or compatible object) instance which + belongs exclusively to the current process. + """ + + pid = os.getpid() + instance = _instances.get(pid) + if instance is not None: + return instance + + constructor = _default_constructor + if not constructor.supports_multiprocessing and pid != _MAIN_PID: + constructor = _multiprocessing_constructor + + instance = constructor() + _instances[pid] = instance + return instance diff --git a/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo b/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo Binary files differnew file mode 100644 index 0000000..3d57192 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_eventloop/global_event_loop.pyo diff --git a/portage_with_autodep/pym/portage/util/_get_vm_info.py b/portage_with_autodep/pym/portage/util/_get_vm_info.py new file mode 100644 index 0000000..e8ad938 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_get_vm_info.py @@ -0,0 +1,80 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import os +import platform +import subprocess + +from portage import _unicode_decode + +def get_vm_info(): + + vm_info = {} + + if platform.system() == 'Linux': + try: + proc = subprocess.Popen(["free"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + pass + else: + output = _unicode_decode(proc.communicate()[0]) + if proc.wait() == os.EX_OK: + for line in output.splitlines(): + line = line.split() + if len(line) < 2: + continue + if line[0] == "Mem:": + try: + vm_info["ram.total"] = int(line[1]) * 1024 + except ValueError: + pass + if len(line) > 3: + try: + vm_info["ram.free"] = int(line[3]) * 1024 + except ValueError: + pass + elif line[0] == "Swap:": + try: + vm_info["swap.total"] = int(line[1]) * 1024 + except ValueError: + pass + if len(line) > 3: + try: + vm_info["swap.free"] = int(line[3]) * 1024 + except ValueError: + pass + + else: + + try: + proc = subprocess.Popen(["sysctl", "-a"], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + pass + else: + output = _unicode_decode(proc.communicate()[0]) + if proc.wait() == os.EX_OK: + for line in output.splitlines(): + line = line.split(":", 1) + if len(line) != 2: + continue + line[1] = line[1].strip() + if line[0] == "hw.physmem": + try: + vm_info["ram.total"] = int(line[1]) + except ValueError: + pass + elif line[0] == "vm.swap_total": + try: + vm_info["swap.total"] = int(line[1]) + except ValueError: + pass + elif line[0] == "Free Memory Pages": + if line[1][-1] == "K": + try: + vm_info["ram.free"] = int(line[1][:-1]) * 1024 + except ValueError: + pass + + return vm_info diff --git a/portage_with_autodep/pym/portage/util/_info_files.py b/portage_with_autodep/pym/portage/util/_info_files.py new file mode 100644 index 0000000..fabf74b --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_info_files.py @@ -0,0 +1,138 @@ +# Copyright 1999-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import errno +import logging +import re +import stat +import subprocess + +import portage +from portage import os + +def chk_updated_info_files(root, infodirs, prev_mtimes): + + if os.path.exists("/usr/bin/install-info"): + out = portage.output.EOutput() + regen_infodirs = [] + for z in infodirs: + if z == '': + continue + inforoot = portage.util.normalize_path(root + z) + if os.path.isdir(inforoot) and \ + not [x for x in os.listdir(inforoot) \ + if x.startswith('.keepinfodir')]: + infomtime = os.stat(inforoot)[stat.ST_MTIME] + if inforoot not in prev_mtimes or \ + prev_mtimes[inforoot] != infomtime: + regen_infodirs.append(inforoot) + + if not regen_infodirs: + portage.util.writemsg_stdout("\n") + if portage.util.noiselimit >= 0: + out.einfo("GNU info directory index is up-to-date.") + else: + portage.util.writemsg_stdout("\n") + if portage.util.noiselimit >= 0: + out.einfo("Regenerating GNU info directory index...") + + dir_extensions = ("", ".gz", ".bz2") + icount = 0 + badcount = 0 + errmsg = "" + for inforoot in regen_infodirs: + if inforoot == '': + continue + + if not os.path.isdir(inforoot) or \ + not os.access(inforoot, os.W_OK): + continue + + file_list = os.listdir(inforoot) + file_list.sort() + dir_file = os.path.join(inforoot, "dir") + moved_old_dir = False + processed_count = 0 + for x in file_list: + if x.startswith(".") or \ + os.path.isdir(os.path.join(inforoot, x)): + continue + if x.startswith("dir"): + skip = False + for ext in dir_extensions: + if x == "dir" + ext or \ + x == "dir" + ext + ".old": + skip = True + break + if skip: + continue + if processed_count == 0: + for ext in dir_extensions: + try: + os.rename(dir_file + ext, dir_file + ext + ".old") + moved_old_dir = True + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + del e + processed_count += 1 + try: + proc = subprocess.Popen( + ['/usr/bin/install-info', + '--dir-file=%s' % os.path.join(inforoot, "dir"), + os.path.join(inforoot, x)], + env=dict(os.environ, LANG="C", LANGUAGE="C"), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except OSError: + myso = None + else: + myso = portage._unicode_decode( + proc.communicate()[0]).rstrip("\n") + proc.wait() + existsstr = "already exists, for file `" + if myso: + if re.search(existsstr, myso): + # Already exists... Don't increment the count for this. + pass + elif myso[:44] == "install-info: warning: no info dir entry in ": + # This info file doesn't contain a DIR-header: install-info produces this + # (harmless) warning (the --quiet switch doesn't seem to work). + # Don't increment the count for this. + pass + else: + badcount += 1 + errmsg += myso + "\n" + icount += 1 + + if moved_old_dir and not os.path.exists(dir_file): + # We didn't generate a new dir file, so put the old file + # back where it was originally found. + for ext in dir_extensions: + try: + os.rename(dir_file + ext + ".old", dir_file + ext) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + del e + + # Clean dir.old cruft so that they don't prevent + # unmerge of otherwise empty directories. + for ext in dir_extensions: + try: + os.unlink(dir_file + ext + ".old") + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + del e + + #update mtime so we can potentially avoid regenerating. + prev_mtimes[inforoot] = os.stat(inforoot)[stat.ST_MTIME] + + if badcount: + out.eerror("Processed %d info files; %d errors." % \ + (icount, badcount)) + portage.util.writemsg_level(errmsg, + level=logging.ERROR, noiselevel=-1) + else: + if icount > 0 and portage.util.noiselimit >= 0: + out.einfo("Processed %d info files." % (icount,)) diff --git a/portage_with_autodep/pym/portage/util/_path.py b/portage_with_autodep/pym/portage/util/_path.py new file mode 100644 index 0000000..6fbcb43 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_path.py @@ -0,0 +1,27 @@ +# Copyright 2013 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import stat + +from portage import os +from portage.exception import PermissionDenied + +def exists_raise_eaccess(path): + try: + os.stat(path) + except OSError as e: + if e.errno == PermissionDenied.errno: + raise PermissionDenied("stat('%s')" % path) + return False + else: + return True + +def isdir_raise_eaccess(path): + try: + st = os.stat(path) + except OSError as e: + if e.errno == PermissionDenied.errno: + raise PermissionDenied("stat('%s')" % path) + return False + else: + return stat.S_ISDIR(st.st_mode) diff --git a/portage_with_autodep/pym/portage/util/_pty.py b/portage_with_autodep/pym/portage/util/_pty.py index f45ff0a..11c8b92 100644 --- a/portage_with_autodep/pym/portage/util/_pty.py +++ b/portage_with_autodep/pym/portage/util/_pty.py @@ -1,144 +1,20 @@ # Copyright 2010-2011 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import array -import fcntl import platform import pty -import select -import sys import termios -from portage import os, _unicode_decode, _unicode_encode +from portage import os from portage.output import get_term_size, set_term_size -from portage.process import spawn_bash from portage.util import writemsg -def _can_test_pty_eof(): - """ - The _test_pty_eof() function seems to hang on most - kernels other than Linux. - This was reported for the following kernels which used to work fine - without this EOF test: Darwin, AIX, FreeBSD. They seem to hang on - the slave_file.close() call. Note that Python's implementation of - openpty on Solaris already caused random hangs without this EOF test - and hence is globally disabled. - @rtype: bool - @returns: True if _test_pty_eof() won't hang, False otherwise. - """ - return platform.system() in ("Linux",) - -def _test_pty_eof(fdopen_buffered=False): - """ - Returns True if this issues is fixed for the currently - running version of python: http://bugs.python.org/issue5380 - Raises an EnvironmentError from openpty() if it fails. - - NOTE: This issue is only problematic when array.fromfile() - is used, rather than os.read(). However, array.fromfile() - is preferred since it is approximately 10% faster. - - New development: It appears that array.fromfile() is usable - with python3 as long as fdopen is called with a bufsize - argument of 0. - """ - - use_fork = False - - test_string = 2 * "blah blah blah\n" - test_string = _unicode_decode(test_string, - encoding='utf_8', errors='strict') - - # may raise EnvironmentError - master_fd, slave_fd = pty.openpty() - - # Non-blocking mode is required for Darwin kernel. - fcntl.fcntl(master_fd, fcntl.F_SETFL, - fcntl.fcntl(master_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - - # Disable post-processing of output since otherwise weird - # things like \n -> \r\n transformations may occur. - mode = termios.tcgetattr(slave_fd) - mode[1] &= ~termios.OPOST - termios.tcsetattr(slave_fd, termios.TCSANOW, mode) - - # Simulate a subprocess writing some data to the - # slave end of the pipe, and then exiting. - pid = None - if use_fork: - pids = spawn_bash(_unicode_encode("echo -n '%s'" % test_string, - encoding='utf_8', errors='strict'), env=os.environ, - fd_pipes={0:sys.stdin.fileno(), 1:slave_fd, 2:slave_fd}, - returnpid=True) - if isinstance(pids, int): - os.close(master_fd) - os.close(slave_fd) - raise EnvironmentError('spawn failed') - pid = pids[0] - else: - os.write(slave_fd, _unicode_encode(test_string, - encoding='utf_8', errors='strict')) - os.close(slave_fd) - - # If using a fork, we must wait for the child here, - # in order to avoid a race condition that would - # lead to inconsistent results. - if pid is not None: - os.waitpid(pid, 0) - - if fdopen_buffered: - master_file = os.fdopen(master_fd, 'rb') - else: - master_file = os.fdopen(master_fd, 'rb', 0) - eof = False - data = [] - iwtd = [master_file] - owtd = [] - ewtd = [] - - while not eof: - - events = select.select(iwtd, owtd, ewtd) - if not events[0]: - eof = True - break - - buf = array.array('B') - try: - buf.fromfile(master_file, 1024) - except (EOFError, IOError): - eof = True - - if not buf: - eof = True - else: - data.append(_unicode_decode(buf.tostring(), - encoding='utf_8', errors='strict')) - - master_file.close() - - return test_string == ''.join(data) - -# If _test_pty_eof() can't be used for runtime detection of -# http://bugs.python.org/issue5380, openpty can't safely be used -# unless we can guarantee that the current version of python has -# been fixed (affects all current versions of python3). When -# this issue is fixed in python3, we can add another sys.hexversion -# conditional to enable openpty support in the fixed versions. -if sys.hexversion >= 0x3000000 and not _can_test_pty_eof(): - _disable_openpty = True -else: - # Disable the use of openpty on Solaris as it seems Python's openpty - # implementation doesn't play nice on Solaris with Portage's - # behaviour causing hangs/deadlocks. - # Additional note for the future: on Interix, pipes do NOT work, so - # _disable_openpty on Interix must *never* be True - _disable_openpty = platform.system() in ("SunOS",) -_tested_pty = False - -if not _can_test_pty_eof(): - # Skip _test_pty_eof() on systems where it hangs. - _tested_pty = True +# Disable the use of openpty on Solaris as it seems Python's openpty +# implementation doesn't play nice on Solaris with Portage's +# behaviour causing hangs/deadlocks. +# Additional note for the future: on Interix, pipes do NOT work, so +# _disable_openpty on Interix must *never* be True +_disable_openpty = platform.system() in ("SunOS",) _fbsd_test_pty = platform.system() == 'FreeBSD' @@ -151,24 +27,14 @@ def _create_pty_or_pipe(copy_term_size=None): then the term size will be copied to the pty. @type copy_term_size: int @rtype: tuple - @returns: A tuple of (is_pty, master_fd, slave_fd) where + @return: A tuple of (is_pty, master_fd, slave_fd) where is_pty is True if a pty was successfully allocated, and False if a normal pipe was allocated. """ got_pty = False - global _disable_openpty, _fbsd_test_pty, _tested_pty - if not (_tested_pty or _disable_openpty): - try: - if not _test_pty_eof(): - _disable_openpty = True - except EnvironmentError as e: - _disable_openpty = True - writemsg("openpty failed: '%s'\n" % str(e), - noiselevel=-1) - del e - _tested_pty = True + global _disable_openpty, _fbsd_test_pty if _fbsd_test_pty and not _disable_openpty: # Test for python openpty breakage after freebsd7 to freebsd8 diff --git a/portage_with_autodep/pym/portage/util/_pty.pyo b/portage_with_autodep/pym/portage/util/_pty.pyo Binary files differnew file mode 100644 index 0000000..70b5eb0 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_pty.pyo diff --git a/portage_with_autodep/pym/portage/util/_urlopen.py b/portage_with_autodep/pym/portage/util/_urlopen.py new file mode 100644 index 0000000..307624b --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_urlopen.py @@ -0,0 +1,42 @@ +# Copyright 2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import sys + +try: + from urllib.request import urlopen as _urlopen + import urllib.parse as urllib_parse + import urllib.request as urllib_request + from urllib.parse import splituser as urllib_parse_splituser +except ImportError: + from urllib import urlopen as _urlopen + import urlparse as urllib_parse + import urllib2 as urllib_request + from urllib import splituser as urllib_parse_splituser + +def urlopen(url): + try: + return _urlopen(url) + except SystemExit: + raise + except Exception: + if sys.hexversion < 0x3000000: + raise + parse_result = urllib_parse.urlparse(url) + if parse_result.scheme not in ("http", "https") or \ + not parse_result.username: + raise + + return _new_urlopen(url) + +def _new_urlopen(url): + # This is experimental code for bug #413983. + parse_result = urllib_parse.urlparse(url) + netloc = urllib_parse_splituser(parse_result.netloc)[1] + url = urllib_parse.urlunparse((parse_result.scheme, netloc, parse_result.path, parse_result.params, parse_result.query, parse_result.fragment)) + password_manager = urllib_request.HTTPPasswordMgrWithDefaultRealm() + if parse_result.username is not None: + password_manager.add_password(None, url, parse_result.username, parse_result.password) + auth_handler = urllib_request.HTTPBasicAuthHandler(password_manager) + opener = urllib_request.build_opener(auth_handler) + return opener.open(url) diff --git a/portage_with_autodep/pym/portage/util/_urlopen.pyo b/portage_with_autodep/pym/portage/util/_urlopen.pyo Binary files differnew file mode 100644 index 0000000..9f51de8 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/_urlopen.pyo diff --git a/portage_with_autodep/pym/portage/util/digraph.py b/portage_with_autodep/pym/portage/util/digraph.py index 1bbe10f..f3ae658 100644 --- a/portage_with_autodep/pym/portage/util/digraph.py +++ b/portage_with_autodep/pym/portage/util/digraph.py @@ -317,16 +317,23 @@ class digraph(object): """ all_cycles = [] for node in self.nodes: + # If we have multiple paths of the same length, we have to + # return them all, so that we always get the same results + # even with PYTHONHASHSEED="random" enabled. shortest_path = None + candidates = [] for child in self.child_nodes(node, ignore_priority): path = self.shortest_path(child, node, ignore_priority) if path is None: continue - if not shortest_path or len(shortest_path) > len(path): + if not shortest_path or len(shortest_path) >= len(path): shortest_path = path - if shortest_path: - if not max_length or len(shortest_path) <= max_length: - all_cycles.append(shortest_path) + candidates.append(path) + if shortest_path and \ + (not max_length or len(shortest_path) <= max_length): + for path in candidates: + if len(path) == len(shortest_path): + all_cycles.append(path) return all_cycles # Backward compatibility diff --git a/portage_with_autodep/pym/portage/util/digraph.pyo b/portage_with_autodep/pym/portage/util/digraph.pyo Binary files differnew file mode 100644 index 0000000..8e503a6 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/digraph.pyo diff --git a/portage_with_autodep/pym/portage/util/env_update.py b/portage_with_autodep/pym/portage/util/env_update.py index eb8a0d9..ace4077 100644 --- a/portage_with_autodep/pym/portage/util/env_update.py +++ b/portage_with_autodep/pym/portage/util/env_update.py @@ -19,12 +19,14 @@ from portage.process import find_binary from portage.util import atomic_ofstream, ensure_dirs, getconfig, \ normalize_path, writemsg from portage.util.listdir import listdir +from portage.dbapi.vartree import vartree +from portage.package.ebuild.config import config if sys.hexversion >= 0x3000000: long = int def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, - env=None, writemsg_level=None): + env=None, writemsg_level=None, vardbapi=None): """ Parse /etc/env.d and use it to generate /etc/profile.env, csh.env, ld.so.conf, and prelink.conf. Finally, run ldconfig. When ldconfig is @@ -39,6 +41,40 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, defaults to portage.settings["ROOT"]. @type target_root: String (Path) """ + if vardbapi is None: + if isinstance(env, config): + vardbapi = vartree(settings=env).dbapi + else: + if target_root is None: + eprefix = portage.settings["EPREFIX"] + target_root = portage.settings["ROOT"] + target_eroot = portage.settings['EROOT'] + else: + eprefix = portage.const.EPREFIX + target_eroot = os.path.join(target_root, + eprefix.lstrip(os.sep)) + target_eroot = target_eroot.rstrip(os.sep) + os.sep + if hasattr(portage, "db") and target_eroot in portage.db: + vardbapi = portage.db[target_eroot]["vartree"].dbapi + else: + settings = config(config_root=target_root, + target_root=target_root, eprefix=eprefix) + target_root = settings["ROOT"] + if env is None: + env = settings + vardbapi = vartree(settings=settings).dbapi + + # Lock the config memory file to prevent symlink creation + # in merge_contents from overlapping with env-update. + vardbapi._fs_lock() + try: + return _env_update(makelinks, target_root, prev_mtimes, contents, + env, writemsg_level) + finally: + vardbapi._fs_unlock() + +def _env_update(makelinks, target_root, prev_mtimes, contents, env, + writemsg_level): if writemsg_level is None: writemsg_level = portage.util.writemsg_level if target_root is None: @@ -46,8 +82,13 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, if prev_mtimes is None: prev_mtimes = portage.mtimedb["ldpath"] if env is None: - env = os.environ - envd_dir = os.path.join(target_root, "etc", "env.d") + settings = portage.settings + else: + settings = env + + eprefix = settings.get("EPREFIX", "") + eprefix_lstrip = eprefix.lstrip(os.sep) + envd_dir = os.path.join(target_root, eprefix_lstrip, "etc", "env.d") ensure_dirs(envd_dir, mode=0o755) fns = listdir(envd_dir, EmptyOnError=1) fns.sort() @@ -123,7 +164,8 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, they won't be overwritten by this dict.update call.""" env.update(myconfig) - ldsoconf_path = os.path.join(target_root, "etc", "ld.so.conf") + ldsoconf_path = os.path.join( + target_root, eprefix_lstrip, "etc", "ld.so.conf") try: myld = io.open(_unicode_encode(ldsoconf_path, encoding=_encodings['fs'], errors='strict'), @@ -141,8 +183,6 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, raise oldld = None - ld_cache_update=False - newld = specials["LDPATH"] if (oldld != newld): #ld.so.conf needs updating and ldconfig needs to be run @@ -152,12 +192,11 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, for x in specials["LDPATH"]: myfd.write(x + "\n") myfd.close() - ld_cache_update=True # Update prelink.conf if we are prelink-enabled if prelink_capable: - newprelink = atomic_ofstream( - os.path.join(target_root, "etc", "prelink.conf")) + newprelink = atomic_ofstream(os.path.join( + target_root, eprefix_lstrip, "etc", "prelink.conf")) newprelink.write("# prelink.conf autogenerated by env-update; make all changes to\n") newprelink.write("# contents of /etc/env.d directory\n") @@ -193,7 +232,7 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, lib_dirs = set() for lib_dir in set(specials["LDPATH"] + \ ['usr/lib','usr/lib64','usr/lib32','lib','lib64','lib32']): - x = os.path.join(target_root, lib_dir.lstrip(os.sep)) + x = os.path.join(target_root, eprefix_lstrip, lib_dir.lstrip(os.sep)) try: newldpathtime = os.stat(x)[stat.ST_MTIME] lib_dirs.add(normalize_path(x)) @@ -223,11 +262,8 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, prev_mtimes[x] = newldpathtime mtime_changed = True - if mtime_changed: - ld_cache_update = True - if makelinks and \ - not ld_cache_update and \ + not mtime_changed and \ contents is not None: libdir_contents_changed = False for mypath, mydata in contents.items(): @@ -241,12 +277,12 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, makelinks = False ldconfig = "/sbin/ldconfig" - if "CHOST" in env and "CBUILD" in env and \ - env["CHOST"] != env["CBUILD"]: - ldconfig = find_binary("%s-ldconfig" % env["CHOST"]) + if "CHOST" in settings and "CBUILD" in settings and \ + settings["CHOST"] != settings["CBUILD"]: + ldconfig = find_binary("%s-ldconfig" % settings["CHOST"]) # Only run ldconfig as needed - if (ld_cache_update or makelinks) and ldconfig: + if makelinks and ldconfig and not eprefix: # ldconfig has very different behaviour between FreeBSD and Linux if ostype == "Linux" or ostype.lower().endswith("gnu"): # We can't update links if we haven't cleaned other versions first, as @@ -272,7 +308,8 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, cenvnotice += "# GO INTO /etc/csh.cshrc NOT /etc/csh.env\n\n" #create /etc/profile.env for bash support - outfile = atomic_ofstream(os.path.join(target_root, "etc", "profile.env")) + outfile = atomic_ofstream(os.path.join( + target_root, eprefix_lstrip, "etc", "profile.env")) outfile.write(penvnotice) env_keys = [ x for x in env if x != "LDPATH" ] @@ -286,7 +323,8 @@ def env_update(makelinks=1, target_root=None, prev_mtimes=None, contents=None, outfile.close() #create /etc/csh.env for (t)csh support - outfile = atomic_ofstream(os.path.join(target_root, "etc", "csh.env")) + outfile = atomic_ofstream(os.path.join( + target_root, eprefix_lstrip, "etc", "csh.env")) outfile.write(cenvnotice) for x in env_keys: outfile.write("setenv %s '%s'\n" % (x, env[x])) diff --git a/portage_with_autodep/pym/portage/util/env_update.pyo b/portage_with_autodep/pym/portage/util/env_update.pyo Binary files differnew file mode 100644 index 0000000..ee3b187 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/env_update.pyo diff --git a/portage_with_autodep/pym/portage/util/lafilefixer.py b/portage_with_autodep/pym/portage/util/lafilefixer.py index 2b093d8..54ff20d 100644 --- a/portage_with_autodep/pym/portage/util/lafilefixer.py +++ b/portage_with_autodep/pym/portage/util/lafilefixer.py @@ -80,7 +80,7 @@ def rewrite_lafile(contents): @param contents: the contents of a libtool archive file @type contents: bytes @rtype: tuple - @returns: (True, fixed_contents) if something needed to be + @return: (True, fixed_contents) if something needed to be fixed, (False, None) otherwise. """ #Parse the 'dependency_libs' and 'inherited_linker_flags' lines. diff --git a/portage_with_autodep/pym/portage/util/lafilefixer.pyo b/portage_with_autodep/pym/portage/util/lafilefixer.pyo Binary files differnew file mode 100644 index 0000000..a6e06ab --- /dev/null +++ b/portage_with_autodep/pym/portage/util/lafilefixer.pyo diff --git a/portage_with_autodep/pym/portage/util/listdir.py b/portage_with_autodep/pym/portage/util/listdir.py index 5753d2f..c2628cb 100644 --- a/portage_with_autodep/pym/portage/util/listdir.py +++ b/portage_with_autodep/pym/portage/util/listdir.py @@ -109,7 +109,7 @@ def listdir(mypath, recursive=False, filesonly=False, ignorecvs=False, ignorelis @param dirsonly: Only return directories. @type dirsonly: Boolean @rtype: List - @returns: A list of files and directories (or just files or just directories) or an empty list. + @return: A list of files and directories (or just files or just directories) or an empty list. """ list, ftype = cacheddir(mypath, ignorecvs, ignorelist, EmptyOnError, followSymlinks) diff --git a/portage_with_autodep/pym/portage/util/listdir.pyo b/portage_with_autodep/pym/portage/util/listdir.pyo Binary files differnew file mode 100644 index 0000000..0f02d6d --- /dev/null +++ b/portage_with_autodep/pym/portage/util/listdir.pyo diff --git a/portage_with_autodep/pym/portage/util/movefile.py b/portage_with_autodep/pym/portage/util/movefile.py index 30cb6f1..10577b5 100644 --- a/portage_with_autodep/pym/portage/util/movefile.py +++ b/portage_with_autodep/pym/portage/util/movefile.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['movefile'] @@ -7,33 +7,98 @@ import errno import os as _os import shutil as _shutil import stat +import subprocess +import textwrap import portage from portage import bsd_chflags, _encodings, _os_overrides, _selinux, \ - _unicode_decode, _unicode_func_wrapper, _unicode_module_wrapper + _unicode_decode, _unicode_encode, _unicode_func_wrapper,\ + _unicode_module_wrapper from portage.const import MOVE_BINARY +from portage.exception import OperationNotSupported from portage.localization import _ from portage.process import spawn from portage.util import writemsg +def _apply_stat(src_stat, dest): + _os.chown(dest, src_stat.st_uid, src_stat.st_gid) + _os.chmod(dest, stat.S_IMODE(src_stat.st_mode)) + +if hasattr(_os, "getxattr"): + # Python >=3.3 and GNU/Linux + def _copyxattr(src, dest): + for attr in _os.listxattr(src): + try: + _os.setxattr(dest, attr, _os.getxattr(src, attr)) + raise_exception = False + except OSError: + raise_exception = True + if raise_exception: + raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) +else: + try: + import xattr + except ImportError: + xattr = None + if xattr is not None: + def _copyxattr(src, dest): + for attr in xattr.list(src): + try: + xattr.set(dest, attr, xattr.get(src, attr)) + raise_exception = False + except IOError: + raise_exception = True + if raise_exception: + raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) + else: + _devnull = open("/dev/null", "wb") + try: + subprocess.call(["getfattr", "--version"], stdout=_devnull) + subprocess.call(["setfattr", "--version"], stdout=_devnull) + _has_getfattr_and_setfattr = True + except OSError: + _has_getfattr_and_setfattr = False + _devnull.close() + if _has_getfattr_and_setfattr: + def _copyxattr(src, dest): + getfattr_process = subprocess.Popen(["getfattr", "-d", "--absolute-names", src], stdout=subprocess.PIPE) + getfattr_process.wait() + extended_attributes = getfattr_process.stdout.readlines() + getfattr_process.stdout.close() + if extended_attributes: + extended_attributes[0] = b"# file: " + _unicode_encode(dest) + b"\n" + setfattr_process = subprocess.Popen(["setfattr", "--restore=-"], stdin=subprocess.PIPE, stderr=subprocess.PIPE) + setfattr_process.communicate(input=b"".join(extended_attributes)) + if setfattr_process.returncode != 0: + raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest) + else: + def _copyxattr(src, dest): + pass + def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, hardlink_candidates=None, encoding=_encodings['fs']): """moves a file from src to dest, preserving all permissions and attributes; mtime will be preserved even when moving across filesystems. Returns true on success and false on failure. Move is atomic.""" - #print "movefile("+str(src)+","+str(dest)+","+str(newmtime)+","+str(sstat)+")" if mysettings is None: mysettings = portage.settings + src_bytes = _unicode_encode(src, encoding=encoding, errors='strict') + dest_bytes = _unicode_encode(dest, encoding=encoding, errors='strict') + xattr_enabled = "xattr" in mysettings.features selinux_enabled = mysettings.selinux_enabled() if selinux_enabled: selinux = _unicode_module_wrapper(_selinux, encoding=encoding) + _copyfile = selinux.copyfile + _rename = selinux.rename + else: + _copyfile = _shutil.copyfile + _rename = _os.rename lchown = _unicode_func_wrapper(portage.data.lchown, encoding=encoding) os = _unicode_module_wrapper(_os, encoding=encoding, overrides=_os_overrides) - shutil = _unicode_module_wrapper(_shutil, encoding=encoding) try: if not sstat: @@ -42,8 +107,9 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, except SystemExit as e: raise except Exception as e: - print(_("!!! Stating source file failed... movefile()")) - print("!!!",e) + writemsg("!!! %s\n" % _("Stating source file failed... movefile()"), + noiselevel=-1) + writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) return None destexists=1 @@ -75,9 +141,9 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, if stat.S_ISLNK(sstat[stat.ST_MODE]): try: target=os.readlink(src) - if mysettings and mysettings["D"]: - if target.find(mysettings["D"])==0: - target=target[len(mysettings["D"]):] + if mysettings and "D" in mysettings and \ + target.startswith(mysettings["D"]): + target = target[len(mysettings["D"])-1:] if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]): os.unlink(dest) try: @@ -100,9 +166,10 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, except SystemExit as e: raise except Exception as e: - print(_("!!! failed to properly create symlink:")) - print("!!!",dest,"->",target) - print("!!!",e) + writemsg("!!! %s\n" % _("failed to properly create symlink:"), + noiselevel=-1) + writemsg("!!! %s -> %s\n" % (dest, target), noiselevel=-1) + writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) return None hardlinked = False @@ -152,26 +219,40 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, except OSError as e: if e.errno != errno.EXDEV: # Some random error. - print(_("!!! Failed to move %(src)s to %(dest)s") % {"src": src, "dest": dest}) - print("!!!",e) + writemsg("!!! %s\n" % _("Failed to move %(src)s to %(dest)s") % + {"src": src, "dest": dest}, noiselevel=-1) + writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) return None # Invalid cross-device-link 'bind' mounted or actually Cross-Device if renamefailed: - didcopy=0 if stat.S_ISREG(sstat[stat.ST_MODE]): + dest_tmp = dest + "#new" + dest_tmp_bytes = _unicode_encode(dest_tmp, encoding=encoding, + errors='strict') try: # For safety copy then move it over. - if selinux_enabled: - selinux.copyfile(src, dest + "#new") - selinux.rename(dest + "#new", dest) - else: - shutil.copyfile(src,dest+"#new") - os.rename(dest+"#new",dest) - didcopy=1 + _copyfile(src_bytes, dest_tmp_bytes) + if xattr_enabled: + try: + _copyxattr(src_bytes, dest_tmp_bytes) + except SystemExit: + raise + except: + msg = _("Failed to copy extended attributes. " + "In order to avoid this error, set " + "FEATURES=\"-xattr\" in make.conf.") + msg = textwrap.wrap(msg, 65) + for line in msg: + writemsg("!!! %s\n" % (line,), noiselevel=-1) + raise + _apply_stat(sstat, dest_tmp_bytes) + _rename(dest_tmp_bytes, dest_bytes) + _os.unlink(src_bytes) except SystemExit as e: raise except Exception as e: - print(_('!!! copy %(src)s -> %(dest)s failed.') % {"src": src, "dest": dest}) - print("!!!",e) + writemsg("!!! %s\n" % _('copy %(src)s -> %(dest)s failed.') % + {"src": src, "dest": dest}, noiselevel=-1) + writemsg(_unicode_decode("!!! %s\n") % (e,), noiselevel=-1) return None else: #we don't yet handle special, so we need to fall back to /bin/mv @@ -183,21 +264,6 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None, "dest": _unicode_decode(dest, encoding=encoding)}, noiselevel=-1) writemsg("!!! %s\n" % a, noiselevel=-1) return None # failure - try: - if didcopy: - if stat.S_ISLNK(sstat[stat.ST_MODE]): - lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) - else: - os.chown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID]) - os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown - os.unlink(src) - except SystemExit as e: - raise - except Exception as e: - print(_("!!! Failed to chown/chmod/unlink in movefile()")) - print("!!!",dest) - print("!!!",e) - return None # Always use stat_obj[stat.ST_MTIME] for the integral timestamp which # is returned, since the stat_obj.st_mtime float attribute rounds *up* diff --git a/portage_with_autodep/pym/portage/util/movefile.pyo b/portage_with_autodep/pym/portage/util/movefile.pyo Binary files differnew file mode 100644 index 0000000..1228ee7 --- /dev/null +++ b/portage_with_autodep/pym/portage/util/movefile.pyo diff --git a/portage_with_autodep/pym/portage/util/mtimedb.py b/portage_with_autodep/pym/portage/util/mtimedb.py index 67f93e8..30922a9 100644 --- a/portage_with_autodep/pym/portage/util/mtimedb.py +++ b/portage_with_autodep/pym/portage/util/mtimedb.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = ['MtimeDB'] @@ -9,35 +9,77 @@ try: except ImportError: import pickle +import errno +import io +import json +import sys + import portage +from portage import _encodings +from portage import _unicode_decode from portage import _unicode_encode from portage.data import portage_gid, uid from portage.localization import _ from portage.util import apply_secpass_permissions, atomic_ofstream, writemsg class MtimeDB(dict): + + # JSON read support has been available since portage-2.1.10.49. + _json_write = True + + _json_write_opts = { + "ensure_ascii": False, + "indent": "\t", + "sort_keys": True + } + if sys.hexversion < 0x30200F0: + # indent only supports int number of spaces + _json_write_opts["indent"] = 4 + def __init__(self, filename): dict.__init__(self) self.filename = filename self._load(filename) def _load(self, filename): + f = None + content = None try: f = open(_unicode_encode(filename), 'rb') - mypickle = pickle.Unpickler(f) - try: - mypickle.find_global = None - except AttributeError: - # TODO: If py3k, override Unpickler.find_class(). + content = f.read() + except EnvironmentError as e: + if getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): pass - d = mypickle.load() - f.close() - del f - except (IOError, OSError, EOFError, ValueError, pickle.UnpicklingError) as e: - if isinstance(e, pickle.UnpicklingError): + else: writemsg(_("!!! Error loading '%s': %s\n") % \ - (filename, str(e)), noiselevel=-1) - del e + (filename, e), noiselevel=-1) + finally: + if f is not None: + f.close() + + d = None + if content: + try: + d = json.loads(_unicode_decode(content, + encoding=_encodings['repo.content'], errors='strict')) + except SystemExit: + raise + except Exception as e: + try: + mypickle = pickle.Unpickler(io.BytesIO(content)) + try: + mypickle.find_global = None + except AttributeError: + # Python >=3 + pass + d = mypickle.load() + except SystemExit: + raise + except Exception: + writemsg(_("!!! Error loading '%s': %s\n") % \ + (filename, e), noiselevel=-1) + + if d is None: d = {} if "old" in d: @@ -74,7 +116,12 @@ class MtimeDB(dict): except EnvironmentError: pass else: - pickle.dump(d, f, protocol=2) + if self._json_write: + f.write(_unicode_encode( + json.dumps(d, **self._json_write_opts), + encoding=_encodings['repo.content'], errors='strict')) + else: + pickle.dump(d, f, protocol=2) f.close() apply_secpass_permissions(self.filename, uid=uid, gid=portage_gid, mode=0o644) diff --git a/portage_with_autodep/pym/portage/util/mtimedb.pyo b/portage_with_autodep/pym/portage/util/mtimedb.pyo Binary files differnew file mode 100644 index 0000000..fda479a --- /dev/null +++ b/portage_with_autodep/pym/portage/util/mtimedb.pyo diff --git a/portage_with_autodep/pym/portage/util/whirlpool.py b/portage_with_autodep/pym/portage/util/whirlpool.py new file mode 100644 index 0000000..c696f6f --- /dev/null +++ b/portage_with_autodep/pym/portage/util/whirlpool.py @@ -0,0 +1,794 @@ +## whirlpool.py - pure Python implementation of the Whirlpool algorithm. +## Bjorn Edstrom <be@bjrn.se> 16 december 2007. +## +## Copyrights +## ========== +## +## This code is based on the reference implementation by +## Paulo S.L.M. Barreto and Vincent Rijmen. The reference implementation +## is placed in the public domain but has the following headers: +## +## * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS +## * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +## * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +## * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE +## * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +## * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +## * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +## * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +## * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +## * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +## * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +## * +## */ +## /* The code contained in this file (Whirlpool.c) is in the public domain. */ +## +## This Python implementation is therefore also placed in the public domain. + +import sys +if sys.hexversion >= 0x3000000: + xrange = range + +#block_size = 64 +digest_size = 64 +digestsize = 64 + +class Whirlpool: + """Return a new Whirlpool object. An optional string argument + may be provided; if present, this string will be automatically + hashed.""" + def __init__(self, arg=None): + self.ctx = WhirlpoolStruct() + if arg: + self.update(arg) + self.digest_status = 0 + + def update(self, arg): + """update(arg)""" + WhirlpoolAdd(arg, len(arg)*8, self.ctx) + self.digest_status = 0 + + def digest(self): + """digest()""" + if self.digest_status == 0: + self.dig = WhirlpoolFinalize(self.ctx) + self.digest_status = 1 + return self.dig + + def hexdigest(self): + """hexdigest()""" + dig = self.digest() + tempstr = '' + for d in dig: + xxx = '%02x' % (ord(d)) + tempstr = tempstr + xxx + return tempstr + + def copy(self): + """copy()""" + import copy + return copy.deepcopy(self) + + +def new(init=None): + """Return a new Whirlpool object. An optional string argument + may be provided; if present, this string will be automatically + hashed.""" + return Whirlpool(init) + +# +# Private. +# + +R = 10 + +C0 = [ +0x18186018c07830d8, 0x23238c2305af4626, 0xc6c63fc67ef991b8, 0xe8e887e8136fcdfb, +0x878726874ca113cb, 0xb8b8dab8a9626d11, 0x0101040108050209, 0x4f4f214f426e9e0d, +0x3636d836adee6c9b, 0xa6a6a2a6590451ff, 0xd2d26fd2debdb90c, 0xf5f5f3f5fb06f70e, +0x7979f979ef80f296, 0x6f6fa16f5fcede30, 0x91917e91fcef3f6d, 0x52525552aa07a4f8, +0x60609d6027fdc047, 0xbcbccabc89766535, 0x9b9b569baccd2b37, 0x8e8e028e048c018a, +0xa3a3b6a371155bd2, 0x0c0c300c603c186c, 0x7b7bf17bff8af684, 0x3535d435b5e16a80, +0x1d1d741de8693af5, 0xe0e0a7e05347ddb3, 0xd7d77bd7f6acb321, 0xc2c22fc25eed999c, +0x2e2eb82e6d965c43, 0x4b4b314b627a9629, 0xfefedffea321e15d, 0x575741578216aed5, +0x15155415a8412abd, 0x7777c1779fb6eee8, 0x3737dc37a5eb6e92, 0xe5e5b3e57b56d79e, +0x9f9f469f8cd92313, 0xf0f0e7f0d317fd23, 0x4a4a354a6a7f9420, 0xdada4fda9e95a944, +0x58587d58fa25b0a2, 0xc9c903c906ca8fcf, 0x2929a429558d527c, 0x0a0a280a5022145a, +0xb1b1feb1e14f7f50, 0xa0a0baa0691a5dc9, 0x6b6bb16b7fdad614, 0x85852e855cab17d9, +0xbdbdcebd8173673c, 0x5d5d695dd234ba8f, 0x1010401080502090, 0xf4f4f7f4f303f507, +0xcbcb0bcb16c08bdd, 0x3e3ef83eedc67cd3, 0x0505140528110a2d, 0x676781671fe6ce78, +0xe4e4b7e47353d597, 0x27279c2725bb4e02, 0x4141194132588273, 0x8b8b168b2c9d0ba7, +0xa7a7a6a7510153f6, 0x7d7de97dcf94fab2, 0x95956e95dcfb3749, 0xd8d847d88e9fad56, +0xfbfbcbfb8b30eb70, 0xeeee9fee2371c1cd, 0x7c7ced7cc791f8bb, 0x6666856617e3cc71, +0xdddd53dda68ea77b, 0x17175c17b84b2eaf, 0x4747014702468e45, 0x9e9e429e84dc211a, +0xcaca0fca1ec589d4, 0x2d2db42d75995a58, 0xbfbfc6bf9179632e, 0x07071c07381b0e3f, +0xadad8ead012347ac, 0x5a5a755aea2fb4b0, 0x838336836cb51bef, 0x3333cc3385ff66b6, +0x636391633ff2c65c, 0x02020802100a0412, 0xaaaa92aa39384993, 0x7171d971afa8e2de, +0xc8c807c80ecf8dc6, 0x19196419c87d32d1, 0x494939497270923b, 0xd9d943d9869aaf5f, +0xf2f2eff2c31df931, 0xe3e3abe34b48dba8, 0x5b5b715be22ab6b9, 0x88881a8834920dbc, +0x9a9a529aa4c8293e, 0x262698262dbe4c0b, 0x3232c8328dfa64bf, 0xb0b0fab0e94a7d59, +0xe9e983e91b6acff2, 0x0f0f3c0f78331e77, 0xd5d573d5e6a6b733, 0x80803a8074ba1df4, +0xbebec2be997c6127, 0xcdcd13cd26de87eb, 0x3434d034bde46889, 0x48483d487a759032, +0xffffdbffab24e354, 0x7a7af57af78ff48d, 0x90907a90f4ea3d64, 0x5f5f615fc23ebe9d, +0x202080201da0403d, 0x6868bd6867d5d00f, 0x1a1a681ad07234ca, 0xaeae82ae192c41b7, +0xb4b4eab4c95e757d, 0x54544d549a19a8ce, 0x93937693ece53b7f, 0x222288220daa442f, +0x64648d6407e9c863, 0xf1f1e3f1db12ff2a, 0x7373d173bfa2e6cc, 0x12124812905a2482, +0x40401d403a5d807a, 0x0808200840281048, 0xc3c32bc356e89b95, 0xecec97ec337bc5df, +0xdbdb4bdb9690ab4d, 0xa1a1bea1611f5fc0, 0x8d8d0e8d1c830791, 0x3d3df43df5c97ac8, +0x97976697ccf1335b, 0x0000000000000000, 0xcfcf1bcf36d483f9, 0x2b2bac2b4587566e, +0x7676c57697b3ece1, 0x8282328264b019e6, 0xd6d67fd6fea9b128, 0x1b1b6c1bd87736c3, +0xb5b5eeb5c15b7774, 0xafaf86af112943be, 0x6a6ab56a77dfd41d, 0x50505d50ba0da0ea, +0x45450945124c8a57, 0xf3f3ebf3cb18fb38, 0x3030c0309df060ad, 0xefef9bef2b74c3c4, +0x3f3ffc3fe5c37eda, 0x55554955921caac7, 0xa2a2b2a2791059db, 0xeaea8fea0365c9e9, +0x656589650fecca6a, 0xbabad2bab9686903, 0x2f2fbc2f65935e4a, 0xc0c027c04ee79d8e, +0xdede5fdebe81a160, 0x1c1c701ce06c38fc, 0xfdfdd3fdbb2ee746, 0x4d4d294d52649a1f, +0x92927292e4e03976, 0x7575c9758fbceafa, 0x06061806301e0c36, 0x8a8a128a249809ae, +0xb2b2f2b2f940794b, 0xe6e6bfe66359d185, 0x0e0e380e70361c7e, 0x1f1f7c1ff8633ee7, +0x6262956237f7c455, 0xd4d477d4eea3b53a, 0xa8a89aa829324d81, 0x96966296c4f43152, +0xf9f9c3f99b3aef62, 0xc5c533c566f697a3, 0x2525942535b14a10, 0x59597959f220b2ab, +0x84842a8454ae15d0, 0x7272d572b7a7e4c5, 0x3939e439d5dd72ec, 0x4c4c2d4c5a619816, +0x5e5e655eca3bbc94, 0x7878fd78e785f09f, 0x3838e038ddd870e5, 0x8c8c0a8c14860598, +0xd1d163d1c6b2bf17, 0xa5a5aea5410b57e4, 0xe2e2afe2434dd9a1, 0x616199612ff8c24e, +0xb3b3f6b3f1457b42, 0x2121842115a54234, 0x9c9c4a9c94d62508, 0x1e1e781ef0663cee, +0x4343114322528661, 0xc7c73bc776fc93b1, 0xfcfcd7fcb32be54f, 0x0404100420140824, +0x51515951b208a2e3, 0x99995e99bcc72f25, 0x6d6da96d4fc4da22, 0x0d0d340d68391a65, +0xfafacffa8335e979, 0xdfdf5bdfb684a369, 0x7e7ee57ed79bfca9, 0x242490243db44819, +0x3b3bec3bc5d776fe, 0xabab96ab313d4b9a, 0xcece1fce3ed181f0, 0x1111441188552299, +0x8f8f068f0c890383, 0x4e4e254e4a6b9c04, 0xb7b7e6b7d1517366, 0xebeb8beb0b60cbe0, +0x3c3cf03cfdcc78c1, 0x81813e817cbf1ffd, 0x94946a94d4fe3540, 0xf7f7fbf7eb0cf31c, +0xb9b9deb9a1676f18, 0x13134c13985f268b, 0x2c2cb02c7d9c5851, 0xd3d36bd3d6b8bb05, +0xe7e7bbe76b5cd38c, 0x6e6ea56e57cbdc39, 0xc4c437c46ef395aa, 0x03030c03180f061b, +0x565645568a13acdc, 0x44440d441a49885e, 0x7f7fe17fdf9efea0, 0xa9a99ea921374f88, +0x2a2aa82a4d825467, 0xbbbbd6bbb16d6b0a, 0xc1c123c146e29f87, 0x53535153a202a6f1, +0xdcdc57dcae8ba572, 0x0b0b2c0b58271653, 0x9d9d4e9d9cd32701, 0x6c6cad6c47c1d82b, +0x3131c43195f562a4, 0x7474cd7487b9e8f3, 0xf6f6fff6e309f115, 0x464605460a438c4c, +0xacac8aac092645a5, 0x89891e893c970fb5, 0x14145014a04428b4, 0xe1e1a3e15b42dfba, +0x16165816b04e2ca6, 0x3a3ae83acdd274f7, 0x6969b9696fd0d206, 0x09092409482d1241, +0x7070dd70a7ade0d7, 0xb6b6e2b6d954716f, 0xd0d067d0ceb7bd1e, 0xeded93ed3b7ec7d6, +0xcccc17cc2edb85e2, 0x424215422a578468, 0x98985a98b4c22d2c, 0xa4a4aaa4490e55ed, +0x2828a0285d885075, 0x5c5c6d5cda31b886, 0xf8f8c7f8933fed6b, 0x8686228644a411c2, +] +C1 = [ +0xd818186018c07830, 0x2623238c2305af46, 0xb8c6c63fc67ef991, 0xfbe8e887e8136fcd, +0xcb878726874ca113, 0x11b8b8dab8a9626d, 0x0901010401080502, 0x0d4f4f214f426e9e, +0x9b3636d836adee6c, 0xffa6a6a2a6590451, 0x0cd2d26fd2debdb9, 0x0ef5f5f3f5fb06f7, +0x967979f979ef80f2, 0x306f6fa16f5fcede, 0x6d91917e91fcef3f, 0xf852525552aa07a4, +0x4760609d6027fdc0, 0x35bcbccabc897665, 0x379b9b569baccd2b, 0x8a8e8e028e048c01, +0xd2a3a3b6a371155b, 0x6c0c0c300c603c18, 0x847b7bf17bff8af6, 0x803535d435b5e16a, +0xf51d1d741de8693a, 0xb3e0e0a7e05347dd, 0x21d7d77bd7f6acb3, 0x9cc2c22fc25eed99, +0x432e2eb82e6d965c, 0x294b4b314b627a96, 0x5dfefedffea321e1, 0xd5575741578216ae, +0xbd15155415a8412a, 0xe87777c1779fb6ee, 0x923737dc37a5eb6e, 0x9ee5e5b3e57b56d7, +0x139f9f469f8cd923, 0x23f0f0e7f0d317fd, 0x204a4a354a6a7f94, 0x44dada4fda9e95a9, +0xa258587d58fa25b0, 0xcfc9c903c906ca8f, 0x7c2929a429558d52, 0x5a0a0a280a502214, +0x50b1b1feb1e14f7f, 0xc9a0a0baa0691a5d, 0x146b6bb16b7fdad6, 0xd985852e855cab17, +0x3cbdbdcebd817367, 0x8f5d5d695dd234ba, 0x9010104010805020, 0x07f4f4f7f4f303f5, +0xddcbcb0bcb16c08b, 0xd33e3ef83eedc67c, 0x2d0505140528110a, 0x78676781671fe6ce, +0x97e4e4b7e47353d5, 0x0227279c2725bb4e, 0x7341411941325882, 0xa78b8b168b2c9d0b, +0xf6a7a7a6a7510153, 0xb27d7de97dcf94fa, 0x4995956e95dcfb37, 0x56d8d847d88e9fad, +0x70fbfbcbfb8b30eb, 0xcdeeee9fee2371c1, 0xbb7c7ced7cc791f8, 0x716666856617e3cc, +0x7bdddd53dda68ea7, 0xaf17175c17b84b2e, 0x454747014702468e, 0x1a9e9e429e84dc21, +0xd4caca0fca1ec589, 0x582d2db42d75995a, 0x2ebfbfc6bf917963, 0x3f07071c07381b0e, +0xacadad8ead012347, 0xb05a5a755aea2fb4, 0xef838336836cb51b, 0xb63333cc3385ff66, +0x5c636391633ff2c6, 0x1202020802100a04, 0x93aaaa92aa393849, 0xde7171d971afa8e2, +0xc6c8c807c80ecf8d, 0xd119196419c87d32, 0x3b49493949727092, 0x5fd9d943d9869aaf, +0x31f2f2eff2c31df9, 0xa8e3e3abe34b48db, 0xb95b5b715be22ab6, 0xbc88881a8834920d, +0x3e9a9a529aa4c829, 0x0b262698262dbe4c, 0xbf3232c8328dfa64, 0x59b0b0fab0e94a7d, +0xf2e9e983e91b6acf, 0x770f0f3c0f78331e, 0x33d5d573d5e6a6b7, 0xf480803a8074ba1d, +0x27bebec2be997c61, 0xebcdcd13cd26de87, 0x893434d034bde468, 0x3248483d487a7590, +0x54ffffdbffab24e3, 0x8d7a7af57af78ff4, 0x6490907a90f4ea3d, 0x9d5f5f615fc23ebe, +0x3d202080201da040, 0x0f6868bd6867d5d0, 0xca1a1a681ad07234, 0xb7aeae82ae192c41, +0x7db4b4eab4c95e75, 0xce54544d549a19a8, 0x7f93937693ece53b, 0x2f222288220daa44, +0x6364648d6407e9c8, 0x2af1f1e3f1db12ff, 0xcc7373d173bfa2e6, 0x8212124812905a24, +0x7a40401d403a5d80, 0x4808082008402810, 0x95c3c32bc356e89b, 0xdfecec97ec337bc5, +0x4ddbdb4bdb9690ab, 0xc0a1a1bea1611f5f, 0x918d8d0e8d1c8307, 0xc83d3df43df5c97a, +0x5b97976697ccf133, 0x0000000000000000, 0xf9cfcf1bcf36d483, 0x6e2b2bac2b458756, +0xe17676c57697b3ec, 0xe68282328264b019, 0x28d6d67fd6fea9b1, 0xc31b1b6c1bd87736, +0x74b5b5eeb5c15b77, 0xbeafaf86af112943, 0x1d6a6ab56a77dfd4, 0xea50505d50ba0da0, +0x5745450945124c8a, 0x38f3f3ebf3cb18fb, 0xad3030c0309df060, 0xc4efef9bef2b74c3, +0xda3f3ffc3fe5c37e, 0xc755554955921caa, 0xdba2a2b2a2791059, 0xe9eaea8fea0365c9, +0x6a656589650fecca, 0x03babad2bab96869, 0x4a2f2fbc2f65935e, 0x8ec0c027c04ee79d, +0x60dede5fdebe81a1, 0xfc1c1c701ce06c38, 0x46fdfdd3fdbb2ee7, 0x1f4d4d294d52649a, +0x7692927292e4e039, 0xfa7575c9758fbcea, 0x3606061806301e0c, 0xae8a8a128a249809, +0x4bb2b2f2b2f94079, 0x85e6e6bfe66359d1, 0x7e0e0e380e70361c, 0xe71f1f7c1ff8633e, +0x556262956237f7c4, 0x3ad4d477d4eea3b5, 0x81a8a89aa829324d, 0x5296966296c4f431, +0x62f9f9c3f99b3aef, 0xa3c5c533c566f697, 0x102525942535b14a, 0xab59597959f220b2, +0xd084842a8454ae15, 0xc57272d572b7a7e4, 0xec3939e439d5dd72, 0x164c4c2d4c5a6198, +0x945e5e655eca3bbc, 0x9f7878fd78e785f0, 0xe53838e038ddd870, 0x988c8c0a8c148605, +0x17d1d163d1c6b2bf, 0xe4a5a5aea5410b57, 0xa1e2e2afe2434dd9, 0x4e616199612ff8c2, +0x42b3b3f6b3f1457b, 0x342121842115a542, 0x089c9c4a9c94d625, 0xee1e1e781ef0663c, +0x6143431143225286, 0xb1c7c73bc776fc93, 0x4ffcfcd7fcb32be5, 0x2404041004201408, +0xe351515951b208a2, 0x2599995e99bcc72f, 0x226d6da96d4fc4da, 0x650d0d340d68391a, +0x79fafacffa8335e9, 0x69dfdf5bdfb684a3, 0xa97e7ee57ed79bfc, 0x19242490243db448, +0xfe3b3bec3bc5d776, 0x9aabab96ab313d4b, 0xf0cece1fce3ed181, 0x9911114411885522, +0x838f8f068f0c8903, 0x044e4e254e4a6b9c, 0x66b7b7e6b7d15173, 0xe0ebeb8beb0b60cb, +0xc13c3cf03cfdcc78, 0xfd81813e817cbf1f, 0x4094946a94d4fe35, 0x1cf7f7fbf7eb0cf3, +0x18b9b9deb9a1676f, 0x8b13134c13985f26, 0x512c2cb02c7d9c58, 0x05d3d36bd3d6b8bb, +0x8ce7e7bbe76b5cd3, 0x396e6ea56e57cbdc, 0xaac4c437c46ef395, 0x1b03030c03180f06, +0xdc565645568a13ac, 0x5e44440d441a4988, 0xa07f7fe17fdf9efe, 0x88a9a99ea921374f, +0x672a2aa82a4d8254, 0x0abbbbd6bbb16d6b, 0x87c1c123c146e29f, 0xf153535153a202a6, +0x72dcdc57dcae8ba5, 0x530b0b2c0b582716, 0x019d9d4e9d9cd327, 0x2b6c6cad6c47c1d8, +0xa43131c43195f562, 0xf37474cd7487b9e8, 0x15f6f6fff6e309f1, 0x4c464605460a438c, +0xa5acac8aac092645, 0xb589891e893c970f, 0xb414145014a04428, 0xbae1e1a3e15b42df, +0xa616165816b04e2c, 0xf73a3ae83acdd274, 0x066969b9696fd0d2, 0x4109092409482d12, +0xd77070dd70a7ade0, 0x6fb6b6e2b6d95471, 0x1ed0d067d0ceb7bd, 0xd6eded93ed3b7ec7, +0xe2cccc17cc2edb85, 0x68424215422a5784, 0x2c98985a98b4c22d, 0xeda4a4aaa4490e55, +0x752828a0285d8850, 0x865c5c6d5cda31b8, 0x6bf8f8c7f8933fed, 0xc28686228644a411, +] +C2 = [ +0x30d818186018c078, 0x462623238c2305af, 0x91b8c6c63fc67ef9, 0xcdfbe8e887e8136f, +0x13cb878726874ca1, 0x6d11b8b8dab8a962, 0x0209010104010805, 0x9e0d4f4f214f426e, +0x6c9b3636d836adee, 0x51ffa6a6a2a65904, 0xb90cd2d26fd2debd, 0xf70ef5f5f3f5fb06, +0xf2967979f979ef80, 0xde306f6fa16f5fce, 0x3f6d91917e91fcef, 0xa4f852525552aa07, +0xc04760609d6027fd, 0x6535bcbccabc8976, 0x2b379b9b569baccd, 0x018a8e8e028e048c, +0x5bd2a3a3b6a37115, 0x186c0c0c300c603c, 0xf6847b7bf17bff8a, 0x6a803535d435b5e1, +0x3af51d1d741de869, 0xddb3e0e0a7e05347, 0xb321d7d77bd7f6ac, 0x999cc2c22fc25eed, +0x5c432e2eb82e6d96, 0x96294b4b314b627a, 0xe15dfefedffea321, 0xaed5575741578216, +0x2abd15155415a841, 0xeee87777c1779fb6, 0x6e923737dc37a5eb, 0xd79ee5e5b3e57b56, +0x23139f9f469f8cd9, 0xfd23f0f0e7f0d317, 0x94204a4a354a6a7f, 0xa944dada4fda9e95, +0xb0a258587d58fa25, 0x8fcfc9c903c906ca, 0x527c2929a429558d, 0x145a0a0a280a5022, +0x7f50b1b1feb1e14f, 0x5dc9a0a0baa0691a, 0xd6146b6bb16b7fda, 0x17d985852e855cab, +0x673cbdbdcebd8173, 0xba8f5d5d695dd234, 0x2090101040108050, 0xf507f4f4f7f4f303, +0x8bddcbcb0bcb16c0, 0x7cd33e3ef83eedc6, 0x0a2d050514052811, 0xce78676781671fe6, +0xd597e4e4b7e47353, 0x4e0227279c2725bb, 0x8273414119413258, 0x0ba78b8b168b2c9d, +0x53f6a7a7a6a75101, 0xfab27d7de97dcf94, 0x374995956e95dcfb, 0xad56d8d847d88e9f, +0xeb70fbfbcbfb8b30, 0xc1cdeeee9fee2371, 0xf8bb7c7ced7cc791, 0xcc716666856617e3, +0xa77bdddd53dda68e, 0x2eaf17175c17b84b, 0x8e45474701470246, 0x211a9e9e429e84dc, +0x89d4caca0fca1ec5, 0x5a582d2db42d7599, 0x632ebfbfc6bf9179, 0x0e3f07071c07381b, +0x47acadad8ead0123, 0xb4b05a5a755aea2f, 0x1bef838336836cb5, 0x66b63333cc3385ff, +0xc65c636391633ff2, 0x041202020802100a, 0x4993aaaa92aa3938, 0xe2de7171d971afa8, +0x8dc6c8c807c80ecf, 0x32d119196419c87d, 0x923b494939497270, 0xaf5fd9d943d9869a, +0xf931f2f2eff2c31d, 0xdba8e3e3abe34b48, 0xb6b95b5b715be22a, 0x0dbc88881a883492, +0x293e9a9a529aa4c8, 0x4c0b262698262dbe, 0x64bf3232c8328dfa, 0x7d59b0b0fab0e94a, +0xcff2e9e983e91b6a, 0x1e770f0f3c0f7833, 0xb733d5d573d5e6a6, 0x1df480803a8074ba, +0x6127bebec2be997c, 0x87ebcdcd13cd26de, 0x68893434d034bde4, 0x903248483d487a75, +0xe354ffffdbffab24, 0xf48d7a7af57af78f, 0x3d6490907a90f4ea, 0xbe9d5f5f615fc23e, +0x403d202080201da0, 0xd00f6868bd6867d5, 0x34ca1a1a681ad072, 0x41b7aeae82ae192c, +0x757db4b4eab4c95e, 0xa8ce54544d549a19, 0x3b7f93937693ece5, 0x442f222288220daa, +0xc86364648d6407e9, 0xff2af1f1e3f1db12, 0xe6cc7373d173bfa2, 0x248212124812905a, +0x807a40401d403a5d, 0x1048080820084028, 0x9b95c3c32bc356e8, 0xc5dfecec97ec337b, +0xab4ddbdb4bdb9690, 0x5fc0a1a1bea1611f, 0x07918d8d0e8d1c83, 0x7ac83d3df43df5c9, +0x335b97976697ccf1, 0x0000000000000000, 0x83f9cfcf1bcf36d4, 0x566e2b2bac2b4587, +0xece17676c57697b3, 0x19e68282328264b0, 0xb128d6d67fd6fea9, 0x36c31b1b6c1bd877, +0x7774b5b5eeb5c15b, 0x43beafaf86af1129, 0xd41d6a6ab56a77df, 0xa0ea50505d50ba0d, +0x8a5745450945124c, 0xfb38f3f3ebf3cb18, 0x60ad3030c0309df0, 0xc3c4efef9bef2b74, +0x7eda3f3ffc3fe5c3, 0xaac755554955921c, 0x59dba2a2b2a27910, 0xc9e9eaea8fea0365, +0xca6a656589650fec, 0x6903babad2bab968, 0x5e4a2f2fbc2f6593, 0x9d8ec0c027c04ee7, +0xa160dede5fdebe81, 0x38fc1c1c701ce06c, 0xe746fdfdd3fdbb2e, 0x9a1f4d4d294d5264, +0x397692927292e4e0, 0xeafa7575c9758fbc, 0x0c3606061806301e, 0x09ae8a8a128a2498, +0x794bb2b2f2b2f940, 0xd185e6e6bfe66359, 0x1c7e0e0e380e7036, 0x3ee71f1f7c1ff863, +0xc4556262956237f7, 0xb53ad4d477d4eea3, 0x4d81a8a89aa82932, 0x315296966296c4f4, +0xef62f9f9c3f99b3a, 0x97a3c5c533c566f6, 0x4a102525942535b1, 0xb2ab59597959f220, +0x15d084842a8454ae, 0xe4c57272d572b7a7, 0x72ec3939e439d5dd, 0x98164c4c2d4c5a61, +0xbc945e5e655eca3b, 0xf09f7878fd78e785, 0x70e53838e038ddd8, 0x05988c8c0a8c1486, +0xbf17d1d163d1c6b2, 0x57e4a5a5aea5410b, 0xd9a1e2e2afe2434d, 0xc24e616199612ff8, +0x7b42b3b3f6b3f145, 0x42342121842115a5, 0x25089c9c4a9c94d6, 0x3cee1e1e781ef066, +0x8661434311432252, 0x93b1c7c73bc776fc, 0xe54ffcfcd7fcb32b, 0x0824040410042014, +0xa2e351515951b208, 0x2f2599995e99bcc7, 0xda226d6da96d4fc4, 0x1a650d0d340d6839, +0xe979fafacffa8335, 0xa369dfdf5bdfb684, 0xfca97e7ee57ed79b, 0x4819242490243db4, +0x76fe3b3bec3bc5d7, 0x4b9aabab96ab313d, 0x81f0cece1fce3ed1, 0x2299111144118855, +0x03838f8f068f0c89, 0x9c044e4e254e4a6b, 0x7366b7b7e6b7d151, 0xcbe0ebeb8beb0b60, +0x78c13c3cf03cfdcc, 0x1ffd81813e817cbf, 0x354094946a94d4fe, 0xf31cf7f7fbf7eb0c, +0x6f18b9b9deb9a167, 0x268b13134c13985f, 0x58512c2cb02c7d9c, 0xbb05d3d36bd3d6b8, +0xd38ce7e7bbe76b5c, 0xdc396e6ea56e57cb, 0x95aac4c437c46ef3, 0x061b03030c03180f, +0xacdc565645568a13, 0x885e44440d441a49, 0xfea07f7fe17fdf9e, 0x4f88a9a99ea92137, +0x54672a2aa82a4d82, 0x6b0abbbbd6bbb16d, 0x9f87c1c123c146e2, 0xa6f153535153a202, +0xa572dcdc57dcae8b, 0x16530b0b2c0b5827, 0x27019d9d4e9d9cd3, 0xd82b6c6cad6c47c1, +0x62a43131c43195f5, 0xe8f37474cd7487b9, 0xf115f6f6fff6e309, 0x8c4c464605460a43, +0x45a5acac8aac0926, 0x0fb589891e893c97, 0x28b414145014a044, 0xdfbae1e1a3e15b42, +0x2ca616165816b04e, 0x74f73a3ae83acdd2, 0xd2066969b9696fd0, 0x124109092409482d, +0xe0d77070dd70a7ad, 0x716fb6b6e2b6d954, 0xbd1ed0d067d0ceb7, 0xc7d6eded93ed3b7e, +0x85e2cccc17cc2edb, 0x8468424215422a57, 0x2d2c98985a98b4c2, 0x55eda4a4aaa4490e, +0x50752828a0285d88, 0xb8865c5c6d5cda31, 0xed6bf8f8c7f8933f, 0x11c28686228644a4, +] +C3 = [ +0x7830d818186018c0, 0xaf462623238c2305, 0xf991b8c6c63fc67e, 0x6fcdfbe8e887e813, +0xa113cb878726874c, 0x626d11b8b8dab8a9, 0x0502090101040108, 0x6e9e0d4f4f214f42, +0xee6c9b3636d836ad, 0x0451ffa6a6a2a659, 0xbdb90cd2d26fd2de, 0x06f70ef5f5f3f5fb, +0x80f2967979f979ef, 0xcede306f6fa16f5f, 0xef3f6d91917e91fc, 0x07a4f852525552aa, +0xfdc04760609d6027, 0x766535bcbccabc89, 0xcd2b379b9b569bac, 0x8c018a8e8e028e04, +0x155bd2a3a3b6a371, 0x3c186c0c0c300c60, 0x8af6847b7bf17bff, 0xe16a803535d435b5, +0x693af51d1d741de8, 0x47ddb3e0e0a7e053, 0xacb321d7d77bd7f6, 0xed999cc2c22fc25e, +0x965c432e2eb82e6d, 0x7a96294b4b314b62, 0x21e15dfefedffea3, 0x16aed55757415782, +0x412abd15155415a8, 0xb6eee87777c1779f, 0xeb6e923737dc37a5, 0x56d79ee5e5b3e57b, +0xd923139f9f469f8c, 0x17fd23f0f0e7f0d3, 0x7f94204a4a354a6a, 0x95a944dada4fda9e, +0x25b0a258587d58fa, 0xca8fcfc9c903c906, 0x8d527c2929a42955, 0x22145a0a0a280a50, +0x4f7f50b1b1feb1e1, 0x1a5dc9a0a0baa069, 0xdad6146b6bb16b7f, 0xab17d985852e855c, +0x73673cbdbdcebd81, 0x34ba8f5d5d695dd2, 0x5020901010401080, 0x03f507f4f4f7f4f3, +0xc08bddcbcb0bcb16, 0xc67cd33e3ef83eed, 0x110a2d0505140528, 0xe6ce78676781671f, +0x53d597e4e4b7e473, 0xbb4e0227279c2725, 0x5882734141194132, 0x9d0ba78b8b168b2c, +0x0153f6a7a7a6a751, 0x94fab27d7de97dcf, 0xfb374995956e95dc, 0x9fad56d8d847d88e, +0x30eb70fbfbcbfb8b, 0x71c1cdeeee9fee23, 0x91f8bb7c7ced7cc7, 0xe3cc716666856617, +0x8ea77bdddd53dda6, 0x4b2eaf17175c17b8, 0x468e454747014702, 0xdc211a9e9e429e84, +0xc589d4caca0fca1e, 0x995a582d2db42d75, 0x79632ebfbfc6bf91, 0x1b0e3f07071c0738, +0x2347acadad8ead01, 0x2fb4b05a5a755aea, 0xb51bef838336836c, 0xff66b63333cc3385, +0xf2c65c636391633f, 0x0a04120202080210, 0x384993aaaa92aa39, 0xa8e2de7171d971af, +0xcf8dc6c8c807c80e, 0x7d32d119196419c8, 0x70923b4949394972, 0x9aaf5fd9d943d986, +0x1df931f2f2eff2c3, 0x48dba8e3e3abe34b, 0x2ab6b95b5b715be2, 0x920dbc88881a8834, +0xc8293e9a9a529aa4, 0xbe4c0b262698262d, 0xfa64bf3232c8328d, 0x4a7d59b0b0fab0e9, +0x6acff2e9e983e91b, 0x331e770f0f3c0f78, 0xa6b733d5d573d5e6, 0xba1df480803a8074, +0x7c6127bebec2be99, 0xde87ebcdcd13cd26, 0xe468893434d034bd, 0x75903248483d487a, +0x24e354ffffdbffab, 0x8ff48d7a7af57af7, 0xea3d6490907a90f4, 0x3ebe9d5f5f615fc2, +0xa0403d202080201d, 0xd5d00f6868bd6867, 0x7234ca1a1a681ad0, 0x2c41b7aeae82ae19, +0x5e757db4b4eab4c9, 0x19a8ce54544d549a, 0xe53b7f93937693ec, 0xaa442f222288220d, +0xe9c86364648d6407, 0x12ff2af1f1e3f1db, 0xa2e6cc7373d173bf, 0x5a24821212481290, +0x5d807a40401d403a, 0x2810480808200840, 0xe89b95c3c32bc356, 0x7bc5dfecec97ec33, +0x90ab4ddbdb4bdb96, 0x1f5fc0a1a1bea161, 0x8307918d8d0e8d1c, 0xc97ac83d3df43df5, +0xf1335b97976697cc, 0x0000000000000000, 0xd483f9cfcf1bcf36, 0x87566e2b2bac2b45, +0xb3ece17676c57697, 0xb019e68282328264, 0xa9b128d6d67fd6fe, 0x7736c31b1b6c1bd8, +0x5b7774b5b5eeb5c1, 0x2943beafaf86af11, 0xdfd41d6a6ab56a77, 0x0da0ea50505d50ba, +0x4c8a574545094512, 0x18fb38f3f3ebf3cb, 0xf060ad3030c0309d, 0x74c3c4efef9bef2b, +0xc37eda3f3ffc3fe5, 0x1caac75555495592, 0x1059dba2a2b2a279, 0x65c9e9eaea8fea03, +0xecca6a656589650f, 0x686903babad2bab9, 0x935e4a2f2fbc2f65, 0xe79d8ec0c027c04e, +0x81a160dede5fdebe, 0x6c38fc1c1c701ce0, 0x2ee746fdfdd3fdbb, 0x649a1f4d4d294d52, +0xe0397692927292e4, 0xbceafa7575c9758f, 0x1e0c360606180630, 0x9809ae8a8a128a24, +0x40794bb2b2f2b2f9, 0x59d185e6e6bfe663, 0x361c7e0e0e380e70, 0x633ee71f1f7c1ff8, +0xf7c4556262956237, 0xa3b53ad4d477d4ee, 0x324d81a8a89aa829, 0xf4315296966296c4, +0x3aef62f9f9c3f99b, 0xf697a3c5c533c566, 0xb14a102525942535, 0x20b2ab59597959f2, +0xae15d084842a8454, 0xa7e4c57272d572b7, 0xdd72ec3939e439d5, 0x6198164c4c2d4c5a, +0x3bbc945e5e655eca, 0x85f09f7878fd78e7, 0xd870e53838e038dd, 0x8605988c8c0a8c14, +0xb2bf17d1d163d1c6, 0x0b57e4a5a5aea541, 0x4dd9a1e2e2afe243, 0xf8c24e616199612f, +0x457b42b3b3f6b3f1, 0xa542342121842115, 0xd625089c9c4a9c94, 0x663cee1e1e781ef0, +0x5286614343114322, 0xfc93b1c7c73bc776, 0x2be54ffcfcd7fcb3, 0x1408240404100420, +0x08a2e351515951b2, 0xc72f2599995e99bc, 0xc4da226d6da96d4f, 0x391a650d0d340d68, +0x35e979fafacffa83, 0x84a369dfdf5bdfb6, 0x9bfca97e7ee57ed7, 0xb44819242490243d, +0xd776fe3b3bec3bc5, 0x3d4b9aabab96ab31, 0xd181f0cece1fce3e, 0x5522991111441188, +0x8903838f8f068f0c, 0x6b9c044e4e254e4a, 0x517366b7b7e6b7d1, 0x60cbe0ebeb8beb0b, +0xcc78c13c3cf03cfd, 0xbf1ffd81813e817c, 0xfe354094946a94d4, 0x0cf31cf7f7fbf7eb, +0x676f18b9b9deb9a1, 0x5f268b13134c1398, 0x9c58512c2cb02c7d, 0xb8bb05d3d36bd3d6, +0x5cd38ce7e7bbe76b, 0xcbdc396e6ea56e57, 0xf395aac4c437c46e, 0x0f061b03030c0318, +0x13acdc565645568a, 0x49885e44440d441a, 0x9efea07f7fe17fdf, 0x374f88a9a99ea921, +0x8254672a2aa82a4d, 0x6d6b0abbbbd6bbb1, 0xe29f87c1c123c146, 0x02a6f153535153a2, +0x8ba572dcdc57dcae, 0x2716530b0b2c0b58, 0xd327019d9d4e9d9c, 0xc1d82b6c6cad6c47, +0xf562a43131c43195, 0xb9e8f37474cd7487, 0x09f115f6f6fff6e3, 0x438c4c464605460a, +0x2645a5acac8aac09, 0x970fb589891e893c, 0x4428b414145014a0, 0x42dfbae1e1a3e15b, +0x4e2ca616165816b0, 0xd274f73a3ae83acd, 0xd0d2066969b9696f, 0x2d12410909240948, +0xade0d77070dd70a7, 0x54716fb6b6e2b6d9, 0xb7bd1ed0d067d0ce, 0x7ec7d6eded93ed3b, +0xdb85e2cccc17cc2e, 0x578468424215422a, 0xc22d2c98985a98b4, 0x0e55eda4a4aaa449, +0x8850752828a0285d, 0x31b8865c5c6d5cda, 0x3fed6bf8f8c7f893, 0xa411c28686228644, +] +C4 = [ +0xc07830d818186018, 0x05af462623238c23, 0x7ef991b8c6c63fc6, 0x136fcdfbe8e887e8, +0x4ca113cb87872687, 0xa9626d11b8b8dab8, 0x0805020901010401, 0x426e9e0d4f4f214f, +0xadee6c9b3636d836, 0x590451ffa6a6a2a6, 0xdebdb90cd2d26fd2, 0xfb06f70ef5f5f3f5, +0xef80f2967979f979, 0x5fcede306f6fa16f, 0xfcef3f6d91917e91, 0xaa07a4f852525552, +0x27fdc04760609d60, 0x89766535bcbccabc, 0xaccd2b379b9b569b, 0x048c018a8e8e028e, +0x71155bd2a3a3b6a3, 0x603c186c0c0c300c, 0xff8af6847b7bf17b, 0xb5e16a803535d435, +0xe8693af51d1d741d, 0x5347ddb3e0e0a7e0, 0xf6acb321d7d77bd7, 0x5eed999cc2c22fc2, +0x6d965c432e2eb82e, 0x627a96294b4b314b, 0xa321e15dfefedffe, 0x8216aed557574157, +0xa8412abd15155415, 0x9fb6eee87777c177, 0xa5eb6e923737dc37, 0x7b56d79ee5e5b3e5, +0x8cd923139f9f469f, 0xd317fd23f0f0e7f0, 0x6a7f94204a4a354a, 0x9e95a944dada4fda, +0xfa25b0a258587d58, 0x06ca8fcfc9c903c9, 0x558d527c2929a429, 0x5022145a0a0a280a, +0xe14f7f50b1b1feb1, 0x691a5dc9a0a0baa0, 0x7fdad6146b6bb16b, 0x5cab17d985852e85, +0x8173673cbdbdcebd, 0xd234ba8f5d5d695d, 0x8050209010104010, 0xf303f507f4f4f7f4, +0x16c08bddcbcb0bcb, 0xedc67cd33e3ef83e, 0x28110a2d05051405, 0x1fe6ce7867678167, +0x7353d597e4e4b7e4, 0x25bb4e0227279c27, 0x3258827341411941, 0x2c9d0ba78b8b168b, +0x510153f6a7a7a6a7, 0xcf94fab27d7de97d, 0xdcfb374995956e95, 0x8e9fad56d8d847d8, +0x8b30eb70fbfbcbfb, 0x2371c1cdeeee9fee, 0xc791f8bb7c7ced7c, 0x17e3cc7166668566, +0xa68ea77bdddd53dd, 0xb84b2eaf17175c17, 0x02468e4547470147, 0x84dc211a9e9e429e, +0x1ec589d4caca0fca, 0x75995a582d2db42d, 0x9179632ebfbfc6bf, 0x381b0e3f07071c07, +0x012347acadad8ead, 0xea2fb4b05a5a755a, 0x6cb51bef83833683, 0x85ff66b63333cc33, +0x3ff2c65c63639163, 0x100a041202020802, 0x39384993aaaa92aa, 0xafa8e2de7171d971, +0x0ecf8dc6c8c807c8, 0xc87d32d119196419, 0x7270923b49493949, 0x869aaf5fd9d943d9, +0xc31df931f2f2eff2, 0x4b48dba8e3e3abe3, 0xe22ab6b95b5b715b, 0x34920dbc88881a88, +0xa4c8293e9a9a529a, 0x2dbe4c0b26269826, 0x8dfa64bf3232c832, 0xe94a7d59b0b0fab0, +0x1b6acff2e9e983e9, 0x78331e770f0f3c0f, 0xe6a6b733d5d573d5, 0x74ba1df480803a80, +0x997c6127bebec2be, 0x26de87ebcdcd13cd, 0xbde468893434d034, 0x7a75903248483d48, +0xab24e354ffffdbff, 0xf78ff48d7a7af57a, 0xf4ea3d6490907a90, 0xc23ebe9d5f5f615f, +0x1da0403d20208020, 0x67d5d00f6868bd68, 0xd07234ca1a1a681a, 0x192c41b7aeae82ae, +0xc95e757db4b4eab4, 0x9a19a8ce54544d54, 0xece53b7f93937693, 0x0daa442f22228822, +0x07e9c86364648d64, 0xdb12ff2af1f1e3f1, 0xbfa2e6cc7373d173, 0x905a248212124812, +0x3a5d807a40401d40, 0x4028104808082008, 0x56e89b95c3c32bc3, 0x337bc5dfecec97ec, +0x9690ab4ddbdb4bdb, 0x611f5fc0a1a1bea1, 0x1c8307918d8d0e8d, 0xf5c97ac83d3df43d, +0xccf1335b97976697, 0x0000000000000000, 0x36d483f9cfcf1bcf, 0x4587566e2b2bac2b, +0x97b3ece17676c576, 0x64b019e682823282, 0xfea9b128d6d67fd6, 0xd87736c31b1b6c1b, +0xc15b7774b5b5eeb5, 0x112943beafaf86af, 0x77dfd41d6a6ab56a, 0xba0da0ea50505d50, +0x124c8a5745450945, 0xcb18fb38f3f3ebf3, 0x9df060ad3030c030, 0x2b74c3c4efef9bef, +0xe5c37eda3f3ffc3f, 0x921caac755554955, 0x791059dba2a2b2a2, 0x0365c9e9eaea8fea, +0x0fecca6a65658965, 0xb9686903babad2ba, 0x65935e4a2f2fbc2f, 0x4ee79d8ec0c027c0, +0xbe81a160dede5fde, 0xe06c38fc1c1c701c, 0xbb2ee746fdfdd3fd, 0x52649a1f4d4d294d, +0xe4e0397692927292, 0x8fbceafa7575c975, 0x301e0c3606061806, 0x249809ae8a8a128a, +0xf940794bb2b2f2b2, 0x6359d185e6e6bfe6, 0x70361c7e0e0e380e, 0xf8633ee71f1f7c1f, +0x37f7c45562629562, 0xeea3b53ad4d477d4, 0x29324d81a8a89aa8, 0xc4f4315296966296, +0x9b3aef62f9f9c3f9, 0x66f697a3c5c533c5, 0x35b14a1025259425, 0xf220b2ab59597959, +0x54ae15d084842a84, 0xb7a7e4c57272d572, 0xd5dd72ec3939e439, 0x5a6198164c4c2d4c, +0xca3bbc945e5e655e, 0xe785f09f7878fd78, 0xddd870e53838e038, 0x148605988c8c0a8c, +0xc6b2bf17d1d163d1, 0x410b57e4a5a5aea5, 0x434dd9a1e2e2afe2, 0x2ff8c24e61619961, +0xf1457b42b3b3f6b3, 0x15a5423421218421, 0x94d625089c9c4a9c, 0xf0663cee1e1e781e, +0x2252866143431143, 0x76fc93b1c7c73bc7, 0xb32be54ffcfcd7fc, 0x2014082404041004, +0xb208a2e351515951, 0xbcc72f2599995e99, 0x4fc4da226d6da96d, 0x68391a650d0d340d, +0x8335e979fafacffa, 0xb684a369dfdf5bdf, 0xd79bfca97e7ee57e, 0x3db4481924249024, +0xc5d776fe3b3bec3b, 0x313d4b9aabab96ab, 0x3ed181f0cece1fce, 0x8855229911114411, +0x0c8903838f8f068f, 0x4a6b9c044e4e254e, 0xd1517366b7b7e6b7, 0x0b60cbe0ebeb8beb, +0xfdcc78c13c3cf03c, 0x7cbf1ffd81813e81, 0xd4fe354094946a94, 0xeb0cf31cf7f7fbf7, +0xa1676f18b9b9deb9, 0x985f268b13134c13, 0x7d9c58512c2cb02c, 0xd6b8bb05d3d36bd3, +0x6b5cd38ce7e7bbe7, 0x57cbdc396e6ea56e, 0x6ef395aac4c437c4, 0x180f061b03030c03, +0x8a13acdc56564556, 0x1a49885e44440d44, 0xdf9efea07f7fe17f, 0x21374f88a9a99ea9, +0x4d8254672a2aa82a, 0xb16d6b0abbbbd6bb, 0x46e29f87c1c123c1, 0xa202a6f153535153, +0xae8ba572dcdc57dc, 0x582716530b0b2c0b, 0x9cd327019d9d4e9d, 0x47c1d82b6c6cad6c, +0x95f562a43131c431, 0x87b9e8f37474cd74, 0xe309f115f6f6fff6, 0x0a438c4c46460546, +0x092645a5acac8aac, 0x3c970fb589891e89, 0xa04428b414145014, 0x5b42dfbae1e1a3e1, +0xb04e2ca616165816, 0xcdd274f73a3ae83a, 0x6fd0d2066969b969, 0x482d124109092409, +0xa7ade0d77070dd70, 0xd954716fb6b6e2b6, 0xceb7bd1ed0d067d0, 0x3b7ec7d6eded93ed, +0x2edb85e2cccc17cc, 0x2a57846842421542, 0xb4c22d2c98985a98, 0x490e55eda4a4aaa4, +0x5d8850752828a028, 0xda31b8865c5c6d5c, 0x933fed6bf8f8c7f8, 0x44a411c286862286, +] +C5 = [ +0x18c07830d8181860, 0x2305af462623238c, 0xc67ef991b8c6c63f, 0xe8136fcdfbe8e887, +0x874ca113cb878726, 0xb8a9626d11b8b8da, 0x0108050209010104, 0x4f426e9e0d4f4f21, +0x36adee6c9b3636d8, 0xa6590451ffa6a6a2, 0xd2debdb90cd2d26f, 0xf5fb06f70ef5f5f3, +0x79ef80f2967979f9, 0x6f5fcede306f6fa1, 0x91fcef3f6d91917e, 0x52aa07a4f8525255, +0x6027fdc04760609d, 0xbc89766535bcbcca, 0x9baccd2b379b9b56, 0x8e048c018a8e8e02, +0xa371155bd2a3a3b6, 0x0c603c186c0c0c30, 0x7bff8af6847b7bf1, 0x35b5e16a803535d4, +0x1de8693af51d1d74, 0xe05347ddb3e0e0a7, 0xd7f6acb321d7d77b, 0xc25eed999cc2c22f, +0x2e6d965c432e2eb8, 0x4b627a96294b4b31, 0xfea321e15dfefedf, 0x578216aed5575741, +0x15a8412abd151554, 0x779fb6eee87777c1, 0x37a5eb6e923737dc, 0xe57b56d79ee5e5b3, +0x9f8cd923139f9f46, 0xf0d317fd23f0f0e7, 0x4a6a7f94204a4a35, 0xda9e95a944dada4f, +0x58fa25b0a258587d, 0xc906ca8fcfc9c903, 0x29558d527c2929a4, 0x0a5022145a0a0a28, +0xb1e14f7f50b1b1fe, 0xa0691a5dc9a0a0ba, 0x6b7fdad6146b6bb1, 0x855cab17d985852e, +0xbd8173673cbdbdce, 0x5dd234ba8f5d5d69, 0x1080502090101040, 0xf4f303f507f4f4f7, +0xcb16c08bddcbcb0b, 0x3eedc67cd33e3ef8, 0x0528110a2d050514, 0x671fe6ce78676781, +0xe47353d597e4e4b7, 0x2725bb4e0227279c, 0x4132588273414119, 0x8b2c9d0ba78b8b16, +0xa7510153f6a7a7a6, 0x7dcf94fab27d7de9, 0x95dcfb374995956e, 0xd88e9fad56d8d847, +0xfb8b30eb70fbfbcb, 0xee2371c1cdeeee9f, 0x7cc791f8bb7c7ced, 0x6617e3cc71666685, +0xdda68ea77bdddd53, 0x17b84b2eaf17175c, 0x4702468e45474701, 0x9e84dc211a9e9e42, +0xca1ec589d4caca0f, 0x2d75995a582d2db4, 0xbf9179632ebfbfc6, 0x07381b0e3f07071c, +0xad012347acadad8e, 0x5aea2fb4b05a5a75, 0x836cb51bef838336, 0x3385ff66b63333cc, +0x633ff2c65c636391, 0x02100a0412020208, 0xaa39384993aaaa92, 0x71afa8e2de7171d9, +0xc80ecf8dc6c8c807, 0x19c87d32d1191964, 0x497270923b494939, 0xd9869aaf5fd9d943, +0xf2c31df931f2f2ef, 0xe34b48dba8e3e3ab, 0x5be22ab6b95b5b71, 0x8834920dbc88881a, +0x9aa4c8293e9a9a52, 0x262dbe4c0b262698, 0x328dfa64bf3232c8, 0xb0e94a7d59b0b0fa, +0xe91b6acff2e9e983, 0x0f78331e770f0f3c, 0xd5e6a6b733d5d573, 0x8074ba1df480803a, +0xbe997c6127bebec2, 0xcd26de87ebcdcd13, 0x34bde468893434d0, 0x487a75903248483d, +0xffab24e354ffffdb, 0x7af78ff48d7a7af5, 0x90f4ea3d6490907a, 0x5fc23ebe9d5f5f61, +0x201da0403d202080, 0x6867d5d00f6868bd, 0x1ad07234ca1a1a68, 0xae192c41b7aeae82, +0xb4c95e757db4b4ea, 0x549a19a8ce54544d, 0x93ece53b7f939376, 0x220daa442f222288, +0x6407e9c86364648d, 0xf1db12ff2af1f1e3, 0x73bfa2e6cc7373d1, 0x12905a2482121248, +0x403a5d807a40401d, 0x0840281048080820, 0xc356e89b95c3c32b, 0xec337bc5dfecec97, +0xdb9690ab4ddbdb4b, 0xa1611f5fc0a1a1be, 0x8d1c8307918d8d0e, 0x3df5c97ac83d3df4, +0x97ccf1335b979766, 0x0000000000000000, 0xcf36d483f9cfcf1b, 0x2b4587566e2b2bac, +0x7697b3ece17676c5, 0x8264b019e6828232, 0xd6fea9b128d6d67f, 0x1bd87736c31b1b6c, +0xb5c15b7774b5b5ee, 0xaf112943beafaf86, 0x6a77dfd41d6a6ab5, 0x50ba0da0ea50505d, +0x45124c8a57454509, 0xf3cb18fb38f3f3eb, 0x309df060ad3030c0, 0xef2b74c3c4efef9b, +0x3fe5c37eda3f3ffc, 0x55921caac7555549, 0xa2791059dba2a2b2, 0xea0365c9e9eaea8f, +0x650fecca6a656589, 0xbab9686903babad2, 0x2f65935e4a2f2fbc, 0xc04ee79d8ec0c027, +0xdebe81a160dede5f, 0x1ce06c38fc1c1c70, 0xfdbb2ee746fdfdd3, 0x4d52649a1f4d4d29, +0x92e4e03976929272, 0x758fbceafa7575c9, 0x06301e0c36060618, 0x8a249809ae8a8a12, +0xb2f940794bb2b2f2, 0xe66359d185e6e6bf, 0x0e70361c7e0e0e38, 0x1ff8633ee71f1f7c, +0x6237f7c455626295, 0xd4eea3b53ad4d477, 0xa829324d81a8a89a, 0x96c4f43152969662, +0xf99b3aef62f9f9c3, 0xc566f697a3c5c533, 0x2535b14a10252594, 0x59f220b2ab595979, +0x8454ae15d084842a, 0x72b7a7e4c57272d5, 0x39d5dd72ec3939e4, 0x4c5a6198164c4c2d, +0x5eca3bbc945e5e65, 0x78e785f09f7878fd, 0x38ddd870e53838e0, 0x8c148605988c8c0a, +0xd1c6b2bf17d1d163, 0xa5410b57e4a5a5ae, 0xe2434dd9a1e2e2af, 0x612ff8c24e616199, +0xb3f1457b42b3b3f6, 0x2115a54234212184, 0x9c94d625089c9c4a, 0x1ef0663cee1e1e78, +0x4322528661434311, 0xc776fc93b1c7c73b, 0xfcb32be54ffcfcd7, 0x0420140824040410, +0x51b208a2e3515159, 0x99bcc72f2599995e, 0x6d4fc4da226d6da9, 0x0d68391a650d0d34, +0xfa8335e979fafacf, 0xdfb684a369dfdf5b, 0x7ed79bfca97e7ee5, 0x243db44819242490, +0x3bc5d776fe3b3bec, 0xab313d4b9aabab96, 0xce3ed181f0cece1f, 0x1188552299111144, +0x8f0c8903838f8f06, 0x4e4a6b9c044e4e25, 0xb7d1517366b7b7e6, 0xeb0b60cbe0ebeb8b, +0x3cfdcc78c13c3cf0, 0x817cbf1ffd81813e, 0x94d4fe354094946a, 0xf7eb0cf31cf7f7fb, +0xb9a1676f18b9b9de, 0x13985f268b13134c, 0x2c7d9c58512c2cb0, 0xd3d6b8bb05d3d36b, +0xe76b5cd38ce7e7bb, 0x6e57cbdc396e6ea5, 0xc46ef395aac4c437, 0x03180f061b03030c, +0x568a13acdc565645, 0x441a49885e44440d, 0x7fdf9efea07f7fe1, 0xa921374f88a9a99e, +0x2a4d8254672a2aa8, 0xbbb16d6b0abbbbd6, 0xc146e29f87c1c123, 0x53a202a6f1535351, +0xdcae8ba572dcdc57, 0x0b582716530b0b2c, 0x9d9cd327019d9d4e, 0x6c47c1d82b6c6cad, +0x3195f562a43131c4, 0x7487b9e8f37474cd, 0xf6e309f115f6f6ff, 0x460a438c4c464605, +0xac092645a5acac8a, 0x893c970fb589891e, 0x14a04428b4141450, 0xe15b42dfbae1e1a3, +0x16b04e2ca6161658, 0x3acdd274f73a3ae8, 0x696fd0d2066969b9, 0x09482d1241090924, +0x70a7ade0d77070dd, 0xb6d954716fb6b6e2, 0xd0ceb7bd1ed0d067, 0xed3b7ec7d6eded93, +0xcc2edb85e2cccc17, 0x422a578468424215, 0x98b4c22d2c98985a, 0xa4490e55eda4a4aa, +0x285d8850752828a0, 0x5cda31b8865c5c6d, 0xf8933fed6bf8f8c7, 0x8644a411c2868622, +] +C6 = [ +0x6018c07830d81818, 0x8c2305af46262323, 0x3fc67ef991b8c6c6, 0x87e8136fcdfbe8e8, +0x26874ca113cb8787, 0xdab8a9626d11b8b8, 0x0401080502090101, 0x214f426e9e0d4f4f, +0xd836adee6c9b3636, 0xa2a6590451ffa6a6, 0x6fd2debdb90cd2d2, 0xf3f5fb06f70ef5f5, +0xf979ef80f2967979, 0xa16f5fcede306f6f, 0x7e91fcef3f6d9191, 0x5552aa07a4f85252, +0x9d6027fdc0476060, 0xcabc89766535bcbc, 0x569baccd2b379b9b, 0x028e048c018a8e8e, +0xb6a371155bd2a3a3, 0x300c603c186c0c0c, 0xf17bff8af6847b7b, 0xd435b5e16a803535, +0x741de8693af51d1d, 0xa7e05347ddb3e0e0, 0x7bd7f6acb321d7d7, 0x2fc25eed999cc2c2, +0xb82e6d965c432e2e, 0x314b627a96294b4b, 0xdffea321e15dfefe, 0x41578216aed55757, +0x5415a8412abd1515, 0xc1779fb6eee87777, 0xdc37a5eb6e923737, 0xb3e57b56d79ee5e5, +0x469f8cd923139f9f, 0xe7f0d317fd23f0f0, 0x354a6a7f94204a4a, 0x4fda9e95a944dada, +0x7d58fa25b0a25858, 0x03c906ca8fcfc9c9, 0xa429558d527c2929, 0x280a5022145a0a0a, +0xfeb1e14f7f50b1b1, 0xbaa0691a5dc9a0a0, 0xb16b7fdad6146b6b, 0x2e855cab17d98585, +0xcebd8173673cbdbd, 0x695dd234ba8f5d5d, 0x4010805020901010, 0xf7f4f303f507f4f4, +0x0bcb16c08bddcbcb, 0xf83eedc67cd33e3e, 0x140528110a2d0505, 0x81671fe6ce786767, +0xb7e47353d597e4e4, 0x9c2725bb4e022727, 0x1941325882734141, 0x168b2c9d0ba78b8b, +0xa6a7510153f6a7a7, 0xe97dcf94fab27d7d, 0x6e95dcfb37499595, 0x47d88e9fad56d8d8, +0xcbfb8b30eb70fbfb, 0x9fee2371c1cdeeee, 0xed7cc791f8bb7c7c, 0x856617e3cc716666, +0x53dda68ea77bdddd, 0x5c17b84b2eaf1717, 0x014702468e454747, 0x429e84dc211a9e9e, +0x0fca1ec589d4caca, 0xb42d75995a582d2d, 0xc6bf9179632ebfbf, 0x1c07381b0e3f0707, +0x8ead012347acadad, 0x755aea2fb4b05a5a, 0x36836cb51bef8383, 0xcc3385ff66b63333, +0x91633ff2c65c6363, 0x0802100a04120202, 0x92aa39384993aaaa, 0xd971afa8e2de7171, +0x07c80ecf8dc6c8c8, 0x6419c87d32d11919, 0x39497270923b4949, 0x43d9869aaf5fd9d9, +0xeff2c31df931f2f2, 0xabe34b48dba8e3e3, 0x715be22ab6b95b5b, 0x1a8834920dbc8888, +0x529aa4c8293e9a9a, 0x98262dbe4c0b2626, 0xc8328dfa64bf3232, 0xfab0e94a7d59b0b0, +0x83e91b6acff2e9e9, 0x3c0f78331e770f0f, 0x73d5e6a6b733d5d5, 0x3a8074ba1df48080, +0xc2be997c6127bebe, 0x13cd26de87ebcdcd, 0xd034bde468893434, 0x3d487a7590324848, +0xdbffab24e354ffff, 0xf57af78ff48d7a7a, 0x7a90f4ea3d649090, 0x615fc23ebe9d5f5f, +0x80201da0403d2020, 0xbd6867d5d00f6868, 0x681ad07234ca1a1a, 0x82ae192c41b7aeae, +0xeab4c95e757db4b4, 0x4d549a19a8ce5454, 0x7693ece53b7f9393, 0x88220daa442f2222, +0x8d6407e9c8636464, 0xe3f1db12ff2af1f1, 0xd173bfa2e6cc7373, 0x4812905a24821212, +0x1d403a5d807a4040, 0x2008402810480808, 0x2bc356e89b95c3c3, 0x97ec337bc5dfecec, +0x4bdb9690ab4ddbdb, 0xbea1611f5fc0a1a1, 0x0e8d1c8307918d8d, 0xf43df5c97ac83d3d, +0x6697ccf1335b9797, 0x0000000000000000, 0x1bcf36d483f9cfcf, 0xac2b4587566e2b2b, +0xc57697b3ece17676, 0x328264b019e68282, 0x7fd6fea9b128d6d6, 0x6c1bd87736c31b1b, +0xeeb5c15b7774b5b5, 0x86af112943beafaf, 0xb56a77dfd41d6a6a, 0x5d50ba0da0ea5050, +0x0945124c8a574545, 0xebf3cb18fb38f3f3, 0xc0309df060ad3030, 0x9bef2b74c3c4efef, +0xfc3fe5c37eda3f3f, 0x4955921caac75555, 0xb2a2791059dba2a2, 0x8fea0365c9e9eaea, +0x89650fecca6a6565, 0xd2bab9686903baba, 0xbc2f65935e4a2f2f, 0x27c04ee79d8ec0c0, +0x5fdebe81a160dede, 0x701ce06c38fc1c1c, 0xd3fdbb2ee746fdfd, 0x294d52649a1f4d4d, +0x7292e4e039769292, 0xc9758fbceafa7575, 0x1806301e0c360606, 0x128a249809ae8a8a, +0xf2b2f940794bb2b2, 0xbfe66359d185e6e6, 0x380e70361c7e0e0e, 0x7c1ff8633ee71f1f, +0x956237f7c4556262, 0x77d4eea3b53ad4d4, 0x9aa829324d81a8a8, 0x6296c4f431529696, +0xc3f99b3aef62f9f9, 0x33c566f697a3c5c5, 0x942535b14a102525, 0x7959f220b2ab5959, +0x2a8454ae15d08484, 0xd572b7a7e4c57272, 0xe439d5dd72ec3939, 0x2d4c5a6198164c4c, +0x655eca3bbc945e5e, 0xfd78e785f09f7878, 0xe038ddd870e53838, 0x0a8c148605988c8c, +0x63d1c6b2bf17d1d1, 0xaea5410b57e4a5a5, 0xafe2434dd9a1e2e2, 0x99612ff8c24e6161, +0xf6b3f1457b42b3b3, 0x842115a542342121, 0x4a9c94d625089c9c, 0x781ef0663cee1e1e, +0x1143225286614343, 0x3bc776fc93b1c7c7, 0xd7fcb32be54ffcfc, 0x1004201408240404, +0x5951b208a2e35151, 0x5e99bcc72f259999, 0xa96d4fc4da226d6d, 0x340d68391a650d0d, +0xcffa8335e979fafa, 0x5bdfb684a369dfdf, 0xe57ed79bfca97e7e, 0x90243db448192424, +0xec3bc5d776fe3b3b, 0x96ab313d4b9aabab, 0x1fce3ed181f0cece, 0x4411885522991111, +0x068f0c8903838f8f, 0x254e4a6b9c044e4e, 0xe6b7d1517366b7b7, 0x8beb0b60cbe0ebeb, +0xf03cfdcc78c13c3c, 0x3e817cbf1ffd8181, 0x6a94d4fe35409494, 0xfbf7eb0cf31cf7f7, +0xdeb9a1676f18b9b9, 0x4c13985f268b1313, 0xb02c7d9c58512c2c, 0x6bd3d6b8bb05d3d3, +0xbbe76b5cd38ce7e7, 0xa56e57cbdc396e6e, 0x37c46ef395aac4c4, 0x0c03180f061b0303, +0x45568a13acdc5656, 0x0d441a49885e4444, 0xe17fdf9efea07f7f, 0x9ea921374f88a9a9, +0xa82a4d8254672a2a, 0xd6bbb16d6b0abbbb, 0x23c146e29f87c1c1, 0x5153a202a6f15353, +0x57dcae8ba572dcdc, 0x2c0b582716530b0b, 0x4e9d9cd327019d9d, 0xad6c47c1d82b6c6c, +0xc43195f562a43131, 0xcd7487b9e8f37474, 0xfff6e309f115f6f6, 0x05460a438c4c4646, +0x8aac092645a5acac, 0x1e893c970fb58989, 0x5014a04428b41414, 0xa3e15b42dfbae1e1, +0x5816b04e2ca61616, 0xe83acdd274f73a3a, 0xb9696fd0d2066969, 0x2409482d12410909, +0xdd70a7ade0d77070, 0xe2b6d954716fb6b6, 0x67d0ceb7bd1ed0d0, 0x93ed3b7ec7d6eded, +0x17cc2edb85e2cccc, 0x15422a5784684242, 0x5a98b4c22d2c9898, 0xaaa4490e55eda4a4, +0xa0285d8850752828, 0x6d5cda31b8865c5c, 0xc7f8933fed6bf8f8, 0x228644a411c28686, +] +C7 = [ +0x186018c07830d818, 0x238c2305af462623, 0xc63fc67ef991b8c6, 0xe887e8136fcdfbe8, +0x8726874ca113cb87, 0xb8dab8a9626d11b8, 0x0104010805020901, 0x4f214f426e9e0d4f, +0x36d836adee6c9b36, 0xa6a2a6590451ffa6, 0xd26fd2debdb90cd2, 0xf5f3f5fb06f70ef5, +0x79f979ef80f29679, 0x6fa16f5fcede306f, 0x917e91fcef3f6d91, 0x525552aa07a4f852, +0x609d6027fdc04760, 0xbccabc89766535bc, 0x9b569baccd2b379b, 0x8e028e048c018a8e, +0xa3b6a371155bd2a3, 0x0c300c603c186c0c, 0x7bf17bff8af6847b, 0x35d435b5e16a8035, +0x1d741de8693af51d, 0xe0a7e05347ddb3e0, 0xd77bd7f6acb321d7, 0xc22fc25eed999cc2, +0x2eb82e6d965c432e, 0x4b314b627a96294b, 0xfedffea321e15dfe, 0x5741578216aed557, +0x155415a8412abd15, 0x77c1779fb6eee877, 0x37dc37a5eb6e9237, 0xe5b3e57b56d79ee5, +0x9f469f8cd923139f, 0xf0e7f0d317fd23f0, 0x4a354a6a7f94204a, 0xda4fda9e95a944da, +0x587d58fa25b0a258, 0xc903c906ca8fcfc9, 0x29a429558d527c29, 0x0a280a5022145a0a, +0xb1feb1e14f7f50b1, 0xa0baa0691a5dc9a0, 0x6bb16b7fdad6146b, 0x852e855cab17d985, +0xbdcebd8173673cbd, 0x5d695dd234ba8f5d, 0x1040108050209010, 0xf4f7f4f303f507f4, +0xcb0bcb16c08bddcb, 0x3ef83eedc67cd33e, 0x05140528110a2d05, 0x6781671fe6ce7867, +0xe4b7e47353d597e4, 0x279c2725bb4e0227, 0x4119413258827341, 0x8b168b2c9d0ba78b, +0xa7a6a7510153f6a7, 0x7de97dcf94fab27d, 0x956e95dcfb374995, 0xd847d88e9fad56d8, +0xfbcbfb8b30eb70fb, 0xee9fee2371c1cdee, 0x7ced7cc791f8bb7c, 0x66856617e3cc7166, +0xdd53dda68ea77bdd, 0x175c17b84b2eaf17, 0x47014702468e4547, 0x9e429e84dc211a9e, +0xca0fca1ec589d4ca, 0x2db42d75995a582d, 0xbfc6bf9179632ebf, 0x071c07381b0e3f07, +0xad8ead012347acad, 0x5a755aea2fb4b05a, 0x8336836cb51bef83, 0x33cc3385ff66b633, +0x6391633ff2c65c63, 0x020802100a041202, 0xaa92aa39384993aa, 0x71d971afa8e2de71, +0xc807c80ecf8dc6c8, 0x196419c87d32d119, 0x4939497270923b49, 0xd943d9869aaf5fd9, +0xf2eff2c31df931f2, 0xe3abe34b48dba8e3, 0x5b715be22ab6b95b, 0x881a8834920dbc88, +0x9a529aa4c8293e9a, 0x2698262dbe4c0b26, 0x32c8328dfa64bf32, 0xb0fab0e94a7d59b0, +0xe983e91b6acff2e9, 0x0f3c0f78331e770f, 0xd573d5e6a6b733d5, 0x803a8074ba1df480, +0xbec2be997c6127be, 0xcd13cd26de87ebcd, 0x34d034bde4688934, 0x483d487a75903248, +0xffdbffab24e354ff, 0x7af57af78ff48d7a, 0x907a90f4ea3d6490, 0x5f615fc23ebe9d5f, +0x2080201da0403d20, 0x68bd6867d5d00f68, 0x1a681ad07234ca1a, 0xae82ae192c41b7ae, +0xb4eab4c95e757db4, 0x544d549a19a8ce54, 0x937693ece53b7f93, 0x2288220daa442f22, +0x648d6407e9c86364, 0xf1e3f1db12ff2af1, 0x73d173bfa2e6cc73, 0x124812905a248212, +0x401d403a5d807a40, 0x0820084028104808, 0xc32bc356e89b95c3, 0xec97ec337bc5dfec, +0xdb4bdb9690ab4ddb, 0xa1bea1611f5fc0a1, 0x8d0e8d1c8307918d, 0x3df43df5c97ac83d, +0x976697ccf1335b97, 0x0000000000000000, 0xcf1bcf36d483f9cf, 0x2bac2b4587566e2b, +0x76c57697b3ece176, 0x82328264b019e682, 0xd67fd6fea9b128d6, 0x1b6c1bd87736c31b, +0xb5eeb5c15b7774b5, 0xaf86af112943beaf, 0x6ab56a77dfd41d6a, 0x505d50ba0da0ea50, +0x450945124c8a5745, 0xf3ebf3cb18fb38f3, 0x30c0309df060ad30, 0xef9bef2b74c3c4ef, +0x3ffc3fe5c37eda3f, 0x554955921caac755, 0xa2b2a2791059dba2, 0xea8fea0365c9e9ea, +0x6589650fecca6a65, 0xbad2bab9686903ba, 0x2fbc2f65935e4a2f, 0xc027c04ee79d8ec0, +0xde5fdebe81a160de, 0x1c701ce06c38fc1c, 0xfdd3fdbb2ee746fd, 0x4d294d52649a1f4d, +0x927292e4e0397692, 0x75c9758fbceafa75, 0x061806301e0c3606, 0x8a128a249809ae8a, +0xb2f2b2f940794bb2, 0xe6bfe66359d185e6, 0x0e380e70361c7e0e, 0x1f7c1ff8633ee71f, +0x62956237f7c45562, 0xd477d4eea3b53ad4, 0xa89aa829324d81a8, 0x966296c4f4315296, +0xf9c3f99b3aef62f9, 0xc533c566f697a3c5, 0x25942535b14a1025, 0x597959f220b2ab59, +0x842a8454ae15d084, 0x72d572b7a7e4c572, 0x39e439d5dd72ec39, 0x4c2d4c5a6198164c, +0x5e655eca3bbc945e, 0x78fd78e785f09f78, 0x38e038ddd870e538, 0x8c0a8c148605988c, +0xd163d1c6b2bf17d1, 0xa5aea5410b57e4a5, 0xe2afe2434dd9a1e2, 0x6199612ff8c24e61, +0xb3f6b3f1457b42b3, 0x21842115a5423421, 0x9c4a9c94d625089c, 0x1e781ef0663cee1e, +0x4311432252866143, 0xc73bc776fc93b1c7, 0xfcd7fcb32be54ffc, 0x0410042014082404, +0x515951b208a2e351, 0x995e99bcc72f2599, 0x6da96d4fc4da226d, 0x0d340d68391a650d, +0xfacffa8335e979fa, 0xdf5bdfb684a369df, 0x7ee57ed79bfca97e, 0x2490243db4481924, +0x3bec3bc5d776fe3b, 0xab96ab313d4b9aab, 0xce1fce3ed181f0ce, 0x1144118855229911, +0x8f068f0c8903838f, 0x4e254e4a6b9c044e, 0xb7e6b7d1517366b7, 0xeb8beb0b60cbe0eb, +0x3cf03cfdcc78c13c, 0x813e817cbf1ffd81, 0x946a94d4fe354094, 0xf7fbf7eb0cf31cf7, +0xb9deb9a1676f18b9, 0x134c13985f268b13, 0x2cb02c7d9c58512c, 0xd36bd3d6b8bb05d3, +0xe7bbe76b5cd38ce7, 0x6ea56e57cbdc396e, 0xc437c46ef395aac4, 0x030c03180f061b03, +0x5645568a13acdc56, 0x440d441a49885e44, 0x7fe17fdf9efea07f, 0xa99ea921374f88a9, +0x2aa82a4d8254672a, 0xbbd6bbb16d6b0abb, 0xc123c146e29f87c1, 0x535153a202a6f153, +0xdc57dcae8ba572dc, 0x0b2c0b582716530b, 0x9d4e9d9cd327019d, 0x6cad6c47c1d82b6c, +0x31c43195f562a431, 0x74cd7487b9e8f374, 0xf6fff6e309f115f6, 0x4605460a438c4c46, +0xac8aac092645a5ac, 0x891e893c970fb589, 0x145014a04428b414, 0xe1a3e15b42dfbae1, +0x165816b04e2ca616, 0x3ae83acdd274f73a, 0x69b9696fd0d20669, 0x092409482d124109, +0x70dd70a7ade0d770, 0xb6e2b6d954716fb6, 0xd067d0ceb7bd1ed0, 0xed93ed3b7ec7d6ed, +0xcc17cc2edb85e2cc, 0x4215422a57846842, 0x985a98b4c22d2c98, 0xa4aaa4490e55eda4, +0x28a0285d88507528, 0x5c6d5cda31b8865c, 0xf8c7f8933fed6bf8, 0x86228644a411c286, +] + +rc = [ +0x0000000000000000, +0x1823c6e887b8014f, +0x36a6d2f5796f9152, +0x60bc9b8ea30c7b35, +0x1de0d7c22e4bfe57, +0x157737e59ff04ada, +0x58c9290ab1a06b85, +0xbd5d10f4cb3e0567, +0xe427418ba77d95d8, +0xfbee7c66dd17479e, +0xca2dbf07ad5a8333 +] + +DIGESTBYTES = 64 +class WhirlpoolStruct: + def __init__(self): + self.bitLength = [0]*32 + self.buffer = [0]*64 + self.bufferBits = 0 + self.bufferPos = 0 + self.hash = [0]*8 + +def WhirlpoolInit(ctx): + ctx = WhirlpoolStruct() + return + +def WhirlpoolAdd(source, sourceBits, ctx): + if sys.hexversion < 0x3000000: + source = [ord(s)&0xff for s in source] + + carry = 0 + value = sourceBits + i = 31 + while i >= 0 and (carry != 0 or value != 0): + carry += ctx.bitLength[i] + ((value % 0x100000000) & 0xff) + ctx.bitLength[i] = carry % 0x100 + carry >>= 8 + value >>= 8 + i -= 1 + + bufferBits = ctx.bufferBits + bufferPos = ctx.bufferPos + sourcePos = 0 + sourceGap = (8 - (sourceBits & 7)) & 7 + bufferRem = ctx.bufferBits & 7 + buffr = ctx.buffer + + while sourceBits > 8: + b = ((source[sourcePos] << sourceGap) & 0xff) | ((source[sourcePos + 1] & 0xff) >> (8 - sourceGap)) + buffr[bufferPos] |= (b >> bufferRem) % 0x100 + bufferPos += 1 + bufferBits += 8 - bufferRem + if bufferBits == 512: + processBuffer(ctx) + bufferBits = 0 + bufferPos = 0 + + buffr[bufferPos] = b << (8 - bufferRem) + bufferBits += bufferRem + + sourceBits -= 8 + sourcePos += 1 + + b = (source[sourcePos] << sourceGap) & 0xff + buffr[bufferPos] |= b >> bufferRem + if bufferRem + sourceBits < 8: + bufferBits += sourceBits + else: + bufferPos += 1 + bufferBits += 8 - bufferRem + sourceBits -= 8 - bufferRem + if bufferBits == 512: + processBuffer(ctx) + bufferBits = 0 + bufferPos = 0 + buffr[bufferPos] = b << (8 - bufferRem) + bufferBits += sourceBits + ctx.bufferBits = bufferBits + ctx.bufferPos = bufferPos + +def WhirlpoolFinalize(ctx): + bufferPos = ctx.bufferPos + ctx.buffer[bufferPos] |= 0x80 >> (ctx.bufferBits & 7) + bufferPos += 1 + if bufferPos > 32: + if bufferPos < 64: + for i in xrange(64 - bufferPos): + ctx.buffer[bufferPos+i] = 0 + processBuffer(ctx) + bufferPos = 0 + if bufferPos < 32: + for i in xrange(32 - bufferPos): + ctx.buffer[bufferPos+i] = 0 + bufferPos = 32 + for i in xrange(32): + ctx.buffer[32+i] = ctx.bitLength[i] + processBuffer(ctx) + digest = '' + for i in xrange(8): + digest += chr((ctx.hash[i] >> 56) % 0x100) + digest += chr((ctx.hash[i] >> 48) % 0x100) + digest += chr((ctx.hash[i] >> 40) % 0x100) + digest += chr((ctx.hash[i] >> 32) % 0x100) + digest += chr((ctx.hash[i] >> 24) % 0x100) + digest += chr((ctx.hash[i] >> 16) % 0x100) + digest += chr((ctx.hash[i] >> 8) % 0x100) + digest += chr((ctx.hash[i]) % 0x100) + ctx.bufferPos = bufferPos + return digest + +def CDo(buf, a0, a1, a2, a3, a4, a5, a6, a7): + return C0[((buf[a0] >> 56) % 0x100000000) & 0xff] ^ \ + C1[((buf[a1] >> 48) % 0x100000000) & 0xff] ^ \ + C2[((buf[a2] >> 40) % 0x100000000) & 0xff] ^ \ + C3[((buf[a3] >> 32) % 0x100000000) & 0xff] ^ \ + C4[((buf[a4] >> 24) % 0x100000000) & 0xff] ^ \ + C5[((buf[a5] >> 16) % 0x100000000) & 0xff] ^ \ + C6[((buf[a6] >> 8) % 0x100000000) & 0xff] ^ \ + C7[((buf[a7] >> 0) % 0x100000000) & 0xff] + +def processBuffer(ctx): + i, r = 0, 0 + K = [0]*8 + block = [0]*8 + state = [0]*8 + L = [0]*8 + buffr = ctx.buffer + + buf_cnt = 0 + for i in xrange(8): + block[i] = ((buffr[buf_cnt+0] & 0xff) << 56) ^ \ + ((buffr[buf_cnt+1] & 0xff) << 48) ^ \ + ((buffr[buf_cnt+2] & 0xff) << 40) ^ \ + ((buffr[buf_cnt+3] & 0xff) << 32) ^ \ + ((buffr[buf_cnt+4] & 0xff) << 24) ^ \ + ((buffr[buf_cnt+5] & 0xff) << 16) ^ \ + ((buffr[buf_cnt+6] & 0xff) << 8) ^ \ + ((buffr[buf_cnt+7] & 0xff) << 0) + buf_cnt += 8 + for i in xrange(8): + K[i] = ctx.hash[i] + state[i] = block[i] ^ K[i] + + for r in xrange(1, R+1): + L[0] = CDo(K, 0, 7, 6, 5, 4, 3, 2, 1) ^ rc[r] + L[1] = CDo(K, 1, 0, 7, 6, 5, 4, 3, 2) + L[2] = CDo(K, 2, 1, 0, 7, 6, 5, 4, 3) + L[3] = CDo(K, 3, 2, 1, 0, 7, 6, 5, 4) + L[4] = CDo(K, 4, 3, 2, 1, 0, 7, 6, 5) + L[5] = CDo(K, 5, 4, 3, 2, 1, 0, 7, 6) + L[6] = CDo(K, 6, 5, 4, 3, 2, 1, 0, 7) + L[7] = CDo(K, 7, 6, 5, 4, 3, 2, 1, 0) + for i in xrange(8): + K[i] = L[i] + L[0] = CDo(state, 0, 7, 6, 5, 4, 3, 2, 1) ^ K[0] + L[1] = CDo(state, 1, 0, 7, 6, 5, 4, 3, 2) ^ K[1] + L[2] = CDo(state, 2, 1, 0, 7, 6, 5, 4, 3) ^ K[2] + L[3] = CDo(state, 3, 2, 1, 0, 7, 6, 5, 4) ^ K[3] + L[4] = CDo(state, 4, 3, 2, 1, 0, 7, 6, 5) ^ K[4] + L[5] = CDo(state, 5, 4, 3, 2, 1, 0, 7, 6) ^ K[5] + L[6] = CDo(state, 6, 5, 4, 3, 2, 1, 0, 7) ^ K[6] + L[7] = CDo(state, 7, 6, 5, 4, 3, 2, 1, 0) ^ K[7] + for i in xrange(8): + state[i] = L[i] + # apply the Miyaguchi-Preneel compression function + for i in xrange(8): + ctx.hash[i] ^= state[i] ^ block[i] + return + +# +# Tests. +# + +if __name__ == '__main__': + assert Whirlpool(b'The quick brown fox jumps over the lazy dog').hexdigest() == \ + 'b97de512e91e3828b40d2b0fdce9ceb3c4a71f9bea8d88e75c4fa854df36725fd2b52eb6544edcacd6f8beddfea403cb55ae31f03ad62a5ef54e42ee82c3fb35' + assert Whirlpool(b'The quick brown fox jumps over the lazy eog').hexdigest() == \ + 'c27ba124205f72e6847f3e19834f925cc666d0974167af915bb462420ed40cc50900d85a1f923219d832357750492d5c143011a76988344c2635e69d06f2d38c' + assert Whirlpool(b'').hexdigest() == \ + '19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3' diff --git a/portage_with_autodep/pym/portage/util/whirlpool.pyo b/portage_with_autodep/pym/portage/util/whirlpool.pyo Binary files differnew file mode 100644 index 0000000..4bcf49a --- /dev/null +++ b/portage_with_autodep/pym/portage/util/whirlpool.pyo diff --git a/portage_with_autodep/pym/portage/versions.py b/portage_with_autodep/pym/portage/versions.py index f8691d1..db14e99 100644 --- a/portage_with_autodep/pym/portage/versions.py +++ b/portage_with_autodep/pym/portage/versions.py @@ -1,5 +1,5 @@ # versions.py -- core Portage functionality -# Copyright 1998-2010 Gentoo Foundation +# Copyright 1998-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 __all__ = [ @@ -9,14 +9,26 @@ __all__ = [ ] import re +import sys import warnings +if sys.hexversion < 0x3000000: + _unicode = unicode +else: + _unicode = str + import portage portage.proxy.lazyimport.lazyimport(globals(), - 'portage.util:cmp_sort_key' + 'portage.repository.config:_gen_valid_repo', + 'portage.util:cmp_sort_key', ) +from portage import _unicode_decode +from portage.eapi import eapi_allows_dots_in_PN +from portage.exception import InvalidData from portage.localization import _ +_unknown_repo = "__unknown__" + # \w is [a-zA-Z0-9_] # 2.1.1 A category name may contain any of the characters [A-Za-z0-9+_.-]. @@ -26,15 +38,27 @@ _cat = r'[\w+][\w+.-]*' # 2.1.2 A package name may contain any of the characters [A-Za-z0-9+_-]. # It must not begin with a hyphen, # and must not end in a hyphen followed by one or more digits. -_pkg = r'[\w+][\w+-]*?' +_pkg = { + "dots_disallowed_in_PN": r'[\w+][\w+-]*?', + "dots_allowed_in_PN": r'[\w+][\w+.-]*?', +} _v = r'(cvs\.)?(\d+)((\.\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\d*)*)' _rev = r'\d+' _vr = _v + '(-r(' + _rev + '))?' -_cp = '(' + _cat + '/' + _pkg + '(-' + _vr + ')?)' -_cpv = '(' + _cp + '-' + _vr + ')' -_pv = '(?P<pn>' + _pkg + '(?P<pn_inval>-' + _vr + ')?)' + '-(?P<ver>' + _v + ')(-r(?P<rev>' + _rev + '))?' +_cp = { + "dots_disallowed_in_PN": '(' + _cat + '/' + _pkg['dots_disallowed_in_PN'] + '(-' + _vr + ')?)', + "dots_allowed_in_PN": '(' + _cat + '/' + _pkg['dots_allowed_in_PN'] + '(-' + _vr + ')?)', +} +_cpv = { + "dots_disallowed_in_PN": '(' + _cp['dots_disallowed_in_PN'] + '-' + _vr + ')', + "dots_allowed_in_PN": '(' + _cp['dots_allowed_in_PN'] + '-' + _vr + ')', +} +_pv = { + "dots_disallowed_in_PN": '(?P<pn>' + _pkg['dots_disallowed_in_PN'] + '(?P<pn_inval>-' + _vr + ')?)' + '-(?P<ver>' + _v + ')(-r(?P<rev>' + _rev + '))?', + "dots_allowed_in_PN": '(?P<pn>' + _pkg['dots_allowed_in_PN'] + '(?P<pn_inval>-' + _vr + ')?)' + '-(?P<ver>' + _v + ')(-r(?P<rev>' + _rev + '))?', +} ver_regexp = re.compile("^" + _vr + "$") suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$") @@ -49,7 +73,6 @@ def ververify(myver, silent=1): print(_("!!! syntax error in version: %s") % myver) return 0 -vercmp_cache = {} def vercmp(ver1, ver2, silent=1): """ Compare two versions @@ -76,11 +99,7 @@ def vercmp(ver1, ver2, silent=1): if ver1 == ver2: return 0 - mykey=ver1+":"+ver2 - try: - return vercmp_cache[mykey] - except KeyError: - pass + match1 = ver_regexp.match(ver1) match2 = ver_regexp.match(ver2) @@ -96,10 +115,8 @@ def vercmp(ver1, ver2, silent=1): # shortcut for cvs ebuilds (new style) if match1.group(1) and not match2.group(1): - vercmp_cache[mykey] = 1 return 1 elif match2.group(1) and not match1.group(1): - vercmp_cache[mykey] = -1 return -1 # building lists of the version parts before the suffix @@ -153,16 +170,13 @@ def vercmp(ver1, ver2, silent=1): for i in range(0, max(len(list1), len(list2))): if len(list1) <= i: - vercmp_cache[mykey] = -1 return -1 elif len(list2) <= i: - vercmp_cache[mykey] = 1 return 1 elif list1[i] != list2[i]: a = list1[i] b = list2[i] rval = (a > b) - (a < b) - vercmp_cache[mykey] = rval return rval # main version is equal, so now compare the _suffix part @@ -183,7 +197,6 @@ def vercmp(ver1, ver2, silent=1): a = suffix_value[s1[0]] b = suffix_value[s2[0]] rval = (a > b) - (a < b) - vercmp_cache[mykey] = rval return rval if s1[1] != s2[1]: # it's possible that the s(1|2)[1] == '' @@ -198,7 +211,6 @@ def vercmp(ver1, ver2, silent=1): r2 = 0 rval = (r1 > r2) - (r1 < r2) if rval: - vercmp_cache[mykey] = rval return rval # the suffix part is equal to, so finally check the revision @@ -211,7 +223,6 @@ def vercmp(ver1, ver2, silent=1): else: r2 = 0 rval = (r1 > r2) - (r1 < r2) - vercmp_cache[mykey] = rval return rval def pkgcmp(pkg1, pkg2): @@ -240,16 +251,25 @@ def pkgcmp(pkg1, pkg2): return None return vercmp("-".join(pkg1[1:]), "-".join(pkg2[1:])) -_pv_re = re.compile('^' + _pv + '$', re.VERBOSE) +_pv_re = { + "dots_disallowed_in_PN": re.compile('^' + _pv['dots_disallowed_in_PN'] + '$', re.VERBOSE), + "dots_allowed_in_PN": re.compile('^' + _pv['dots_allowed_in_PN'] + '$', re.VERBOSE), +} -def _pkgsplit(mypkg): +def _get_pv_re(eapi): + if eapi is None or eapi_allows_dots_in_PN(eapi): + return _pv_re["dots_allowed_in_PN"] + else: + return _pv_re["dots_disallowed_in_PN"] + +def _pkgsplit(mypkg, eapi=None): """ @param mypkg: pv @return: 1. None if input is invalid. 2. (pn, ver, rev) if input is pv """ - m = _pv_re.match(mypkg) + m = _get_pv_re(eapi).match(mypkg) if m is None: return None @@ -266,8 +286,8 @@ def _pkgsplit(mypkg): _cat_re = re.compile('^%s$' % _cat) _missing_cat = 'null' -catcache={} -def catpkgsplit(mydata,silent=1): + +def catpkgsplit(mydata, silent=1, eapi=None): """ Takes a Category/Package-Version-Rev and returns a list of each. @@ -281,28 +301,65 @@ def catpkgsplit(mydata,silent=1): 2. If cat is not specificed in mydata, cat will be "null" 3. if rev does not exist it will be '-r0' """ - try: - return catcache[mydata] - except KeyError: + return mydata.cpv_split + except AttributeError: pass mysplit = mydata.split('/', 1) p_split=None if len(mysplit)==1: cat = _missing_cat - p_split = _pkgsplit(mydata) + p_split = _pkgsplit(mydata, eapi=eapi) elif len(mysplit)==2: cat = mysplit[0] if _cat_re.match(cat) is not None: - p_split = _pkgsplit(mysplit[1]) + p_split = _pkgsplit(mysplit[1], eapi=eapi) if not p_split: - catcache[mydata]=None return None retval = (cat, p_split[0], p_split[1], p_split[2]) - catcache[mydata]=retval return retval -def pkgsplit(mypkg, silent=1): +class _pkg_str(_unicode): + """ + This class represents a cpv. It inherits from str (unicode in python2) and + has attributes that cache results for use by functions like catpkgsplit and + cpv_getkey which are called frequently (especially in match_from_list). + Instances are typically created in dbapi.cp_list() or the Atom contructor, + and propagate from there. Generally, code that pickles these objects will + manually convert them to a plain unicode object first. + """ + + def __new__(cls, cpv, slot=None, repo=None, eapi=None): + return _unicode.__new__(cls, cpv) + + def __init__(self, cpv, slot=None, repo=None, eapi=None): + if not isinstance(cpv, _unicode): + # Avoid TypeError from _unicode.__init__ with PyPy. + cpv = _unicode_decode(cpv) + _unicode.__init__(cpv) + self.__dict__['cpv_split'] = catpkgsplit(cpv, eapi=eapi) + if self.cpv_split is None: + raise InvalidData(cpv) + self.__dict__['cp'] = self.cpv_split[0] + '/' + self.cpv_split[1] + if self.cpv_split[-1] == "r0" and cpv[-3:] != "-r0": + self.__dict__['version'] = "-".join(self.cpv_split[2:-1]) + else: + self.__dict__['version'] = "-".join(self.cpv_split[2:]) + # for match_from_list introspection + self.__dict__['cpv'] = self + if slot is not None: + self.__dict__['slot'] = slot + if repo is not None: + repo = _gen_valid_repo(repo) + if not repo: + repo = _unknown_repo + self.__dict__['repo'] = repo + + def __setattr__(self, name, value): + raise AttributeError("_pkg_str instances are immutable", + self.__class__, name, value) + +def pkgsplit(mypkg, silent=1, eapi=None): """ @param mypkg: either a pv or cpv @return: @@ -310,7 +367,7 @@ def pkgsplit(mypkg, silent=1): 2. (pn, ver, rev) if input is pv 3. (cp, ver, rev) if input is a cpv """ - catpsplit = catpkgsplit(mypkg) + catpsplit = catpkgsplit(mypkg, eapi=eapi) if catpsplit is None: return None cat, pn, ver, rev = catpsplit @@ -319,9 +376,13 @@ def pkgsplit(mypkg, silent=1): else: return (cat + '/' + pn, ver, rev) -def cpv_getkey(mycpv): +def cpv_getkey(mycpv, eapi=None): """Calls catpkgsplit on a cpv and returns only the cp.""" - mysplit = catpkgsplit(mycpv) + try: + return mycpv.cp + except AttributeError: + pass + mysplit = catpkgsplit(mycpv, eapi=eapi) if mysplit is not None: return mysplit[0] + '/' + mysplit[1] @@ -330,7 +391,7 @@ def cpv_getkey(mycpv): DeprecationWarning, stacklevel=2) myslash = mycpv.split("/", 1) - mysplit = _pkgsplit(myslash[-1]) + mysplit = _pkgsplit(myslash[-1], eapi=eapi) if mysplit is None: return None mylen = len(myslash) @@ -339,14 +400,18 @@ def cpv_getkey(mycpv): else: return mysplit[0] -def cpv_getversion(mycpv): +def cpv_getversion(mycpv, eapi=None): """Returns the v (including revision) from an cpv.""" - cp = cpv_getkey(mycpv) + try: + return mycpv.version + except AttributeError: + pass + cp = cpv_getkey(mycpv, eapi=eapi) if cp is None: return None return mycpv[len(cp+"-"):] -def cpv_sort_key(): +def cpv_sort_key(eapi=None): """ Create an object for sorting cpvs, to be used as the 'key' parameter in places like list.sort() or sorted(). This calls catpkgsplit() once for @@ -365,39 +430,55 @@ def cpv_sort_key(): split1 = split_cache.get(cpv1, False) if split1 is False: - split1 = catpkgsplit(cpv1) - if split1 is not None: - split1 = (split1[:2], '-'.join(split1[2:])) + split1 = None + try: + split1 = cpv1.cpv + except AttributeError: + try: + split1 = _pkg_str(cpv1, eapi=eapi) + except InvalidData: + pass split_cache[cpv1] = split1 split2 = split_cache.get(cpv2, False) if split2 is False: - split2 = catpkgsplit(cpv2) - if split2 is not None: - split2 = (split2[:2], '-'.join(split2[2:])) + split2 = None + try: + split2 = cpv2.cpv + except AttributeError: + try: + split2 = _pkg_str(cpv2, eapi=eapi) + except InvalidData: + pass split_cache[cpv2] = split2 - if split1 is None or split2 is None or split1[0] != split2[0]: + if split1 is None or split2 is None or split1.cp != split2.cp: return (cpv1 > cpv2) - (cpv1 < cpv2) - return vercmp(split1[1], split2[1]) + return vercmp(split1.version, split2.version) return cmp_sort_key(cmp_cpv) def catsplit(mydep): return mydep.split("/", 1) -def best(mymatches): +def best(mymatches, eapi=None): """Accepts None arguments; assumes matches are valid.""" if not mymatches: return "" if len(mymatches) == 1: return mymatches[0] bestmatch = mymatches[0] - p2 = catpkgsplit(bestmatch)[1:] + try: + v2 = bestmatch.version + except AttributeError: + v2 = _pkg_str(bestmatch, eapi=eapi).version for x in mymatches[1:]: - p1 = catpkgsplit(x)[1:] - if pkgcmp(p1, p2) > 0: + try: + v1 = x.version + except AttributeError: + v1 = _pkg_str(x, eapi=eapi).version + if vercmp(v1, v2) > 0: bestmatch = x - p2 = catpkgsplit(bestmatch)[1:] + v2 = v1 return bestmatch diff --git a/portage_with_autodep/pym/portage/versions.pyo b/portage_with_autodep/pym/portage/versions.pyo Binary files differnew file mode 100644 index 0000000..eae2743 --- /dev/null +++ b/portage_with_autodep/pym/portage/versions.pyo diff --git a/portage_with_autodep/pym/portage/xml/__init__.pyo b/portage_with_autodep/pym/portage/xml/__init__.pyo Binary files differnew file mode 100644 index 0000000..15f1b77 --- /dev/null +++ b/portage_with_autodep/pym/portage/xml/__init__.pyo diff --git a/portage_with_autodep/pym/portage/xml/metadata.py b/portage_with_autodep/pym/portage/xml/metadata.py index 7acc1f3..25f801a 100644 --- a/portage_with_autodep/pym/portage/xml/metadata.py +++ b/portage_with_autodep/pym/portage/xml/metadata.py @@ -1,4 +1,4 @@ -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """Provides an easy-to-use python interface to Gentoo's metadata.xml file. @@ -30,16 +30,40 @@ __all__ = ('MetaDataXML',) -try: - import xml.etree.cElementTree as etree -except ImportError: +import sys + +if sys.hexversion < 0x2070000 or \ + (sys.hexversion < 0x3020000 and sys.hexversion >= 0x3000000): + # Our _MetadataTreeBuilder usage is incompatible with + # cElementTree in Python 2.6, 3.0, and 3.1: + # File "/usr/lib/python2.6/xml/etree/ElementTree.py", line 644, in findall + # assert self._root is not None import xml.etree.ElementTree as etree +else: + try: + import xml.etree.cElementTree as etree + except (ImportError, SystemError): + import xml.etree.ElementTree as etree + +try: + from xml.parsers.expat import ExpatError +except (ImportError, SystemError): + ExpatError = SyntaxError import re +import xml.etree.ElementTree import portage -from portage import os +from portage import os, _unicode_decode from portage.util import unique_everseen +class _MetadataTreeBuilder(xml.etree.ElementTree.TreeBuilder): + """ + Implements doctype() as required to avoid deprecation warnings with + Python >=2.7. + """ + def doctype(self, name, pubid, system): + pass + class _Maintainer(object): """An object for representing one maintainer. @@ -63,8 +87,7 @@ class _Maintainer(object): self.description = None self.restrict = node.get('restrict') self.status = node.get('status') - maint_attrs = node.getchildren() - for attr in maint_attrs: + for attr in node: setattr(self, attr.tag, attr.text) def __repr__(self): @@ -174,9 +197,12 @@ class MetaDataXML(object): self._xml_tree = None try: - self._xml_tree = etree.parse(metadata_xml_path) + self._xml_tree = etree.parse(metadata_xml_path, + parser=etree.XMLParser(target=_MetadataTreeBuilder())) except ImportError: pass + except ExpatError as e: + raise SyntaxError(_unicode_decode("%s") % (e,)) if isinstance(herds, etree.ElementTree): herds_etree = herds @@ -209,7 +235,8 @@ class MetaDataXML(object): if self._herdstree is None: try: - self._herdstree = etree.parse(self._herds_path) + self._herdstree = etree.parse(self._herds_path, + parser=etree.XMLParser(target=_MetadataTreeBuilder())) except (ImportError, IOError, SyntaxError): return None @@ -217,7 +244,13 @@ class MetaDataXML(object): if herd in ('no-herd', 'maintainer-wanted', 'maintainer-needed'): return None - for node in self._herdstree.getiterator('herd'): + try: + # Python 2.7 or >=3.2 + iterate = self._herdstree.iter + except AttributeError: + iterate = self._herdstree.getiterator + + for node in iterate('herd'): if node.findtext('name') == herd: return node.findtext('email') @@ -292,8 +325,13 @@ class MetaDataXML(object): if self._xml_tree is None: self._useflags = tuple() else: + try: + # Python 2.7 or >=3.2 + iterate = self._xml_tree.iter + except AttributeError: + iterate = self._xml_tree.getiterator self._useflags = tuple(_Useflag(node) \ - for node in self._xml_tree.getiterator('flag')) + for node in iterate('flag')) return self._useflags diff --git a/portage_with_autodep/pym/portage/xml/metadata.pyo b/portage_with_autodep/pym/portage/xml/metadata.pyo Binary files differnew file mode 100644 index 0000000..0103456 --- /dev/null +++ b/portage_with_autodep/pym/portage/xml/metadata.pyo diff --git a/portage_with_autodep/pym/portage/xpak.py b/portage_with_autodep/pym/portage/xpak.py index 7487d67..3262326 100644 --- a/portage_with_autodep/pym/portage/xpak.py +++ b/portage_with_autodep/pym/portage/xpak.py @@ -1,4 +1,4 @@ -# Copyright 2001-2010 Gentoo Foundation +# Copyright 2001-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 @@ -22,11 +22,11 @@ __all__ = ['addtolist', 'decodeint', 'encodeint', 'getboth', import array import errno -import shutil import sys import portage from portage import os +from portage import shutil from portage import normalize_path from portage import _encodings from portage import _unicode_decode @@ -62,11 +62,15 @@ def encodeint(myint): """Takes a 4 byte integer and converts it into a string of 4 characters. Returns the characters in a string.""" a = array.array('B') - a.append((myint >> 24 ) & 0xff) - a.append((myint >> 16 ) & 0xff) - a.append((myint >> 8 ) & 0xff) + a.append((myint >> 24) & 0xff) + a.append((myint >> 16) & 0xff) + a.append((myint >> 8) & 0xff) a.append(myint & 0xff) - return a.tostring() + try: + # Python >= 3.2 + return a.tobytes() + except AttributeError: + return a.tostring() def decodeint(mystring): """Takes a 4 byte string and converts it into a 4 byte integer. @@ -80,12 +84,12 @@ def decodeint(mystring): myint += mystring[0] << 24 return myint -def xpak(rootdir,outfile=None): - """(rootdir,outfile) -- creates an xpak segment of the directory 'rootdir' +def xpak(rootdir, outfile=None): + """(rootdir, outfile) -- creates an xpak segment of the directory 'rootdir' and under the name 'outfile' if it is specified. Otherwise it returns the xpak segment.""" - mylist=[] + mylist = [] addtolist(mylist, rootdir) mylist.sort() @@ -95,7 +99,8 @@ def xpak(rootdir,outfile=None): # CONTENTS is generated during the merge process. continue x = _unicode_encode(x, encoding=_encodings['fs'], errors='strict') - mydata[x] = open(os.path.join(rootdir, x), 'rb').read() + with open(os.path.join(rootdir, x), 'rb') as f: + mydata[x] = f.read() xpak_segment = xpak_mem(mydata) if outfile: @@ -107,7 +112,7 @@ def xpak(rootdir,outfile=None): return xpak_segment def xpak_mem(mydata): - """Create an xpack segement from a map object.""" + """Create an xpack segment from a map object.""" mydata_encoded = {} for k, v in mydata.items(): @@ -120,21 +125,21 @@ def xpak_mem(mydata): del mydata_encoded indexglob = b'' - indexpos=0 + indexpos = 0 dataglob = b'' - datapos=0 + datapos = 0 for x, newglob in mydata.items(): - mydatasize=len(newglob) - indexglob=indexglob+encodeint(len(x))+x+encodeint(datapos)+encodeint(mydatasize) - indexpos=indexpos+4+len(x)+4+4 - dataglob=dataglob+newglob - datapos=datapos+mydatasize + mydatasize = len(newglob) + indexglob = indexglob + encodeint(len(x)) + x + encodeint(datapos) + encodeint(mydatasize) + indexpos = indexpos + 4 + len(x) + 4 + 4 + dataglob = dataglob + newglob + datapos = datapos + mydatasize return b'XPAKPACK' \ - + encodeint(len(indexglob)) \ - + encodeint(len(dataglob)) \ - + indexglob \ - + dataglob \ - + b'XPAKSTOP' + + encodeint(len(indexglob)) \ + + encodeint(len(dataglob)) \ + + indexglob \ + + dataglob \ + + b'XPAKSTOP' def xsplit(infile): """(infile) -- Splits the infile into two files. @@ -144,7 +149,7 @@ def xsplit(infile): encoding=_encodings['fs'], errors='strict') myfile = open(_unicode_encode(infile, encoding=_encodings['fs'], errors='strict'), 'rb') - mydat=myfile.read() + mydat = myfile.read() myfile.close() splits = xsplit_mem(mydat) @@ -166,35 +171,35 @@ def xsplit_mem(mydat): return None if mydat[-8:] != b'XPAKSTOP': return None - indexsize=decodeint(mydat[8:12]) - return (mydat[16:indexsize+16], mydat[indexsize+16:-8]) + indexsize = decodeint(mydat[8:12]) + return (mydat[16:indexsize + 16], mydat[indexsize + 16:-8]) def getindex(infile): """(infile) -- grabs the index segment from the infile and returns it.""" myfile = open(_unicode_encode(infile, encoding=_encodings['fs'], errors='strict'), 'rb') - myheader=myfile.read(16) + myheader = myfile.read(16) if myheader[0:8] != b'XPAKPACK': myfile.close() return - indexsize=decodeint(myheader[8:12]) - myindex=myfile.read(indexsize) + indexsize = decodeint(myheader[8:12]) + myindex = myfile.read(indexsize) myfile.close() return myindex def getboth(infile): """(infile) -- grabs the index and data segments from the infile. - Returns an array [indexSegment,dataSegment]""" + Returns an array [indexSegment, dataSegment]""" myfile = open(_unicode_encode(infile, encoding=_encodings['fs'], errors='strict'), 'rb') - myheader=myfile.read(16) + myheader = myfile.read(16) if myheader[0:8] != b'XPAKPACK': myfile.close() return - indexsize=decodeint(myheader[8:12]) - datasize=decodeint(myheader[12:16]) - myindex=myfile.read(indexsize) - mydata=myfile.read(datasize) + indexsize = decodeint(myheader[8:12]) + datasize = decodeint(myheader[12:16]) + myindex = myfile.read(indexsize) + mydata = myfile.read(datasize) myfile.close() return myindex, mydata @@ -205,83 +210,82 @@ def listindex(myindex): def getindex_mem(myindex): """Returns the filenames listed in the indexglob passed in.""" - myindexlen=len(myindex) - startpos=0 - myret=[] - while ((startpos+8)<myindexlen): - mytestlen=decodeint(myindex[startpos:startpos+4]) - myret=myret+[myindex[startpos+4:startpos+4+mytestlen]] - startpos=startpos+mytestlen+12 + myindexlen = len(myindex) + startpos = 0 + myret = [] + while ((startpos + 8) < myindexlen): + mytestlen = decodeint(myindex[startpos:startpos + 4]) + myret = myret + [myindex[startpos + 4:startpos + 4 + mytestlen]] + startpos = startpos + mytestlen + 12 return myret -def searchindex(myindex,myitem): - """(index,item) -- Finds the offset and length of the file 'item' in the +def searchindex(myindex, myitem): + """(index, item) -- Finds the offset and length of the file 'item' in the datasegment via the index 'index' provided.""" myitem = _unicode_encode(myitem, encoding=_encodings['repo.content'], errors='backslashreplace') - mylen=len(myitem) - myindexlen=len(myindex) - startpos=0 - while ((startpos+8)<myindexlen): - mytestlen=decodeint(myindex[startpos:startpos+4]) - if mytestlen==mylen: - if myitem==myindex[startpos+4:startpos+4+mytestlen]: + mylen = len(myitem) + myindexlen = len(myindex) + startpos = 0 + while ((startpos + 8) < myindexlen): + mytestlen = decodeint(myindex[startpos:startpos + 4]) + if mytestlen == mylen: + if myitem == myindex[startpos + 4:startpos + 4 + mytestlen]: #found - datapos=decodeint(myindex[startpos+4+mytestlen:startpos+8+mytestlen]); - datalen=decodeint(myindex[startpos+8+mytestlen:startpos+12+mytestlen]); + datapos = decodeint(myindex[startpos + 4 + mytestlen:startpos + 8 + mytestlen]) + datalen = decodeint(myindex[startpos + 8 + mytestlen:startpos + 12 + mytestlen]) return datapos, datalen - startpos=startpos+mytestlen+12 + startpos = startpos + mytestlen + 12 -def getitem(myid,myitem): - myindex=myid[0] - mydata=myid[1] - myloc=searchindex(myindex,myitem) +def getitem(myid, myitem): + myindex = myid[0] + mydata = myid[1] + myloc = searchindex(myindex, myitem) if not myloc: return None - return mydata[myloc[0]:myloc[0]+myloc[1]] - -def xpand(myid,mydest): - myindex=myid[0] - mydata=myid[1] - try: - origdir=os.getcwd() - except SystemExit as e: - raise - except: - os.chdir("/") - origdir="/" - os.chdir(mydest) - myindexlen=len(myindex) - startpos=0 - while ((startpos+8)<myindexlen): - namelen=decodeint(myindex[startpos:startpos+4]) - datapos=decodeint(myindex[startpos+4+namelen:startpos+8+namelen]); - datalen=decodeint(myindex[startpos+8+namelen:startpos+12+namelen]); - myname=myindex[startpos+4:startpos+4+namelen] - dirname=os.path.dirname(myname) + return mydata[myloc[0]:myloc[0] + myloc[1]] + +def xpand(myid, mydest): + mydest = normalize_path(mydest) + os.sep + myindex = myid[0] + mydata = myid[1] + myindexlen = len(myindex) + startpos = 0 + while ((startpos + 8) < myindexlen): + namelen = decodeint(myindex[startpos:startpos + 4]) + datapos = decodeint(myindex[startpos + 4 + namelen:startpos + 8 + namelen]) + datalen = decodeint(myindex[startpos + 8 + namelen:startpos + 12 + namelen]) + myname = myindex[startpos + 4:startpos + 4 + namelen] + myname = _unicode_decode(myname, + encoding=_encodings['repo.content'], errors='replace') + filename = os.path.join(mydest, myname.lstrip(os.sep)) + filename = normalize_path(filename) + if not filename.startswith(mydest): + # myname contains invalid ../ component(s) + continue + dirname = os.path.dirname(filename) if dirname: if not os.path.exists(dirname): os.makedirs(dirname) - mydat = open(_unicode_encode(myname, + mydat = open(_unicode_encode(filename, encoding=_encodings['fs'], errors='strict'), 'wb') - mydat.write(mydata[datapos:datapos+datalen]) + mydat.write(mydata[datapos:datapos + datalen]) mydat.close() - startpos=startpos+namelen+12 - os.chdir(origdir) + startpos = startpos + namelen + 12 class tbz2(object): - def __init__(self,myfile): - self.file=myfile - self.filestat=None + def __init__(self, myfile): + self.file = myfile + self.filestat = None self.index = b'' - self.infosize=0 - self.xpaksize=0 - self.indexsize=None - self.datasize=None - self.indexpos=None - self.datapos=None - - def decompose(self,datadir,cleanup=1): + self.infosize = 0 + self.xpaksize = 0 + self.indexsize = None + self.datasize = None + self.indexpos = None + self.datapos = None + + def decompose(self, datadir, cleanup=1): """Alias for unpackinfo() --- Complement to recompose() but optionally deletes the destination directory. Extracts the xpak from the tbz2 into the directory provided. Raises IOError if scan() fails. @@ -293,9 +297,9 @@ class tbz2(object): if not os.path.exists(datadir): os.makedirs(datadir) return self.unpackinfo(datadir) - def compose(self,datadir,cleanup=0): + def compose(self, datadir, cleanup=0): """Alias for recompose().""" - return self.recompose(datadir,cleanup) + return self.recompose(datadir, cleanup) def recompose(self, datadir, cleanup=0, break_hardlinks=True): """Creates an xpak segment from the datadir provided, truncates the tbz2 @@ -333,9 +337,9 @@ class tbz2(object): encoding=_encodings['fs'], errors='strict'), 'ab+') if not myfile: raise IOError - myfile.seek(-self.xpaksize,2) # 0,2 or -0,2 just mean EOF. + myfile.seek(-self.xpaksize, 2) # 0,2 or -0,2 just mean EOF. myfile.truncate() - myfile.write(xpdata+encodeint(len(xpdata)) + b'STOP') + myfile.write(xpdata + encodeint(len(xpdata)) + b'STOP') myfile.flush() myfile.close() return 1 @@ -356,47 +360,47 @@ class tbz2(object): def scan(self): """Scans the tbz2 to locate the xpak segment and setup internal values. This function is called by relevant functions already.""" + a = None try: - mystat=os.stat(self.file) + mystat = os.stat(self.file) if self.filestat: - changed=0 + changed = 0 if mystat.st_size != self.filestat.st_size \ or mystat.st_mtime != self.filestat.st_mtime \ or mystat.st_ctime != self.filestat.st_ctime: changed = True if not changed: return 1 - self.filestat=mystat + self.filestat = mystat a = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'rb') - a.seek(-16,2) - trailer=a.read() - self.infosize=0 - self.xpaksize=0 + a.seek(-16, 2) + trailer = a.read() + self.infosize = 0 + self.xpaksize = 0 if trailer[-4:] != b'STOP': - a.close() return 0 if trailer[0:8] != b'XPAKSTOP': - a.close() return 0 - self.infosize=decodeint(trailer[8:12]) - self.xpaksize=self.infosize+8 - a.seek(-(self.xpaksize),2) - header=a.read(16) + self.infosize = decodeint(trailer[8:12]) + self.xpaksize = self.infosize + 8 + a.seek(-(self.xpaksize), 2) + header = a.read(16) if header[0:8] != b'XPAKPACK': - a.close() return 0 - self.indexsize=decodeint(header[8:12]) - self.datasize=decodeint(header[12:16]) - self.indexpos=a.tell() - self.index=a.read(self.indexsize) - self.datapos=a.tell() - a.close() + self.indexsize = decodeint(header[8:12]) + self.datasize = decodeint(header[12:16]) + self.indexpos = a.tell() + self.index = a.read(self.indexsize) + self.datapos = a.tell() return 2 - except SystemExit as e: + except SystemExit: raise except: return 0 + finally: + if a is not None: + a.close() def filelist(self): """Return an array of each file listed in the index.""" @@ -404,63 +408,60 @@ class tbz2(object): return None return getindex_mem(self.index) - def getfile(self,myfile,mydefault=None): + def getfile(self, myfile, mydefault=None): """Finds 'myfile' in the data segment and returns it.""" if not self.scan(): return None - myresult=searchindex(self.index,myfile) + myresult = searchindex(self.index, myfile) if not myresult: return mydefault a = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'rb') - a.seek(self.datapos+myresult[0],0) - myreturn=a.read(myresult[1]) + a.seek(self.datapos + myresult[0], 0) + myreturn = a.read(myresult[1]) a.close() return myreturn - def getelements(self,myfile): + def getelements(self, myfile): """A split/array representation of tbz2.getfile()""" - mydat=self.getfile(myfile) + mydat = self.getfile(myfile) if not mydat: return [] return mydat.split() - def unpackinfo(self,mydest): + def unpackinfo(self, mydest): """Unpacks all the files from the dataSegment into 'mydest'.""" if not self.scan(): return 0 - try: - origdir=os.getcwd() - except SystemExit as e: - raise - except: - os.chdir("/") - origdir="/" + mydest = normalize_path(mydest) + os.sep a = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'rb') if not os.path.exists(mydest): os.makedirs(mydest) - os.chdir(mydest) - startpos=0 - while ((startpos+8)<self.indexsize): - namelen=decodeint(self.index[startpos:startpos+4]) - datapos=decodeint(self.index[startpos+4+namelen:startpos+8+namelen]); - datalen=decodeint(self.index[startpos+8+namelen:startpos+12+namelen]); - myname=self.index[startpos+4:startpos+4+namelen] + startpos = 0 + while ((startpos + 8) < self.indexsize): + namelen = decodeint(self.index[startpos:startpos + 4]) + datapos = decodeint(self.index[startpos + 4 + namelen:startpos + 8 + namelen]) + datalen = decodeint(self.index[startpos + 8 + namelen:startpos + 12 + namelen]) + myname = self.index[startpos + 4:startpos + 4 + namelen] myname = _unicode_decode(myname, encoding=_encodings['repo.content'], errors='replace') - dirname=os.path.dirname(myname) + filename = os.path.join(mydest, myname.lstrip(os.sep)) + filename = normalize_path(filename) + if not filename.startswith(mydest): + # myname contains invalid ../ component(s) + continue + dirname = os.path.dirname(filename) if dirname: if not os.path.exists(dirname): os.makedirs(dirname) - mydat = open(_unicode_encode(myname, + mydat = open(_unicode_encode(filename, encoding=_encodings['fs'], errors='strict'), 'wb') - a.seek(self.datapos+datapos) + a.seek(self.datapos + datapos) mydat.write(a.read(datalen)) mydat.close() - startpos=startpos+namelen+12 + startpos = startpos + namelen + 12 a.close() - os.chdir(origdir) return 1 def get_data(self): @@ -470,28 +471,27 @@ class tbz2(object): a = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'rb') mydata = {} - startpos=0 - while ((startpos+8)<self.indexsize): - namelen=decodeint(self.index[startpos:startpos+4]) - datapos=decodeint(self.index[startpos+4+namelen:startpos+8+namelen]); - datalen=decodeint(self.index[startpos+8+namelen:startpos+12+namelen]); - myname=self.index[startpos+4:startpos+4+namelen] - a.seek(self.datapos+datapos) + startpos = 0 + while ((startpos + 8) < self.indexsize): + namelen = decodeint(self.index[startpos:startpos + 4]) + datapos = decodeint(self.index[startpos + 4 + namelen:startpos + 8 + namelen]) + datalen = decodeint(self.index[startpos + 8 + namelen:startpos + 12 + namelen]) + myname = self.index[startpos + 4:startpos + 4 + namelen] + a.seek(self.datapos + datapos) mydata[myname] = a.read(datalen) - startpos=startpos+namelen+12 + startpos = startpos + namelen + 12 a.close() return mydata def getboth(self): - """Returns an array [indexSegment,dataSegment]""" + """Returns an array [indexSegment, dataSegment]""" if not self.scan(): return None a = open(_unicode_encode(self.file, encoding=_encodings['fs'], errors='strict'), 'rb') a.seek(self.datapos) - mydata =a.read(self.datasize) + mydata = a.read(self.datasize) a.close() return self.index, mydata - diff --git a/portage_with_autodep/pym/portage/xpak.pyo b/portage_with_autodep/pym/portage/xpak.pyo Binary files differnew file mode 100644 index 0000000..361a709 --- /dev/null +++ b/portage_with_autodep/pym/portage/xpak.pyo diff --git a/portage_with_autodep/pym/repoman/__init__.pyo b/portage_with_autodep/pym/repoman/__init__.pyo Binary files differnew file mode 100644 index 0000000..b443f38 --- /dev/null +++ b/portage_with_autodep/pym/repoman/__init__.pyo diff --git a/portage_with_autodep/pym/repoman/checks.py b/portage_with_autodep/pym/repoman/checks.py index e7472df..77df603 100644 --- a/portage_with_autodep/pym/repoman/checks.py +++ b/portage_with_autodep/pym/repoman/checks.py @@ -1,5 +1,5 @@ # repoman: Checks -# Copyright 2007, 2011 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """This module contains functions used in Repoman to ascertain the quality @@ -8,6 +8,7 @@ and correctness of an ebuild.""" import re import time import repoman.errors as errors +import portage from portage.eapi import eapi_supports_prefix, eapi_has_implicit_rdepend, \ eapi_has_src_prepare_and_src_configure, eapi_has_dosed_dohard, \ eapi_exports_AA, eapi_exports_KV @@ -77,6 +78,7 @@ class EbuildHeader(LineCheck): # gentoo_license = re.compile(r'^# Distributed under the terms of the GNU General Public License v2$') gentoo_license = '# Distributed under the terms of the GNU General Public License v2' cvs_header = re.compile(r'^# \$Header: .*\$$') + ignore_comment = False def new(self, pkg): if pkg.mtime is None: @@ -282,21 +284,35 @@ class EbuildUselessCdS(LineCheck): self.check_next_line = True class EapiDefinition(LineCheck): - """ Check that EAPI is defined before inherits""" + """ + Check that EAPI assignment conforms to PMS section 7.3.1 + (first non-comment, non-blank line). + """ repoman_check_name = 'EAPI.definition' - - eapi_re = re.compile(r'^EAPI=') - inherit_re = re.compile(r'^\s*inherit\s') + ignore_comment = True + _eapi_re = portage._pms_eapi_re def new(self, pkg): - self.inherit_line = None + self._cached_eapi = pkg.metadata['EAPI'] + self._parsed_eapi = None + self._eapi_line_num = None def check(self, num, line): - if self.eapi_re.match(line) is not None: - if self.inherit_line is not None: - return errors.EAPI_DEFINED_AFTER_INHERIT - elif self.inherit_re.match(line) is not None: - self.inherit_line = line + if self._eapi_line_num is None and line.strip(): + self._eapi_line_num = num + 1 + m = self._eapi_re.match(line) + if m is not None: + self._parsed_eapi = m.group(2) + + def end(self): + if self._parsed_eapi is None: + if self._cached_eapi != "0": + yield "valid EAPI assignment must occur on or before line: %s" % \ + self._eapi_line_num + elif self._parsed_eapi != self._cached_eapi: + yield ("bash returned EAPI '%s' which does not match " + "assignment on line: %s") % \ + (self._cached_eapi, self._eapi_line_num) class EbuildPatches(LineCheck): """Ensure ebuilds use bash arrays for PATCHES to ensure white space safety""" @@ -341,7 +357,7 @@ class NoOffsetWithHelpers(LineCheck): repoman_check_name = 'variable.usedwithhelpers' # Ignore matches in quoted strings like this: # elog "installed into ${ROOT}usr/share/php5/apc/." - re = re.compile(r'^[^#"\']*\b(dodir|dohard|exeinto|insinto|into)\s+"?\$\{?(D|ROOT|ED|EROOT|EPREFIX)\b.*') + re = re.compile(r'^[^#"\']*\b(docinto|docompress|dodir|dohard|exeinto|fowners|fperms|insinto|into)\s+"?\$\{?(D|ROOT|ED|EROOT|EPREFIX)\b.*') error = errors.NO_OFFSET_WITH_HELPERS class ImplicitRuntimeDeps(LineCheck): @@ -384,6 +400,7 @@ class InheritDeprecated(LineCheck): # deprecated eclass : new eclass (False if no new eclass) deprecated_classes = { + "bash-completion": "bash-completion-r1", "gems": "ruby-fakegem", "git": "git-2", "mozconfig-2": "mozconfig-3", @@ -646,13 +663,17 @@ class Eapi4GoneVars(LineCheck): class PortageInternal(LineCheck): repoman_check_name = 'portage.internal' - re = re.compile(r'[^#]*\b(ecompress|ecompressdir|prepall|prepalldocs|preplib)\b') + ignore_comment = True + # Match when the command is preceded only by leading whitespace or a shell + # operator such as (, {, |, ||, or &&. This prevents false postives in + # things like elog messages, as reported in bug #413285. + re = re.compile(r'^(\s*|.*[|&{(]+\s*)\b(ecompress|ecompressdir|env-update|prepall|prepalldocs|preplib)\b') def check(self, num, line): """Run the check on line and return error if there is one""" m = self.re.match(line) if m is not None: - return ("'%s'" % m.group(1)) + " called on line: %d" + return ("'%s'" % m.group(2)) + " called on line: %d" _constant_checks = tuple((c() for c in ( EbuildHeader, EbuildWhitespace, EbuildBlankLine, EbuildQuote, diff --git a/portage_with_autodep/pym/repoman/checks.pyo b/portage_with_autodep/pym/repoman/checks.pyo Binary files differnew file mode 100644 index 0000000..1db2838 --- /dev/null +++ b/portage_with_autodep/pym/repoman/checks.pyo diff --git a/portage_with_autodep/pym/repoman/errors.pyo b/portage_with_autodep/pym/repoman/errors.pyo Binary files differnew file mode 100644 index 0000000..b7c2b8d --- /dev/null +++ b/portage_with_autodep/pym/repoman/errors.pyo diff --git a/portage_with_autodep/pym/repoman/herdbase.py b/portage_with_autodep/pym/repoman/herdbase.py index 64b59e6..91a32cb 100644 --- a/portage_with_autodep/pym/repoman/herdbase.py +++ b/portage_with_autodep/pym/repoman/herdbase.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- # repoman: Herd database analysis -# Copyright 2010-2011 Gentoo Foundation +# Copyright 2010-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 or later import errno import xml.etree.ElementTree try: from xml.parsers.expat import ExpatError -except ImportError: +except (ImportError, SystemError): # This means that python is built without xml support. # We tolerate global scope import failures for optional # modules, so that ImportModulesTestCase can succeed (or @@ -19,8 +19,6 @@ __all__ = [ "make_herd_base" ] -_SPECIAL_HERDS = set(('no-herd',)) - def _make_email(nick_name): if not nick_name.endswith('@gentoo.org'): nick_name = nick_name + '@gentoo.org' @@ -33,8 +31,6 @@ class HerdBase(object): self.all_emails = all_emails def known_herd(self, herd_name): - if herd_name in _SPECIAL_HERDS: - return True return herd_name in self.herd_to_emails def known_maintainer(self, nick_name): diff --git a/portage_with_autodep/pym/repoman/herdbase.pyo b/portage_with_autodep/pym/repoman/herdbase.pyo Binary files differnew file mode 100644 index 0000000..112152f --- /dev/null +++ b/portage_with_autodep/pym/repoman/herdbase.pyo diff --git a/portage_with_autodep/pym/repoman/utilities.py b/portage_with_autodep/pym/repoman/utilities.py index 232739e..bee67aa 100644 --- a/portage_with_autodep/pym/repoman/utilities.py +++ b/portage_with_autodep/pym/repoman/utilities.py @@ -1,10 +1,12 @@ # repoman: Utilities -# Copyright 2007-2011 Gentoo Foundation +# Copyright 2007-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 """This module contains utility functions to help repoman find ebuilds to scan""" +from __future__ import print_function + __all__ = [ "detect_vcs_conflicts", "editor_is_executable", @@ -14,17 +16,30 @@ __all__ = [ "format_qa_output", "get_commit_message_with_editor", "get_commit_message_with_stdin", + "get_committer_name", + "have_ebuild_dir", "have_profile_dir", "parse_metadata_use", "UnknownHerdsError", - "check_metadata" + "check_metadata", + "UpdateChangeLog" ] import errno import io +from itertools import chain import logging +import pwd +import re +import stat import sys +import time +import textwrap +import difflib +from tempfile import mkstemp + from portage import os +from portage import shutil from portage import subprocess_getstatusoutput from portage import _encodings from portage import _unicode_decode @@ -60,7 +75,7 @@ def detect_vcs_conflicts(options, vcs): if vcs == 'cvs': logging.info("Performing a " + output.green("cvs -n up") + \ " with a little magic grep to check for updates.") - retval = subprocess_getstatusoutput("cvs -n up 2>&1 | " + \ + retval = subprocess_getstatusoutput("cvs -n up 2>/dev/null | " + \ "egrep '^[^\?] .*' | " + \ "egrep -v '^. .*/digest-[^/]+|^cvs server: .* -- ignored$'") if vcs == 'svn': @@ -113,6 +128,33 @@ def have_profile_dir(path, maxdepth=3, filename="profiles.desc"): path = normalize_path(path + "/..") maxdepth -= 1 +def have_ebuild_dir(path, maxdepth=3): + """ + Try to figure out if 'path' or a subdirectory contains one or more + ebuild files named appropriately for their parent directory. + """ + stack = [(normalize_path(path), 1)] + while stack: + path, depth = stack.pop() + basename = os.path.basename(path) + try: + listdir = os.listdir(path) + except OSError: + continue + for filename in listdir: + abs_filename = os.path.join(path, filename) + try: + st = os.stat(abs_filename) + except OSError: + continue + if stat.S_ISDIR(st.st_mode): + if depth < maxdepth: + stack.append((abs_filename, depth + 1)) + elif stat.S_ISREG(st.st_mode): + if filename.endswith(".ebuild") and \ + filename.startswith(basename + "-"): + return os.path.dirname(os.path.dirname(path)) + def parse_metadata_use(xml_tree): """ Records are wrapped in XML as per GLEP 56 @@ -285,7 +327,7 @@ def editor_is_executable(editor): @param editor: An EDITOR value from the environment. @type: string @rtype: bool - @returns: True if an executable is found, False otherwise. + @return: True if an executable is found, False otherwise. """ editor_split = util.shlex_split(editor) if not editor_split: @@ -306,9 +348,8 @@ def get_commit_message_with_editor(editor, message=None): @param message: An iterable of lines to show in the editor. @type: iterable @rtype: string or None - @returns: A string on success or None if an error occurs. + @return: A string on success or None if an error occurs. """ - from tempfile import mkstemp fd, filename = mkstemp() try: os.write(fd, _unicode_encode(_( @@ -348,7 +389,7 @@ def get_commit_message_with_stdin(): Read a commit message from the user and return it. @rtype: string or None - @returns: A string on success or None if an error occurs. + @return: A string on success or None if an error occurs. """ print("Please enter a commit message. Use Ctrl-d to finish or Ctrl-c to abort.") commitmessage = [] @@ -435,6 +476,8 @@ def FindPortdir(settings): # file. if not portdir_overlay: portdir_overlay = have_profile_dir(location, filename="repo_name") + if not portdir_overlay: + portdir_overlay = have_ebuild_dir(location) if portdir_overlay: subdir = location[len(portdir_overlay):] if subdir and subdir[-1] != os.sep: @@ -472,7 +515,7 @@ def FindVCS(): outvcs = [] def seek(depth = None): - """ Seek for distributed VCSes. """ + """ Seek for VCSes that have a top-level data directory only. """ retvcs = [] pathprep = '' @@ -483,6 +526,8 @@ def FindVCS(): retvcs.append('bzr') if os.path.isdir(os.path.join(pathprep, '.hg')): retvcs.append('hg') + if os.path.isdir(os.path.join(pathprep, '.svn')): # >=1.7 + retvcs.append('svn') if retvcs: break @@ -497,7 +542,7 @@ def FindVCS(): # Level zero VCS-es. if os.path.isdir('CVS'): outvcs.append('cvs') - if os.path.isdir('.svn'): + if os.path.isdir('.svn'): # <1.7 outvcs.append('svn') # If we already found one of 'level zeros', just take a quick look @@ -508,4 +553,325 @@ def FindVCS(): else: outvcs = seek() + if len(outvcs) > 1: + # eliminate duplicates, like for svn in bug #391199 + outvcs = list(set(outvcs)) + return outvcs + +_copyright_re1 = re.compile(br'^(# Copyright \d\d\d\d)-\d\d\d\d ') +_copyright_re2 = re.compile(br'^(# Copyright )(\d\d\d\d) ') + + +class _copyright_repl(object): + __slots__ = ('year',) + def __init__(self, year): + self.year = year + def __call__(self, matchobj): + if matchobj.group(2) == self.year: + return matchobj.group(0) + else: + return matchobj.group(1) + matchobj.group(2) + \ + b'-' + self.year + b' ' + +def _update_copyright_year(year, line): + """ + These two regexes are taken from echangelog + update_copyright(), except that we don't hardcode + 1999 here (in order to be more generic). + """ + is_bytes = isinstance(line, bytes) + if is_bytes: + if not line.startswith(b'# Copyright '): + return line + else: + if not line.startswith('# Copyright '): + return line + + year = _unicode_encode(year) + line = _unicode_encode(line) + + line = _copyright_re1.sub(br'\1-' + year + b' ', line) + line = _copyright_re2.sub(_copyright_repl(year), line) + if not is_bytes: + line = _unicode_decode(line) + return line + +def update_copyright(fn_path, year, pretend=False): + """ + Check file for a Copyright statement, and update its year. The + patterns used for replacing copyrights are taken from echangelog. + Only the first lines of each file that start with a hash ('#') are + considered, until a line is found that doesn't start with a hash. + Files are read and written in binary mode, so that this function + will work correctly with files encoded in any character set, as + long as the copyright statements consist of plain ASCII. + """ + + try: + fn_hdl = io.open(_unicode_encode(fn_path, + encoding=_encodings['fs'], errors='strict'), + mode='rb') + except EnvironmentError: + return + + orig_header = [] + new_header = [] + + for line in fn_hdl: + line_strip = line.strip() + orig_header.append(line) + if not line_strip or line_strip[:1] != b'#': + new_header.append(line) + break + + line = _update_copyright_year(year, line) + new_header.append(line) + + difflines = 0 + for line in difflib.unified_diff( + [_unicode_decode(line) for line in orig_header], + [_unicode_decode(line) for line in new_header], + fromfile=fn_path, tofile=fn_path, n=0): + util.writemsg_stdout(line, noiselevel=-1) + difflines += 1 + util.writemsg_stdout("\n", noiselevel=-1) + + # unified diff has three lines to start with + if difflines > 3 and not pretend: + # write new file with changed header + f, fnnew_path = mkstemp() + f = io.open(f, mode='wb') + for line in new_header: + f.write(line) + for line in fn_hdl: + f.write(line) + f.close() + try: + fn_stat = os.stat(fn_path) + except OSError: + fn_stat = None + + shutil.move(fnnew_path, fn_path) + + if fn_stat is None: + util.apply_permissions(fn_path, mode=0o644) + else: + util.apply_stat_permissions(fn_path, fn_stat) + fn_hdl.close() + +def get_committer_name(env=None): + """Generate a committer string like echangelog does.""" + if env is None: + env = os.environ + if 'GENTOO_COMMITTER_NAME' in env and \ + 'GENTOO_COMMITTER_EMAIL' in env: + user = '%s <%s>' % (env['GENTOO_COMMITTER_NAME'], + env['GENTOO_COMMITTER_EMAIL']) + elif 'GENTOO_AUTHOR_NAME' in env and \ + 'GENTOO_AUTHOR_EMAIL' in env: + user = '%s <%s>' % (env['GENTOO_AUTHOR_NAME'], + env['GENTOO_AUTHOR_EMAIL']) + elif 'ECHANGELOG_USER' in env: + user = env['ECHANGELOG_USER'] + else: + pwd_struct = pwd.getpwuid(os.getuid()) + gecos = pwd_struct.pw_gecos.split(',')[0] # bug #80011 + user = '%s <%s@gentoo.org>' % (gecos, pwd_struct.pw_name) + return user + +def UpdateChangeLog(pkgdir, user, msg, skel_path, category, package, + new=(), removed=(), changed=(), pretend=False): + """ + Write an entry to an existing ChangeLog, or create a new one. + Updates copyright year on changed files, and updates the header of + ChangeLog with the contents of skel.ChangeLog. + """ + + if '<root@' in user: + err = 'Please set ECHANGELOG_USER or run as non-root' + logging.critical(err) + return None + + # ChangeLog times are in UTC + gmtime = time.gmtime() + year = time.strftime('%Y', gmtime) + date = time.strftime('%d %b %Y', gmtime) + + # check modified files and the ChangeLog for copyright updates + # patches and diffs (identified by .patch and .diff) are excluded + for fn in chain(new, changed): + if fn.endswith('.diff') or fn.endswith('.patch'): + continue + update_copyright(os.path.join(pkgdir, fn), year, pretend=pretend) + + cl_path = os.path.join(pkgdir, 'ChangeLog') + clold_lines = [] + clnew_lines = [] + old_header_lines = [] + header_lines = [] + + try: + clold_file = io.open(_unicode_encode(cl_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], errors='replace') + except EnvironmentError: + clold_file = None + + clskel_file = None + if clold_file is None: + # we will only need the ChangeLog skeleton if there is no + # ChangeLog yet + try: + clskel_file = io.open(_unicode_encode(skel_path, + encoding=_encodings['fs'], errors='strict'), + mode='r', encoding=_encodings['repo.content'], + errors='replace') + except EnvironmentError: + pass + + f, clnew_path = mkstemp() + + # construct correct header first + try: + if clold_file is not None: + # retain header from old ChangeLog + for line in clold_file: + line_strip = line.strip() + if line_strip and line[:1] != "#": + clold_lines.append(line) + break + old_header_lines.append(line) + header_lines.append(_update_copyright_year(year, line)) + if not line_strip: + break + + elif clskel_file is not None: + # read skel.ChangeLog up to first empty line + for line in clskel_file: + line_strip = line.strip() + if not line_strip: + break + line = line.replace('<CATEGORY>', category) + line = line.replace('<PACKAGE_NAME>', package) + line = _update_copyright_year(year, line) + header_lines.append(line) + header_lines.append(_unicode_decode('\n')) + clskel_file.close() + + # write new ChangeLog entry + clnew_lines.extend(header_lines) + newebuild = False + for fn in new: + if not fn.endswith('.ebuild'): + continue + ebuild = fn.split(os.sep)[-1][0:-7] + clnew_lines.append(_unicode_decode('*%s (%s)\n' % (ebuild, date))) + newebuild = True + if newebuild: + clnew_lines.append(_unicode_decode('\n')) + trivial_files = ('ChangeLog', 'Manifest') + display_new = ['+' + elem for elem in new + if elem not in trivial_files] + display_removed = ['-' + elem for elem in removed] + display_changed = [elem for elem in changed + if elem not in trivial_files] + if not (display_new or display_removed or display_changed): + # If there's nothing else to display, show one of the + # trivial files. + for fn in trivial_files: + if fn in new: + display_new = ['+' + fn] + break + elif fn in changed: + display_changed = [fn] + break + + display_new.sort() + display_removed.sort() + display_changed.sort() + + mesg = '%s; %s %s:' % (date, user, ', '.join(chain( + display_new, display_removed, display_changed))) + for line in textwrap.wrap(mesg, 80, \ + initial_indent=' ', subsequent_indent=' ', \ + break_on_hyphens=False): + clnew_lines.append(_unicode_decode('%s\n' % line)) + for line in textwrap.wrap(msg, 80, \ + initial_indent=' ', subsequent_indent=' '): + clnew_lines.append(_unicode_decode('%s\n' % line)) + clnew_lines.append(_unicode_decode('\n')) + + f = io.open(f, mode='w', encoding=_encodings['repo.content'], + errors='backslashreplace') + + for line in clnew_lines: + f.write(line) + + # append stuff from old ChangeLog + if clold_file is not None: + + if clold_lines: + # clold_lines may contain a saved non-header line + # that we want to write first. + # Also, append this line to clnew_lines so that the + # unified_diff call doesn't show it as removed. + for line in clold_lines: + f.write(line) + clnew_lines.append(line) + + else: + # ensure that there is no more than one blank + # line after our new entry + for line in clold_file: + if line.strip(): + f.write(line) + break + + # Now prepend old_header_lines to clold_lines, for use + # in the unified_diff call below. + clold_lines = old_header_lines + clold_lines + + for line in clold_file: + f.write(line) + clold_file.close() + f.close() + + # show diff (do we want to keep on doing this, or only when + # pretend?) + for line in difflib.unified_diff(clold_lines, clnew_lines, + fromfile=cl_path, tofile=cl_path, n=0): + util.writemsg_stdout(line, noiselevel=-1) + util.writemsg_stdout("\n", noiselevel=-1) + + if pretend: + # remove what we've done + os.remove(clnew_path) + else: + # rename to ChangeLog, and set permissions + try: + clold_stat = os.stat(cl_path) + except OSError: + clold_stat = None + + shutil.move(clnew_path, cl_path) + + if clold_stat is None: + util.apply_permissions(cl_path, mode=0o644) + else: + util.apply_stat_permissions(cl_path, clold_stat) + + if clold_file is None: + return True + else: + return False + except IOError as e: + err = 'Repoman is unable to create/write to Changelog.new file: %s' % (e,) + logging.critical(err) + # try to remove if possible + try: + os.remove(clnew_path) + except OSError: + pass + return None + diff --git a/portage_with_autodep/pym/repoman/utilities.pyo b/portage_with_autodep/pym/repoman/utilities.pyo Binary files differnew file mode 100644 index 0000000..c9f6734 --- /dev/null +++ b/portage_with_autodep/pym/repoman/utilities.pyo |