diff options
author | fuzzyray <fuzzyray@gentoo.org> | 2010-03-09 16:42:04 +0000 |
---|---|---|
committer | fuzzyray <fuzzyray@gentoo.org> | 2010-03-09 16:42:04 +0000 |
commit | 2f90a4b9ceff920f793541376da21d313af083d9 (patch) | |
tree | 4eda986a753ea80a16a3f416e22daae7946a6dbd | |
parent | glsa-check: hide non-vuln glsas in quiet mode (diff) | |
download | gentoolkit-2f90a4b9ceff920f793541376da21d313af083d9.tar.gz gentoolkit-2f90a4b9ceff920f793541376da21d313af083d9.tar.bz2 gentoolkit-2f90a4b9ceff920f793541376da21d313af083d9.zip |
sync with genscripts rev 343. This adds the initial py3k support and the analyse utility to gentoolkit
svn path=/trunk/gentoolkit/; revision=751
59 files changed, 3732 insertions, 1061 deletions
@@ -1,5 +1,10 @@ +2010-03-09: Paul Varner <fuzzyray@gentoo.org> + * gentoolkit: Add inital py3k support. + * analyse: Add new analyse utility from dol-sen. This will probably + change to a different name for final gentoolkit-0.3.0 release. + 2010-02-05: Paul Varner <fuzzyray@gentoo.org> - *revdep-rebuild: Update revdep-rebuild to use extended regular + * revdep-rebuild: Update revdep-rebuild to use extended regular expressions instead of basic regular expressions. (Bug 143498) 2010-02-04: Paul Varner <fuzzyray@gentoo.org> @@ -25,7 +25,7 @@ svn copy svn+ssh://<dev>@svn.gentoo.org/var/svnroot/gentoolkit/trunk/gentoolkit svn update to pull the tag from subversion cd to the local tags/gentoolkit-0.3.0 directory -- Create a source distribution (you need to add VERSION here, too): +- Create a source distribution (you need to add VERSION here): VERSION="0.3.0" ./setup.py sdist Transfer dist/gentoolkit-0.3.0.tar.gz to dev.gentoo.org:/space/distfiles-local @@ -20,10 +20,22 @@ - use ~/.gentoo/gentoolkit/ebump.conf - use /etc/gentoolkit/ebump.conf -equery: +equery (modern): Add more --debug stuff Write tests for Dependencies._parser Profile Dependencies._parser + Tighten up CPV.split_cpv, it's slow and bad + Extend PackageFormatter usage to everything that outputs packages to + allow for purvasive use of -F, --format goodness + Add package::repo search syntax to do_lookup + _do_repository_lookup? + Move do_lookout and all it's silly friends into the new query module + and Query class. Essentially, Query, when applied to a pkgspec input + should contain most of the common 'helper' methods. So we should be + be able to do: + Query('portage').find_best(), + Query('portage').find_package(), + Query('portag*').is_regex() or .uses_globbing(), etc. Refactor each module to be useful for import. Done modules: +depends +belongs diff --git a/bin/analyse b/bin/analyse new file mode 100755 index 0000000..a90410b --- /dev/null +++ b/bin/analyse @@ -0,0 +1,48 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com> +# Copyright 2002-2010 Gentoo Technologies, Inc. +# Distributed under the terms of the GNU General Public License v2 or later +# +# $Header$ + +"""'analyse' is a flexible utility for Gentoo linux which can display various +information about installed packages, such as the USE flags used and the +packages that use them. It can also be used to help rebuild /etc/portage/package.* +files in the event of corruption, and possibly more. +""" + +from __future__ import print_function + +import sys +# This block ensures that ^C interrupts are handled quietly. +try: + import signal + + def exithandler(signum,frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + print() + sys.exit(1) + + signal.signal(signal.SIGINT, exithandler) + signal.signal(signal.SIGTERM, exithandler) + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +except KeyboardInterrupt: + print() + sys.exit(1) + +from gentoolkit import analyse, errors + +try: + analyse.main() +except errors.GentoolkitException as err: + if '--debug' in sys.argv: + raise + else: + from gentoolkit import pprinter as pp + sys.stderr.write(pp.error(str(err))) + print() + print("Add '--debug' to global options for traceback.") + sys.exit(1) @@ -1,9 +1,10 @@ #!/usr/bin/python -# Copyright 2003-2010 Gentoo Foundation +# Copyright 2003-2005 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # $Header: $ -from __future__ import with_statement +from __future__ import print_function + ############################################################################### # Meta: @@ -18,15 +19,17 @@ __description__ = "A cleaning tool for Gentoo distfiles and binaries." # Python imports: import sys -import os, stat +import stat import re import time import getopt -import fpformat import signal import portage from portage.output import * +from portage import os + +from gentoolkit.helpers import walk listdir = portage.listdir @@ -40,12 +43,12 @@ pkgdir = port_settings["PKGDIR"] ############################################################################### # printVersion: def printVersion(): - print "%s (%s) - %s" \ - % (__productname__, __version__, __description__) - print - print "Author: %s <%s>" % (__author__,__email__) - print "Copyright 2003-2010 Gentoo Foundation" - print "Distributed under the terms of the GNU General Public License v2" + print("%s (%s) - %s" \ + % (__productname__, __version__, __description__)) + print() + print("Author: %s <%s>" % (__author__,__email__)) + print("Copyright 2003-2009 Gentoo Foundation") + print("Distributed under the terms of the GNU General Public License v2") ############################################################################### @@ -62,112 +65,112 @@ def printUsage(error=None,help=None): if not error and not help: help = 'all' if error == 'time': eerror("Wrong time specification") - print >>out, "Time specification should be an integer followed by a"+ \ - " single letter unit." - print >>out, "Available units are: y (years), m (months), w (weeks), "+ \ - "d (days) and h (hours)." - print >>out, "For instance: \"1y\" is \"one year\", \"2w\" is \"two"+ \ - " weeks\", etc. " + print("Time specification should be an integer followed by a"+ \ + " single letter unit.", file=out) + print("Available units are: y (years), m (months), w (weeks), "+ \ + "d (days) and h (hours).", file=out) + print("For instance: \"1y\" is \"one year\", \"2w\" is \"two"+ \ + " weeks\", etc. ", file=out) return if error == 'size': eerror("Wrong size specification") - print >>out, "Size specification should be an integer followed by a"+ \ - " single letter unit." - print >>out, "Available units are: G, M, K and B." - print >>out, "For instance: \"10M\" is \"ten megabytes\", \"200K\" "+ \ - "is \"two hundreds kilobytes\", etc." + print("Size specification should be an integer followed by a"+ \ + " single letter unit.", file=out) + print("Available units are: G, M, K and B.", file=out) + print("For instance: \"10M\" is \"ten megabytes\", \"200K\" "+ \ + "is \"two hundreds kilobytes\", etc.", file=out) return if error in ('global-options', 'packages-options', 'distfiles-options', \ 'merged-packages-options', 'merged-distfiles-options',): eerror("Wrong option on command line.") - print >>out + print(file=out) elif error == 'actions': eerror("Wrong or missing action name on command line.") - print >>out - print >>out, white("Usage:") + print(file=out) + print(white("Usage:"), file=out) if error in ('actions','global-options', 'packages-options', \ 'distfiles-options') or help == 'all': - print >>out, " "+turquoise(__productname__), \ + print(" "+turquoise(__productname__), \ yellow("[global-option] ..."), \ green("<action>"), \ - yellow("[action-option] ...") + yellow("[action-option] ..."), file=out) if error == 'merged-distfiles-options' or help in ('all','distfiles'): - print >>out, " "+turquoise(__productname__+'-dist'), \ - yellow("[global-option, distfiles-option] ...") + print(" "+turquoise(__productname__+'-dist'), \ + yellow("[global-option, distfiles-option] ..."), file=out) if error == 'merged-packages-options' or help in ('all','packages'): - print >>out, " "+turquoise(__productname__+'-pkg'), \ - yellow("[global-option, packages-option] ...") + print(" "+turquoise(__productname__+'-pkg'), \ + yellow("[global-option, packages-option] ..."), file=out) if error in ('global-options', 'actions'): - print >>out, " "+turquoise(__productname__), \ - yellow("[--help, --version]") + print(" "+turquoise(__productname__), \ + yellow("[--help, --version]"), file=out) if help == 'all': - print >>out, " "+turquoise(__productname__+"(-dist,-pkg)"), \ - yellow("[--help, --version]") + print(" "+turquoise(__productname__+"(-dist,-pkg)"), \ + yellow("[--help, --version]"), file=out) if error == 'merged-packages-options' or help == 'packages': - print >>out, " "+turquoise(__productname__+'-pkg'), \ - yellow("[--help, --version]") + print(" "+turquoise(__productname__+'-pkg'), \ + yellow("[--help, --version]"), file=out) if error == 'merged-distfiles-options' or help == 'distfiles': - print >>out, " "+turquoise(__productname__+'-dist'), \ - yellow("[--help, --version]") - print >>out + print(" "+turquoise(__productname__+'-dist'), \ + yellow("[--help, --version]"), file=out) + print(file=out) if error in ('global-options', 'merged-packages-options', \ 'merged-distfiles-options') or help: - print >>out, "Available global", yellow("options")+":" - print >>out, yellow(" -C, --nocolor")+ \ - " - turn off colors on output" - print >>out, yellow(" -d, --destructive")+ \ - " - only keep the minimum for a reinstallation" - print >>out, yellow(" -e, --exclude-file=<path>")+ \ - " - path to the exclusion file" - print >>out, yellow(" -i, --interactive")+ \ - " - ask confirmation before deletions" - print >>out, yellow(" -n, --package-names")+ \ - " - protect all versions (when --destructive)" - print >>out, yellow(" -p, --pretend")+ \ - " - only display what would be cleaned" - print >>out, yellow(" -q, --quiet")+ \ - " - be as quiet as possible" - print >>out, yellow(" -t, --time-limit=<time>")+ \ - " - don't delete files modified since "+yellow("<time>") - print >>out, " "+yellow("<time>"), "is a duration: \"1y\" is"+ \ - " \"one year\", \"2w\" is \"two weeks\", etc. " - print >>out, " "+"Units are: y (years), m (months), w (weeks), "+ \ - "d (days) and h (hours)." - print >>out, yellow(" -h, --help")+ \ - " - display the help screen" - print >>out, yellow(" -V, --version")+ \ - " - display version info" - print >>out + print("Available global", yellow("options")+":", file=out) + print(yellow(" -C, --nocolor")+ \ + " - turn off colors on output", file=out) + print(yellow(" -d, --destructive")+ \ + " - only keep the minimum for a reinstallation", file=out) + print(yellow(" -e, --exclude-file=<path>")+ \ + " - path to the exclusion file", file=out) + print(yellow(" -i, --interactive")+ \ + " - ask confirmation before deletions", file=out) + print(yellow(" -n, --package-names")+ \ + " - protect all versions (when --destructive)", file=out) + print(yellow(" -p, --pretend")+ \ + " - only display what would be cleaned", file=out) + print(yellow(" -q, --quiet")+ \ + " - be as quiet as possible", file=out) + print(yellow(" -t, --time-limit=<time>")+ \ + " - don't delete files modified since "+yellow("<time>"), file=out) + print(" "+yellow("<time>"), "is a duration: \"1y\" is"+ \ + " \"one year\", \"2w\" is \"two weeks\", etc. ", file=out) + print(" "+"Units are: y (years), m (months), w (weeks), "+ \ + "d (days) and h (hours).", file=out) + print(yellow(" -h, --help")+ \ + " - display the help screen", file=out) + print(yellow(" -V, --version")+ \ + " - display version info", file=out) + print(file=out) if error == 'actions' or help == 'all': - print >>out, "Available", green("actions")+":" - print >>out, green(" packages")+ \ - " - clean outdated binary packages from:" - print >>out, " ",teal(pkgdir) - print >>out, green(" distfiles")+ \ - " - clean outdated packages sources files from:" - print >>out, " ",teal(distdir) - print >>out + print("Available", green("actions")+":", file=out) + print(green(" packages")+ \ + " - clean outdated binary packages from:", file=out) + print(" ",teal(pkgdir), file=out) + print(green(" distfiles")+ \ + " - clean outdated packages sources files from:", file=out) + print(" ",teal(distdir), file=out) + print(file=out) if error in ('packages-options','merged-packages-options') \ or help in ('all','packages'): - print >>out, "Available", yellow("options"),"for the", \ - green("packages"),"action:" - print >>out, yellow(" NONE :)") - print >>out + print("Available", yellow("options"),"for the", \ + green("packages"),"action:", file=out) + print(yellow(" NONE :)"), file=out) + print(file=out) if error in ('distfiles-options', 'merged-distfiles-options') \ or help in ('all','distfiles'): - print >>out, "Available", yellow("options"),"for the", \ - green("distfiles"),"action:" - print >>out, yellow(" -f, --fetch-restricted")+ \ - " - protect fetch-restricted files (when --destructive)" - print >>out, yellow(" -s, --size-limit=<size>")+ \ - " - don't delete distfiles bigger than "+yellow("<size>") - print >>out, " "+yellow("<size>"), "is a size specification: "+ \ - "\"10M\" is \"ten megabytes\", \"200K\" is" - print >>out, " "+"\"two hundreds kilobytes\", etc. Units are: "+ \ - "G, M, K and B." - print >>out - print >>out, "More detailed instruction can be found in", \ - turquoise("`man %s`" % __productname__) + print("Available", yellow("options"),"for the", \ + green("distfiles"),"action:", file=out) + print(yellow(" -f, --fetch-restricted")+ \ + " - protect fetch-restricted files (when --destructive)", file=out) + print(yellow(" -s, --size-limit=<size>")+ \ + " - don't delete distfiles bigger than "+yellow("<size>"), file=out) + print(" "+yellow("<size>"), "is a size specification: "+ \ + "\"10M\" is \"ten megabytes\", \"200K\" is", file=out) + print(" "+"\"two hundreds kilobytes\", etc. Units are: "+ \ + "G, M, K and B.", file=out) + print(file=out) + print("More detailed instruction can be found in", \ + turquoise("`man %s`" % __productname__), file=out) ############################################################################### @@ -175,7 +178,7 @@ def printUsage(error=None,help=None): def einfo(message="", nocolor=False): if not nocolor: prefix = " "+green('*') else: prefix = ">>>" - print prefix,message + print(prefix,message) ############################################################################### @@ -183,7 +186,7 @@ def einfo(message="", nocolor=False): def eerror(message="", nocolor=False): if not nocolor: prefix = " "+red('*') else: prefix = "!!!" - print >>sys.stderr,prefix,message + print(prefix,message, file=sys.stderr) ############################################################################### @@ -200,12 +203,12 @@ def eprompt(message, nocolor=False): # result. Output is a string. def prettySize(size,justify=False): units = [" G"," M"," K"," B"] - approx = 0 + fmt = "{0:.0f}" while len(units) and size >= 1000: - approx = 1 + fmt = "{0:.1f}" size = size / 1024. units.pop() - sizestr = fpformat.fix(size,approx)+units[-1] + sizestr = fmt.format(size)+units[-1] if justify: sizestr = " " + blue("[ ") + " "*(7-len(sizestr)) \ + green(sizestr) + blue(" ]") @@ -221,7 +224,7 @@ def yesNoAllPrompt(myoptions,message="Do you want to proceed?"): user_string="xxx" while not user_string.lower() in ["","y","n","a","yes","no","all"]: eprompt(message+" [Y/n/a]: ", myoptions['nocolor']) - user_string = raw_input() + user_string = sys.stdin.readline() if user_string.lower() in ["a","all"]: myoptions['accept_all'] = True myanswer = user_string.lower() in ["","y","a","yes","all"] @@ -610,7 +613,7 @@ def findPackages( \ eerror("Please set PKGDIR to a sane value.", myoptions['nocolor']) eerror("(Check your /etc/make.conf and environment).", myoptions['nocolor']) exit(1) - for root, dirs, files in os.walk(pkgdir): + for root, dirs, files in walk(pkgdir): if root[-3:] == 'All': continue for file in files: if not file[-5:] == ".tbz2": @@ -661,7 +664,7 @@ def doCleanup(clean_dict,action,myoptions): if action == 'distfiles': file_type = 'file' else: file_type = 'binary package' # sorting helps reading - clean_keys = clean_dict.keys() + clean_keys = list(clean_dict.keys()) clean_keys.sort() clean_size = 0 # clean all entries one by one @@ -676,10 +679,10 @@ def doCleanup(clean_dict,action,myoptions): myoptions['nocolor']) if not myoptions['quiet']: # pretty print mode - print prettySize(key_size,True),teal(mykey) + print(prettySize(key_size,True),teal(mykey)) elif myoptions['pretend'] or myoptions['interactive']: # file list mode - for file in clean_dict[mykey]: print file + for file in clean_dict[mykey]: print(file) #else: actually delete stuff, but don't print anything if myoptions['pretend']: clean_size += key_size elif not myoptions['interactive'] \ @@ -750,7 +753,7 @@ def doAction(action,myoptions,exclude_dict={}): time_limit=myoptions['time-limit'], \ size_limit=myoptions['size-limit']) # actually clean files if something was found - if len(clean_dict.keys()): + if clean_dict: # verbose pretend message if myoptions['pretend'] and not myoptions['quiet']: einfo("Here are "+files_type+" that would be deleted:", \ @@ -786,7 +789,7 @@ def main(): # parse command line options and actions try: myaction = parseArgs(myoptions) # filter exception to know what message to display - except ParseArgsException, e: + except ParseArgsException as e: if e.value == 'help': printUsage(help='all') sys.exit(0) @@ -806,7 +809,7 @@ def main(): myoptions['exclude-file'] = my_exclude_file if 'exclude-file' in myoptions: try: exclude_dict = parseExcludeFile(myoptions['exclude-file']) - except ParseExcludeFileException, e: + except ParseExcludeFileException as e: eerror(e, myoptions['nocolor']) eerror("Invalid exclusion file: %s" % myoptions['exclude-file'], \ myoptions['nocolor']) @@ -828,7 +831,7 @@ def main(): if __name__ == "__main__": try: main() except KeyboardInterrupt: - print "Aborted." + print("Aborted.") sys.exit(130) sys.exit(0) diff --git a/bin/epkginfo b/bin/epkginfo index c1f457c..30f2ab5 100755 --- a/bin/epkginfo +++ b/bin/epkginfo @@ -1,12 +1,14 @@ #!/usr/bin/python # -# Copyright 2009-2010 Gentoo Technologies, Inc. +# Copyright 2009 Gentoo Technologies, Inc. # Distributed under the terms of the GNU General Public License v2 or later # # $Header$ """Shortcut to equery meta""" +from __future__ import print_function + __authors__ = ( 'Douglas Anderson <douglasjanderson@gmail.com>: equery meta', 'Ned Ludd <solar@gentoo.org>: first full implimentation' @@ -21,8 +23,8 @@ from gentoolkit.equery import mod_usage from gentoolkit.equery.meta import main, print_help def print_epkginfo_help(): - print mod_usage(mod_name="epkginfo") - print + print(mod_usage(mod_name="epkginfo")) + print() print_help(with_usage=False) equery.initialize_configuration() @@ -32,7 +34,7 @@ if not args or set(('-h', '--help')).intersection(args): else: try: main(args) - except errors.GentoolkitException, err: + except errors.GentoolkitException as err: from gentoolkit import pprinter as pp pp.die(1, str(err)) @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2002-2010 Gentoo Technologies, Inc. +# Copyright 2002-2009 Gentoo Technologies, Inc. # Distributed under the terms of the GNU General Public License v2 or later # # $Header$ @@ -10,6 +10,8 @@ information about packages, such as the files they own, their USE flags, the MD5 sum of each file owned by a given package, and many other things. """ +from __future__ import print_function + import sys # This block ensures that ^C interrupts are handled quietly. try: @@ -18,7 +20,7 @@ try: def exithandler(signum,frame): signal.signal(signal.SIGINT, signal.SIG_IGN) signal.signal(signal.SIGTERM, signal.SIG_IGN) - print + print() sys.exit(1) signal.signal(signal.SIGINT, exithandler) @@ -26,19 +28,19 @@ try: signal.signal(signal.SIGPIPE, signal.SIG_DFL) except KeyboardInterrupt: - print + print() sys.exit(1) from gentoolkit import equery, errors try: equery.main() -except errors.GentoolkitException, err: +except errors.GentoolkitException as err: if '--debug' in sys.argv: raise else: from gentoolkit import pprinter as pp sys.stderr.write(pp.error(str(err))) - print - print "Add '--debug' to global options for traceback." + print() + print("Add '--debug' to global options for traceback.") sys.exit(1) @@ -116,7 +116,7 @@ cat << VER ${PROGRAM_NAME} (${VERSION}) Written by Marius Mauch -Copyright (C) 2004-2010 Gentoo Foundation, Inc. +Copyright (C) 2004-2009 Gentoo Foundation, Inc. This is free software; see the source for copying conditions. VER } diff --git a/bin/glsa-check b/bin/glsa-check index 78fa86b..705e7b4 100755 --- a/bin/glsa-check +++ b/bin/glsa-check @@ -3,12 +3,13 @@ # $Header: $ # This program is licensed under the GPL, version 2 -import os import sys import codecs +from functools import reduce import portage from portage.output import * +from portage import os from getopt import getopt, GetoptError @@ -93,7 +94,7 @@ try: if args in [o for o in m[:-1]]: mode = m[1][2:] -except GetoptError, e: +except GetoptError as e: sys.stderr.write("unknown option given: ") sys.stderr.write(str(e)+"\n") mode = "HELP" @@ -178,7 +179,7 @@ if "affected" in params: for x in todolist: try: myglsa = Glsa(x, glsaconfig) - except (GlsaTypeException, GlsaFormatException), e: + except (GlsaTypeException, GlsaFormatException) as e: if verbose: sys.stderr.write(("invalid GLSA: %s (error message was: %s)\n" % (x, e))) continue @@ -195,6 +196,13 @@ for p in params[:]: glsalist.extend([g for g in params if g not in glsalist]) def summarylist(myglsalist, fd1=sys.stdout, fd2=sys.stderr, encoding="utf-8"): + # Get to the raw streams in py3k before wrapping them with an encoded writer + # to avoid writing bytes to a text stream (stdout/stderr are text streams + # by default in py3k) + if hasattr(fd1, "buffer"): + fd1 = fd1.buffer + if hasattr(fd2, "buffer"): + fd2 = fd2.buffer fd1 = codecs.getwriter(encoding)(fd1) fd2 = codecs.getwriter(encoding)(fd2) if not quiet: @@ -206,7 +214,7 @@ def summarylist(myglsalist, fd1=sys.stdout, fd2=sys.stderr, encoding="utf-8"): for myid in myglsalist: try: myglsa = Glsa(myid, glsaconfig) - except (GlsaTypeException, GlsaFormatException), e: + except (GlsaTypeException, GlsaFormatException) as e: if verbose: fd2.write(("invalid GLSA: %s (error message was: %s)\n" % (myid, e))) continue @@ -227,7 +235,7 @@ def summarylist(myglsalist, fd1=sys.stdout, fd2=sys.stderr, encoding="utf-8"): fd1.write(color(myglsa.nr) + " " + color(status) + " " + color(access) + myglsa.title + " (") if not verbose: - for pkg in myglsa.packages.keys()[:3]: + for pkg in list(myglsa.packages.keys())[:3]: fd1.write(" " + pkg + " ") if len(myglsa.packages) > 3: fd1.write("... ") @@ -252,7 +260,7 @@ if mode in ["dump", "fix", "inject", "pretend"]: for myid in glsalist: try: myglsa = Glsa(myid, glsaconfig) - except (GlsaTypeException, GlsaFormatException), e: + except (GlsaTypeException, GlsaFormatException) as e: if verbose: sys.stderr.write(("invalid GLSA: %s (error message was: %s)\n" % (myid, e))) continue @@ -309,7 +317,7 @@ if mode in ["dump", "fix", "inject", "pretend"]: # see if anything is left that can be upgraded if mergedict: sys.stdout.write(">>> Updates that will be performed:\n") - for (upd, vuln) in mergedict.iteritems(): + for (upd, vuln) in mergedict.items(): sys.stdout.write(" " + green(upd) + " (vulnerable: " + red(", ".join(vuln)) + ")\n") if no_upgrades: @@ -326,7 +334,7 @@ if mode == "test": for myid in glsalist: try: myglsa = Glsa(myid, glsaconfig) - except (GlsaTypeException, GlsaFormatException), e: + except (GlsaTypeException, GlsaFormatException) as e: if verbose: sys.stderr.write(("invalid GLSA: %s (error message was: %s)\n" % (myid, e))) continue @@ -350,7 +358,7 @@ if mode == "mail": import portage_mail import socket - from StringIO import StringIO + from io import StringIO try: from email.mime.text import MIMEText except ImportError: @@ -383,7 +391,7 @@ if mode == "mail": for myid in glsalist: try: myglsa = Glsa(myid, glsaconfig) - except (GlsaTypeException, GlsaFormatException), e: + except (GlsaTypeException, GlsaFormatException) as e: if verbose: sys.stderr.write(("invalid GLSA: %s (error message was: %s)\n" % (myid, e))) continue @@ -392,7 +400,7 @@ if mode == "mail": myattachments.append(MIMEText(str(myfd.getvalue()), _charset="utf8")) myfd.close() - if glsalist or not quiet: + if glsalist or not quiet: mymessage = portage_mail.create_message(myfrom, myrecipient, mysubject, summary, myattachments) portage_mail.send_mail(glsaconfig, mymessage) diff --git a/bin/revdep-rebuild b/bin/revdep-rebuild index fe288e1..0dd0e87 100755 --- a/bin/revdep-rebuild +++ b/bin/revdep-rebuild @@ -548,7 +548,7 @@ verify_tmpdir() { get_search_env() { local new_env local old_env - local uid=$(python -c 'import os; import pwd; print pwd.getpwuid(os.getuid())[0]') + local uid=$(python -c 'import os; import pwd; print(pwd.getpwuid(os.getuid())[0])') # Find a place to put temporary files if [[ "$uid" == "root" ]]; then local tmp_target="/var/cache/${APP_NAME}" diff --git a/man/analyse.1 b/man/analyse.1 new file mode 100644 index 0000000..e6d21c7 --- /dev/null +++ b/man/analyse.1 @@ -0,0 +1,202 @@ +.TH "ANALYSE" "22" "Febuary 2010" "GENTOOLKIT" +.SH "NAME" +analyse \- Gentoo Installed Package Analysis Tool + +.SH "SYNOPSIS" +.BI "analyse " "[global-options] " "module " "[local-options]" "TARGET" + +.SH "DESCRIPTION" +.B Analyse +Is a collection of modules for analysing the state of installed Gentoo packages for +USE flags or keywords used for installation, and their current masking status. +.br +It can also optionally (re)generate new /etc/portage/package.* files. + +.SH "GLOBAL OPTIONS" +.HP +.B \-h, \-\-help +.br +Output a help message. +.HP +.B \-q, \-\-quiet +.br +Be less verbose where possible. In some modules, this option can increase the output speed. +.HP +.B \-C, \-\-no-color +.br +Do not colorize output. +.HP +.B \-N, \-\-no\-pipe +.br +Turn off automatic pipe detection. Use this option if you do not want +.B analyse +To detect if the output is being directed to the screen or to another program +and adjust color and verbosity accordingly. +.HP +.B \-V, \-\-version +.br +Display \fBGentoolkit\fP's version. Please include this in all bug reports. (see +.B BUGS +below) + +.SH "MODULES" +.B Analyse +Uses a system of modules. Each module has both a long and short name. +The list below uses the notation "\fBmodule (m)\fP", where \fIm\fP is the short name +and \fImodule\fP is the long name. +.P +You can view the +.B help +message for a specific module by using +.BR "-h" ", " "--help " +as either a global option (after +.B analyse +and before the module name) or as a local option (after the module name). + +.SS +.BI "analyse (a) [OPTIONS] TARGET" +Report on all installed packages for \fITARGET\fP. +.P + +.IR "TARGET" ":" +.HP +.B use +.br +Will analyse the installed with USE flags for output results. +.HP +.B pkguse +.br +Will analyse the USE flags information from the installed pkg's 'PKGUSE' file which contains +only flags settings from /etc/portage/package.use at the time of installation. +.HP +.B keywords +.br +Will analyse the recorded keywords for output results. +.HP +.B unmask +.br +Will analyse the installed packages and portage trees for pkgs that require unmasking and report them. +.br +.P +.IR "LOCAL OPTIONS" ":" +.HP +.B \-u, \-\-unset +.br +Will also include any USE flags used that were not enabled for some packages. +.HP +.B \-v, \-\-verebose +.br +Gives more detail about the results found and the current task being performed. + +.P +.IR "EXAMPLES" ":" +.EX +.HP +analyse a --verbose --unset use +.EE +.br +Report on all use flags used to install the packages. (--unset) Include in the report all flags +that have been used but also were not set enabled for some packages. +(--verbose) Also list the packages that used the USE flag setting. +The report will break down the useage and report the USE flag up to 3 times indicating its +setting {"+","-"," "= unset} prepended to the flag name. +It will also color the output, red = Disabled, blue = Enabled, plain text = unset +.br + +.SS +.BI "rebuild (r) [OPTIONS] TARGET" +Create a list all packages for \fITARGET\fP settings that are needed for +other than the default settings. + +.IR "TARGET" ":" +.HP +.B use +.br +Will analyse the USE flags for output results. +.HP +.B keywords +.br +Will analyse the keywords for output results. +.HP +.B unmask +.br +Will analyse the installed packages and portage trees for pkgs that require +unmasking and produce output/a new /etc/portage/package.unmask file. +.P +.IR "LOCAL OPTIONS" ":" +.HP +.B \-a, \-\-all +.br +Create files/output for all TARGET(s) found to need it. (not Implemented yet) +.HP +.B \-e, \-\-excact +.br +Will prepend the pkg with = as well as use the version information for the entries. +.br +eg.: =CAT/PKG-VER flag1 flag2 +.HP +.B \-p, \-\-pretend +.br +Sends the output to the screen instead of a file. +.HP +.B \-v, \-\-verebose +.br +Gives more detail about the results found and the current task being performed. +.P +.IR "EXAMPLES" ":" +.EX +.HP +analyse rebuild -p use +.EE +.br +Analyse the installed packages database and current system USE flag settings + and output the results in the form of: +.br + +.br +.EX +CAT/PKG -flag1 -flag2 flag3 flag4... + +.SS +.BI "clean (c) [OPTIONS] TARGET" +Clean all packages for \fITARGET\fP settings that are found with obsolete settings +for the current settings and pkg ebuild. (not Implemented yet) + +.IR "TARGET" ":" +.HP +.B use +.br +Will analyse the USE flags and /etc/portage/package.use file(s) for entries that +are redundant or no longer used by the pkg. +.HP +.B keywords +.br +Will analyse the keywords and /etc/portage/package.keywords file(s) for entries +that are no longer needed. +.HP +.B unmask +.br +Will analyse the installed packages, /etc/portage/package.unmask file(s) and +portage trees for pkgs that no longer require unmasking. +.P +.IR "LOCAL OPTIONS" ":" +.HP +.B \-a, \-\-all +.br +Clean files/output for all TARGET(s) found to need it. (not Implemented yet) +.HP +.B \-p, \-\-pretend +.br +Sends the output to the screen instead of a file. +.HP +.B \-v, \-\-verebose +.br +Gives more detail about the results found and the current task being performed. + + +.SH "BUGS" +Submit bug reports to http://bugs.gentoo.org. + +.SH "AUTHORS" +.br +Brian Dolbec <brian.dolbec@gmail.com>, 2010 diff --git a/man/equery.1 b/man/equery.1 index bc80c2d..f705d13 100644 --- a/man/equery.1 +++ b/man/equery.1 @@ -429,7 +429,7 @@ In this output from \fBequery meta boost\fP, -r5 is the highest available versio .HP .B \-m, \-\-maintainer .br -Show the package maintainer(s) email address. If the metadata is available, also show the maintainer's name and/or job description. (shown by default) +Show the package maintainer(s) email address. If the metadata is available, also show the maitainer's name and/or job description. (shown by default) .HP .B \-u, \-\-useflags .br diff --git a/pym/analyse b/pym/analyse new file mode 100755 index 0000000..402a5d1 --- /dev/null +++ b/pym/analyse @@ -0,0 +1,46 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com> +# Copyright 2002-2010 Gentoo Technologies, Inc. +# Distributed under the terms of the GNU General Public License v2 or later +# +# $Header$ + +"""'analyse' is a flexible utility for Gentoo linux which can display various +information about installed packages, such as the USE flags used and the +packages that use them. It can also be used to help rebuild /etc/portage/package.* +files in the event of corruption, and possibly more. +""" + +import sys +# This block ensures that ^C interrupts are handled quietly. +try: + import signal + + def exithandler(signum,frame): + signal.signal(signal.SIGINT, signal.SIG_IGN) + signal.signal(signal.SIGTERM, signal.SIG_IGN) + print + sys.exit(1) + + signal.signal(signal.SIGINT, exithandler) + signal.signal(signal.SIGTERM, exithandler) + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +except KeyboardInterrupt: + print + sys.exit(1) + +from gentoolkit import analyse, errors + +try: + analyse.main() +except errors.GentoolkitException, err: + if '--debug' in sys.argv: + raise + else: + from gentoolkit import pprinter as pp + sys.stderr.write(pp.error(str(err))) + print + print "Add '--debug' to global options for traceback." + sys.exit(1) diff --git a/pym/gentoolkit/__init__.py b/pym/gentoolkit/__init__.py index 37d609b..5032c8d 100644 --- a/pym/gentoolkit/__init__.py +++ b/pym/gentoolkit/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/python # # Copyright 2003-2004 Karl Trygve Kalleberg -# Copyright 2003-2010 Gentoo Foundation +# Copyright 2003-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # $Header$ @@ -17,6 +17,8 @@ CONFIG = { 'piping': False if sys.stdout.isatty() else True, # Set some defaults: 'quiet': False, + # verbose is True if not quiet and not piping + 'verbose': True, 'debug': False } diff --git a/pym/gentoolkit/analyse/__init__.py b/pym/gentoolkit/analyse/__init__.py new file mode 100644 index 0000000..7a5fbec --- /dev/null +++ b/pym/gentoolkit/analyse/__init__.py @@ -0,0 +1,129 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com> +# Copyright(c) 2010, Gentoo Foundation +# Copyright 2003-2004 Karl Trygve Kalleberg +# Licensed under the GNU General Public License, v2 +# +# $Header: $ + +"""Gentoo's installed packages analysis and repair tool""" + + +# Move to Imports section after Python 2.6 is stable + + +__docformat__ = 'epytext' +# version is dynamically set by distutils sdist +__version__ = "svn" +__productname__ = "analyse" +__authors__ = ( + 'Brian Dolbec, <brian.dolbec@gmail.com>' + +) + +# make an exportable copy of the info for help output +MODULE_INFO = { + "__docformat__": __docformat__, + "__doc__": __doc__, + "__version__": __version__, + "__productname__": __productname__, + "__authors__": __authors__ + +} + +import errno +import sys +import time +from getopt import getopt, GetoptError + +import portage +from portage import os + +import gentoolkit as gen +from gentoolkit import errors +from gentoolkit import pprinter as pp +from gentoolkit.base import (initialize_configuration, split_arguments, + parse_global_options, print_help) +from gentoolkit.formatters import format_options + + +NAME_MAP = { + 'a': 'analyse', + 'r': 'rebuild' +} + +FORMATTED_OPTIONS = ( + (" (a)nalyse", + "analyses the installed PKG database USE flag or keyword useage"), + (" (r)ebuild", + "analyses the Installed PKG database and generates files suitable"), + (" ", + "to replace corrupted or missing /etc/portage/package.* files") + ) + +def expand_module_name(module_name): + """Returns one of the values of NAME_MAP or raises KeyError""" + + if module_name == 'list': + # list is a Python builtin type, so we must rename our module + return 'list_' + elif module_name in NAME_MAP.values(): + return module_name + else: + return NAME_MAP[module_name] + + +def main(): + """Parse input and run the program.""" + + short_opts = "hqCNV" + long_opts = ( + 'help', 'quiet', 'nocolor', 'no-color', 'no-pipe', 'version', 'debug' + ) + + initialize_configuration() + + try: + global_opts, args = getopt(sys.argv[1:], short_opts, long_opts) + except GetoptError as err: + sys.stderr.write(" \n") + sys.stderr.write(pp.error("Global %s\n" % err)) + print_help(MODULE_INFO, FORMATTED_OPTIONS, with_description=False) + sys.exit(2) + + # Parse global options + need_help = parse_global_options(global_opts, args, MODULE_INFO, FORMATTED_OPTIONS) + + if gen.CONFIG['quiet']: + gen.CONFIG['verbose'] = False + + try: + module_name, module_args = split_arguments(args) + except IndexError: + print_help(MODULE_INFO, FORMATTED_OPTIONS) + sys.exit(2) + + if need_help: + module_args.append('--help') + + try: + expanded_module_name = expand_module_name(module_name) + except KeyError: + sys.stderr.write(pp.error("Unknown module '%s'" % module_name)) + print_help(MODULE_INFO, FORMATTED_OPTIONS, with_description=False) + sys.exit(2) + + try: + loaded_module = __import__( + expanded_module_name, globals(), locals(), [], -1 + ) + loaded_module.main(module_args) + except portage.exception.AmbiguousPackageName as err: + raise errors.GentoolkitAmbiguousPackage(err) + except IOError as err: + if err.errno != errno.EPIPE: + raise + +if __name__ == '__main__': + main() diff --git a/pym/gentoolkit/analyse/analyse.py b/pym/gentoolkit/analyse/analyse.py new file mode 100644 index 0000000..6797510 --- /dev/null +++ b/pym/gentoolkit/analyse/analyse.py @@ -0,0 +1,355 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com> +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# + +"""Provides a breakdown list of USE flags or keywords used and by +what packages according to the Installed package database""" + +from __future__ import print_function + +import sys + +import gentoolkit +from gentoolkit.dbapi import PORTDB, VARDB +from gentoolkit.analyse.base import ModuleBase +from gentoolkit import pprinter as pp +from gentoolkit.analyse.lib import (get_installed_use, get_iuse, abs_flag, + abs_list, get_all_cpv_use, get_flags, FlagAnalyzer, KeywordAnalyser) +from gentoolkit.analyse.output import nl, AnalysisPrinter +from gentoolkit.package import Package + +import portage + + +def gather_flags_info( + cpvs=None, + system_flags=None, + include_unset=False, + target="USE", + use_portage=False, + # override-able for testing + _get_flags=get_flags, + _get_used=get_installed_use + ): + """Analyse the installed pkgs USE flags for frequency of use + + @param cpvs: optional list of [cat/pkg-ver,...] to analyse or + defaults to entire installed pkg db + @rtype dict. {flag:{"+":[cat/pkg-ver,...], "-":[cat/pkg-ver,...], "unset":[]} + """ + if cpvs is None: + cpvs = VARDB.cpv_all() + # pass them in to override for tests + flags = FlagAnalyzer(system_flags, + _get_flags=_get_flags, + _get_used=get_installed_use + ) + flag_users = {} + for cpv in cpvs: + if cpv.startswith("virtual"): + continue + if use_portage: + plus, minus, unset = flags.analyse_cpv(cpv) + else: + pkg = Package(cpv) + plus, minus, unset = flags.analyse_pkg(pkg) + for flag in plus: + if flag in flag_users: + flag_users[flag]["+"].append(cpv) + else: + flag_users[flag] = {"+": [cpv], "-": []} + for flag in minus: + if flag in flag_users: + flag_users[flag]["-"].append(cpv) + else: + flag_users[flag] = {"+":[], "-": [cpv]} + if include_unset: + for flag in unset: + if flag in flag_users: + if "unset" in flag_users[flag]: + flag_users[flag]["unset"].append(cpv) + else: + flag_users[flag]["unset"] = [cpv] + else: + flag_users[flag] = {"+": [], "-": [], "unset": [cpv]} + return flag_users + + +def gather_keywords_info( + cpvs=None, + system_keywords=None, + use_portage=False, + # override-able for testing + keywords=portage.settings["ACCEPT_KEYWORDS"], + analyser = None + ): + """Analyse the installed pkgs 'keywords' for frequency of use + + @param cpvs: optional list of [cat/pkg-ver,...] to analyse or + defaults to entire installed pkg db + @param system_keywords: list of the system keywords + @param keywords: user defined list of keywords to check and report on + or reports on all relevant keywords found to have been used. + @param _get_kwds: overridable function for testing + @param _get_used: overridable function for testing + @rtype dict. {keyword:{"stable":[cat/pkg-ver,...], "testing":[cat/pkg-ver,...]} + """ + if cpvs is None: + cpvs = VARDB.cpv_all() + keyword_users = {} + for cpv in cpvs: + if cpv.startswith("virtual"): + continue + if use_portage: + keyword = analyser.get_inst_keyword_cpv(cpv) + else: + pkg = Package(cpv) + keyword = analyser.get_inst_keyword_pkg(pkg) + #print "returned keyword =", cpv, keyword, keyword[0] + key = keyword[0] + if key in ["~", "-"]: + _kwd = keyword[1:] + if _kwd in keyword_users: + if key in ["~"]: + keyword_users[_kwd]["testing"].append(cpv) + elif key in ["-"]: + #print "adding cpv to missing:", cpv + keyword_users[_kwd]["missing"].append(cpv) + else: + if key in ["~"]: + keyword_users[_kwd] = {"stable": [], + "testing": [cpv], "missing": []} + elif key in ["-"]: + keyword_users[_kwd] = {"stable": [], + "testing": [], "missing": [cpv]} + else: + keyword_users[_kwd] = {"stable": [cpv], + "testing": [], "missing": []} + elif keyword in keyword_users: + keyword_users[keyword]["stable"].append(cpv) + else: + keyword_users[keyword] = {"stable": [cpv], "testing": [], "missing": []} + return keyword_users + + +class Analyse(ModuleBase): + """Installed db analysis tool to query the installed databse + and produce/output stats for USE flags or keywords/mask. + The 'rebuild' action output is in the form suitable for file type output + to create a new package.use, package.keywords, package.unmask + type files in the event of needing to rebuild the + /etc/portage/* user configs + """ + def __init__(self): + ModuleBase.__init__(self) + self.module_name = "analyse" + self.options = { + "flags": False, + "keywords": False, + "unset": False, + "verbose": False, + "quiet": False, + 'prefix': False, + 'portage': False + } + self.module_opts = { + "-f": ("flags", "boolean", True), + "--flags": ("flags", "boolean", True), + "-k": ("keywords", "boolean", True), + "--keywords": ("keywords", "boolean", True), + "-u": ("unset", "boolean", True), + "--unset": ("unset", "boolean", True), + "-v": ("verbose", "boolean", True), + "--verbose": ("verbose", "boolean", True), + "-p": ("prefix", "boolean", True), + "--prefix": ("prefix", "boolean", True), + "-G": ("portage", "boolean", True), + "--portage": ("portage", "boolean", True), + } + self.formatted_options = [ + (" -h, --help", "Outputs this useage message"), + (" -a, --analyse", + "Action, sets the module to gather data and output the"), + ("", "formatted stats/information to the screen"), + (" -u, --unset", + "Additionally include any unset USE flags and the packages"), + ("", "that could use them"), + (" -v, --verbose", + "Used in the analyse action to output more detailed information"), + (" -p, --prefix", + "Used for testing purposes only, runs report using " + + "a prefix keyword and 'prefix' USE flag"), + (" -G, --portage", + "Use portage directly instead of gentoolkit's Package " + + "object for some operations. Usually a little faster."), + ] + self.formatted_args = [ + (" use", + "causes the action to analyse the installed packages USE flags"), + (" pkguse", + "causes the action to analyse the installed packages PKGUSE flags"), + (" ", + "These are flags that have been set in /etc/portage/package.use"), + (" keywords", + "causes the action to analyse the installed packages keywords"), + ] + self.short_opts = "huvpG" + self.long_opts = ("help", "unset", "verbose", "prefix", "portage") + self.need_queries = True + self.arg_spec = "Target" + self.arg_options = ['use', 'pkguse','keywords'] + self.arg_option = False + + def run(self, input_args, quiet=False): + """runs the module + + @param input_args: input arguments to be parsed + """ + query = self.main_setup(input_args) + query = self.validate_query(query) + if query in ["use", "pkguse"]: + self.analyse_flags(query) + elif query in ["keywords"]: + self.analyse_keywords() + + def analyse_flags(self, target): + """This will scan the installed packages db and analyse the + USE flags used for installation and produce a report on how + they were used. + + @type target: string + @param target: the target to be analysed, one of ["use", "pkguse"] + """ + system_use = portage.settings["USE"].split() + self.printer = AnalysisPrinter("use", self.options["verbose"], system_use) + if self.options["verbose"]: + cpvs = VARDB.cpv_all() + #print "Total number of installed ebuilds =", len(cpvs) + flag_users = gather_flags_info(cpvs, system_use, + self.options["unset"], target=target.upper(), + use_portage=self.options['portage']) + else: + flag_users = gather_flags_info(system_flags=system_use, + include_unset=self.options["unset"], target=target.upper(), + use_portage=self.options['portage']) + #print flag_users + flag_keys = list(flag_users.keys()) + flag_keys.sort() + if self.options["verbose"]: + print(" Flag System #pkgs cat/pkg-ver") + blankline = nl + elif not self.options['quiet']: + print(" Flag System #pkgs") + blankline = lambda: None + for flag in flag_keys: + flag_pos = flag_users[flag]["+"] + if len(flag_pos): + self.printer(flag, "+", flag_pos) + #blankline() + flag_neg = flag_users[flag]["-"] + if len(flag_neg): + self.printer(flag, "-", flag_neg) + #blankline() + if "unset" in flag_users[flag] and flag_users[flag]["unset"]: + flag_unset = flag_users[flag]["unset"] + self.printer(flag, "unset", flag_unset) + #blankline() + if not self.options['quiet']: + print("===================================================") + print("Total number of flags in report =", pp.output.red(str(len(flag_keys)))) + if self.options["verbose"]: + print("Total number of installed ebuilds =", pp.output.red(str(len(cpvs)))) + print() + + + def analyse_keywords(self, keywords=None): + """This will scan the installed packages db and analyse the + keywords used for installation and produce a report on them. + """ + print() + system_keywords = portage.settings["ACCEPT_KEYWORDS"] + arch = portage.settings["ARCH"] + if self.options["prefix"]: + # build a new keyword for testing + system_keywords = "~" + arch + "-linux" + if self.options["verbose"] or self.options["prefix"]: + print("Current system ARCH =", arch) + print("Current system ACCEPT_KEYWORDS =", system_keywords) + system_keywords = system_keywords.split() + self.printer = AnalysisPrinter("keywords", self.options["verbose"], system_keywords) + self.analyser = KeywordAnalyser( arch, system_keywords, VARDB) + #self.analyser.set_order(portage.settings["USE"].split()) + # only for testing + test_use = portage.settings["USE"].split() + if self.options['prefix'] and 'prefix' not in test_use: + print("ANALYSE_KEYWORDS() 'prefix' flag not found in system USE flags!!! appending for testing") + print() + test_use.append('prefix') + self.analyser.set_order(test_use) + # /end testing + + if self.options["verbose"]: + cpvs = VARDB.cpv_all() + #print "Total number of installed ebuilds =", len(cpvs) + keyword_users = gather_keywords_info( + cpvs=cpvs, + system_keywords=system_keywords, + use_portage=self.options['portage'], + keywords=keywords, analyser = self.analyser + ) + blankline = nl + else: + keyword_users = gather_keywords_info( + system_keywords=system_keywords, + use_portage=self.options['portage'], + keywords=keywords, + analyser = self.analyser + ) + blankline = lambda: None + #print keyword_users + keyword_keys = list(keyword_users.keys()) + keyword_keys.sort() + if self.options["verbose"]: + print(" Keyword System #pkgs cat/pkg-ver") + elif not self.options['quiet']: + print(" Keyword System #pkgs") + for keyword in keyword_keys: + kwd_stable = keyword_users[keyword]["stable"] + if len(kwd_stable): + self.printer(keyword, " ", kwd_stable) + blankline() + kwd_testing = keyword_users[keyword]["testing"] + if len(kwd_testing): + self.printer(keyword, "~", kwd_testing) + blankline() + kwd_missing = keyword_users[keyword]["missing"] + if len(kwd_missing): + self.printer(keyword, "-", kwd_missing) + blankline + if not self.options['quiet']: + if self.analyser.mismatched: + print("_________________________________________________") + print(("The following packages were found to have a \n" + + "different recorded ARCH than the current system ARCH")) + for cpv in self.analyser.mismatched: + print("\t", pp.cpv(cpv)) + print("===================================================") + print("Total number of keywords in report =", pp.output.red(str(len(keyword_keys)))) + if self.options["verbose"]: + print("Total number of installed ebuilds =", pp.output.red(str(len(cpvs)))) + print() + + +def main(input_args): + """Common starting method by the analyse master + unless all modules are converted to this class method. + + @param input_args: input args as supplied by equery master module. + """ + query_module = Analyse() + query_module.run(input_args, gentoolkit.CONFIG['quiet']) + +# vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/analyse/base.py b/pym/gentoolkit/analyse/base.py new file mode 100644 index 0000000..bb0a8bc --- /dev/null +++ b/pym/gentoolkit/analyse/base.py @@ -0,0 +1,136 @@ +# Copyright(c) 2009, Gentoo Foundation +# +# Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com> +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# +# $Header: $ + +"""Analyse Base Module class to hold common module operation functions +""" + +from __future__ import print_function + +__docformat__ = 'epytext' + + +import errno +import sys +import time +from getopt import gnu_getopt, GetoptError + +from portage import os + +import gentoolkit.pprinter as pp +from gentoolkit.formatters import format_options +from gentoolkit.base import mod_usage + + +class ModuleBase(object): + """Analyse base module class to parse module options print module help, etc..""" + + def __init__(self): + self.module_name = None + self.options = {} + self.formatted_options = None + self.short_opts = None + self.long_opts = None + self.module_opts = {} + self.depwarning = None + self.need_queries = True + + + def print_help(self, with_description=True): + """Print description, usage and a detailed help message. + + @type with_description: bool + @param with_description: if true, print module's __doc__ string + """ + + if with_description: + print() + print(__doc__.strip()) + print() + if self.depwarning: + print() + for line in self.depwarning: + sys.stderr.write(pp.warn(line)) + print() + print(mod_usage(mod_name=self.module_name, arg=self.arg_spec, optional=self.arg_option)) + print() + print(pp.command("options")) + print(format_options( self.formatted_options )) + if self.formatted_args: + print() + print(pp.command(self.arg_spec)) + print(format_options(self.formatted_args)) + print() + + def parse_module_options(self, module_opts): + """Parse module options and update self.options""" + + opts = (x[0] for x in module_opts) + posargs = (x[1] for x in module_opts) + for opt, posarg in zip(opts, posargs): + if opt in ('-h', '--help'): + self.print_help() + sys.exit(0) + opt_name, opt_type, opt_setting = self.module_opts[opt] + if opt_type == 'boolean': + self.options[opt_name] = opt_setting + elif opt_type == 'int': + if posarg.isdigit(): + val = int(posarg) + else: + print() + err = "Module option %s requires integer (got '%s')" + sys.stdout.write(pp.error(err % (opt,posarg))) + print() + self.print_help(with_description=False) + sys.exit(2) + self.options[opt_name] = val + if self.options['quiet']: + self.options['verbose'] = False + + def validate_query(self, query, depth=0): + """check that the query meets the modules TargetSpec + If not it attempts to reduce it to a valid TargetSpec + or prints the help message and exits + """ + if depth > 1: + return [] + if len(query) > 1: + query = list(set(self.arg_options).intersection(query)) + #print "reduced query =", query + query = self.validate_query(query, depth+1) + if isinstance(query, list): + query = query[0] + if query not in self.arg_options: + print() + print(pp.error( + "Error starting module. Incorrect or No TargetSpec specified!" + )) + print("query = ", query) + self.print_help() + sys.exit(2) + return query + + + def main_setup(self, input_args): + """Parse input and prepares the program""" + + try: + module_opts, queries = gnu_getopt(input_args, self.short_opts, self.long_opts) + except GetoptError as err: + sys.stderr.write(pp.error("Module %s" % err)) + print() + self.print_help(with_description=False) + sys.exit(2) + self.parse_module_options(module_opts) + if self.need_queries and not queries: + self.print_help() + sys.exit(2) + return queries + + +# vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/analyse/lib.py b/pym/gentoolkit/analyse/lib.py new file mode 100644 index 0000000..17e2118 --- /dev/null +++ b/pym/gentoolkit/analyse/lib.py @@ -0,0 +1,477 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com> +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# + + +"""Provides support functions to analyse modules""" + +import sys + +from gentoolkit.dbapi import PORTDB, VARDB +from gentoolkit import errors +from gentoolkit.keyword import abs_keywords +#from gentoolkit.package import Package + +import portage + + +def get_installed_use(cpv, use="USE"): + """Gets the installed USE flags from the VARDB + + @type: cpv: string + @param cpv: cat/pkg-ver + @type use: string + @param use: 1 of ["USE", "PKGUSE"] + @rtype list + @returns [] or the list of IUSE flags + """ + return VARDB.aux_get(cpv,[use])[0].split() + + +def get_iuse(cpv): + """Gets the current IUSE flags from the tree + + @type: cpv: string + @param cpv: cat/pkg-ver + @rtype list + @returns [] or the list of IUSE flags + """ + try: + return PORTDB.aux_get(cpv, ["IUSE"])[0].split() + except: + return [] + + +def abs_flag(flag): + """Absolute value function for a USE flag + + @type flag: string + @param flag: the use flag to absolute. + @rtype: string + @return absolute USE flag + """ + if flag[0] in ["+","-"]: + return flag[1:] + else: + return flag + + +def abs_list(the_list): + """Absolute value function for a USE flag list + + @type the_list: list + @param the_list: the use flags to absolute. + @rtype: list + @return absolute USE flags + """ + r=[] + for member in the_list: + r.append(abs_flag(member)) + return r + + +def filter_flags(use, use_expand_hidden, usemasked, useforced): + """Filter function to remove hidden or otherwise not normally + visible USE flags from a list. + + @type use: list + @param use: the USE flag list to be filtered. + @type use_expand_hidden: list + @param use_expand_hidden: list of flags hidden. + @type usemasked: list + @param usemasked: list of masked USE flags. + @type useforced: list + @param useforced: the forced USE flags. + @rtype: list + @return the filtered USE flags. + """ + # clean out some environment flags, since they will most probably + # be confusing for the user + for f in use_expand_hidden: + f=f.lower() + "_" + for x in use: + if f in x: + use.remove(x) + # clean out any arch's + archlist = portage.settings["PORTAGE_ARCHLIST"].split() + for a in use[:]: + if a in archlist: + use.remove(a) + # dbl check if any from usemasked or useforced are still there + masked = usemasked + useforced + for a in use[:]: + if a in masked: + use.remove(a) + return use + + +def get_all_cpv_use(cpv): + """Uses portage to determine final USE flags and settings for an emerge + + @type cpv: string + @param cpv: eg cat/pkg-ver + @rtype: lists + @return use, use_expand_hidden, usemask, useforce + """ + use = None + PORTDB.settings.unlock() + try: + PORTDB.settings.setcpv(cpv, use_cache=True, mydb=portage.portdb) + use = portage.settings['PORTAGE_USE'].split() + use_expand_hidden = portage.settings["USE_EXPAND_HIDDEN"].split() + usemask = list(PORTDB.settings.usemask) + useforce = list(PORTDB.settings.useforce) + except KeyError: + PORTDB.settings.reset() + PORTDB.settings.lock() + return [], [], [], [] + # reset cpv filter + PORTDB.settings.reset() + PORTDB.settings.lock() + return use, use_expand_hidden, usemask, useforce + + +def get_flags(cpv, final_setting=False): + """Retrieves all information needed to filter out hidded, masked, etc. + USE flags for a given package. + + @type cpv: string + @param cpv: eg. cat/pkg-ver + @type final_setting: boolean + @param final_setting: used to also determine the final + enviroment USE flag settings and return them as well. + @rtype: list or list, list + @return IUSE or IUSE, final_flags + """ + final_use, use_expand_hidden, usemasked, useforced = get_all_cpv_use(cpv) + iuse_flags = filter_flags(get_iuse(cpv), use_expand_hidden, usemasked, useforced) + #flags = filter_flags(use_flags, use_expand_hidden, usemasked, useforced) + if final_setting: + final_flags = filter_flags(final_use, use_expand_hidden, usemasked, useforced) + return iuse_flags, final_flags + return iuse_flags + +class FlagAnalyzer(object): + """Specialty functions for analysing an installed package's + USE flags. Can be used for single or mulitple use without + needing to be reset unless the system USE flags are changed. + + @type system: list or set + @param system: the default system USE flags. + @type _get_flags: function + @param _get_flags: Normally defaulted, can be overriden for testing + @type _get_used: function + @param _get_used: Normally defaulted, can be overriden for testing + """ + def __init__(self, + system, + _get_flags=get_flags, + _get_used=get_installed_use + ): + self.get_flags = _get_flags + self.get_used = _get_used + self.reset(system) + + def reset(self, system): + """Resets the internal system USE flags and use_expand variables + to the new setting. The use_expand variable is handled internally. + + @type system: list or set + @param system: the default system USE flags. + """ + self.system = set(system) + self.use_expand = portage.settings['USE_EXPAND'].lower().split() + + def analyse_cpv(self, cpv): + """Gets all relavent USE flag info for a cpv and breaks them down + into 3 sets, plus (package.use enabled), minus ( package.use disabled), + unset. + + @param cpv: string. 'cat/pkg-ver' + @rtype tuple of sets + @return (plus, minus, unset) sets of USE flags + """ + installed = set(self.get_used(cpv, "USE")) + iuse = set(abs_list(self.get_flags(cpv))) + return self._analyse(installed, iuse) + + def _analyse(self, installed, iuse): + """Analyses the supplied info and returns the flag settings + that differ from the defaults + + @type installed: set + @param installed: the installed with use flags + @type iuse: set + @param iuse: the current ebuilds IUSE + """ + defaults = self.system.intersection(iuse) + usedflags = iuse.intersection(set(installed)) + plus = usedflags.difference(defaults) + minus = defaults.difference(usedflags) + unset = iuse.difference(defaults, plus, minus) + cleaned = self.remove_expanding(unset) + return (plus, minus, cleaned) + + def analyse_pkg(self, pkg): + """Gets all relevent USE flag info for a pkg and breaks them down + into 3 sets, plus (package.use enabled), minus ( package.use disabled), + unset. + + @param pkg: gentoolkit.package.Package object + @rtype tuple of sets + @return (plus, minus, unset) sets of USE flags + """ + installed = set(self.pkg_used(pkg)) + iuse = set(abs_list(self.pkg_flags(pkg))) + return self._analyse(installed, iuse) + + def pkg_used(self, pkg): + return pkg.use().split() + + def pkg_flags(self, pkg): + final_use, use_expand_hidden, usemasked, useforced = \ + get_all_cpv_use(pkg.cpv) + flags = pkg.environment("IUSE", prefer_vdb=False).split() + return filter_flags(flags, use_expand_hidden, usemasked, useforced) + + def redundant(self, cpv, iuse): + """Checks for redundant settings. + future function. Not yet implemented. + """ + pass + + def remove_expanding(self, flags): + """Remove unwanted USE_EXPAND flags + from unset IUSE sets + + @param flags: short list or set of USE flags + @rtype set + @return USE flags + """ + _flags = set(flags) + for expander in self.use_expand: + for flag in flags: + if expander in flag: + _flags.remove(flag) + if not _flags: + break + return _flags + + +class KeywordAnalyser(object): + """Specialty functions for analysing the installed package db for + keyword useage and the packages that used them. + + Note: should be initialized with the internal set_order() before use. + See internal set_order() for more details. + This class of functions can be used for single cpv checks or + used repeatedly for an entire package db. + + @type arch: string + @param arch: the system ARCH setting + @type accept_keywords: list + @param accept_keywords: eg. ['x86', '~x86'] + @type get_aux: function, defaults to: VARDB.aux_get + @param vardb: vardb class of functions, needed=aux_get() + to return => KEYWORDS & USE flags for a cpv + = aux_get(cpv, ["KEYWORDS", "USE"]) + """ + + # parsing order to determine appropriate keyword used for installation + normal_order = ['stable', 'testing', 'prefix', 'testing_prefix', 'missing'] + prefix_order = ['prefix', 'testing_prefix', 'stable', 'testing', 'missing'] + parse_range = list(range(len(normal_order))) + + + def __init__(self, arch, accept_keywords, vardb=VARDB): + self.arch = arch + self.accept_keywords = accept_keywords + self.vardb = vardb + self.prefix = '' + self.parse_order = None + self.check_key = { + 'stable': self._stable, + 'testing': self._testing, + 'prefix': self._prefix, + 'testing_prefix': self._testing_prefix, + 'missing': self._missing + } + self.mismatched = [] + + def determine_keyword(self, keywords, used, cpv): + """Determine the keyword from the installed USE flags and + the KEYWORDS that was used to install a package. + + @param keywords: list of keywords available to install a pkg + @param used: list of USE flalgs recorded for the installed pkg + @rtype: string + @return a keyword or null string + """ + used = set(used) + kwd = None + result = '' + if keywords: + absolute_kwds = abs_keywords(keywords) + kwd = list(used.intersection(absolute_kwds)) + #if keywords == ['~ppc64']: + #print "Checked keywords for kwd", keywords, used, "kwd =", kwd + if not kwd: + #print "Checking for kwd against portage.archlist" + absolute_kwds = abs_keywords(keywords) + # check for one against archlist then re-check + kwd = list(absolute_kwds.intersection(portage.archlist)) + #print "determined keyword =", kwd + if len(kwd) == 1: + key = kwd[0] + #print "determined keyword =", key + elif not kwd: + #print "kwd != 1", kwd, cpv + result = self._missing(self.keyword, keywords) + else: # too many, try to narrow them dowm + #print "too many kwd's, trying to match against arch" + _kwd = list(set(kwd).intersection(self.arch)) + key = '' + if _kwd: + #print "found one! :)", _kwd + key = _kwd + else: # try re-running the short list against archlist + #print "Checking kwd for _kwd against portage.archlist" + _kwd = list(set(kwd).intersection(portage.archlist)) + if _kwd and len(_kwd) == 1: + #print "found one! :)", _kwd + key = _kwd[0] + else: + #print " :( didn't work, _kwd =", _kwd, "giving up on:", cpv + result = self._missing(self.keyword, keywords) + i = 0 + while not result and i in self.parse_range: + parsekey = self.parse_order[i] + result = self.check_key[parsekey](key, keywords) + i += 1 + return result + + def _stable(self, key, keywords): + """test for a normal stable keyword""" + if key in keywords: + return key + return '' + + def _testing(self, key, keywords): + """test for a normal testing keyword""" + if ("~" + key) in keywords: + return "~" + key + return '' + + def _prefix(self, key, keywords): + """test for a stable prefix keyword""" + if not self.prefix: + return '' + _key = '-'.join([key, self.prefix]) + if _key in keywords: + #print key, "is in", keywords + return _key + return '' + + def _testing_prefix(self, key, keywords): + """test for a testing prefix keyword""" + if not self.prefix: + return '' + _key = "~" +'-'.join([key, self.prefix]) + if _key in keywords: + #print key, "is in", keywords + return _key + return '' + + def _missing(self, key, keywords): + """generates a missing keyword to return""" + if self.prefix and key != self.keyword: + _key = '-'.join([key, self.prefix]) + else: + _key = '-' + key + #print "_missisng :( _key =", _key + return _key + + def get_inst_keyword_cpv(self, cpv): + """Determines the installed with keyword for cpv + + @type cpv: string + @param cpv: an installed CAT/PKG-VER + @rtype: string + @returns a keyword determined to have been used to install cpv + """ + keywords, used = self.vardb.aux_get(cpv, ["KEYWORDS", "USE"]) + keywords = keywords.split() + used = used.split() + return self._parse(keywords, used, cpv=cpv) + + def get_inst_keyword_pkg(self, pkg): + """Determines the installed with keyword for cpv + + @param pkg: gentoolkit.package.Package object + @rtype: string + @returns a keyword determined to have been used to install cpv + """ + keywords, used = pkg.environment(["KEYWORDS", "USE"], + prefer_vdb=True, fallback=False) + keywords = keywords.split() + used = used.split() + return self._parse(keywords, used, pkg=pkg) + + def _parse(self, keywords, used, pkg=None, cpv=None): + if pkg: + _cpv = pkg.cpv + else: + _cpv = cpv + if not self.parse_order: + self.set_order(used) + keyword = self.keyword + # sanity check + if self.arch not in used: + #print "Found a mismatch = ", cpv, self.arch, used + self.mismatched.append(_cpv) + if keyword in keywords: + #print "keyword", keyword, "is in", keywords + return keyword + elif "~"+keyword in keywords: + #print "~keyword", keyword, "is in", keywords + return "~"+keyword + else: + keyword = self.determine_keyword(keywords, used, _cpv) + if not keyword: + raise errors.GentoolkitUnknownKeyword(_cpv, ' '.join(keywords), used) + return keyword + + def set_order(self, used): + """Used to set the parsing order to determine a keyword + used for installation. + + This is needed due to the way prefix arch's and keywords + work with portage. It looks for the 'prefix' flag. A positive result + sets it to the prefix order and keyword. + + @type used: list + @param used: a list of pkg USE flags or the system USE flags""" + if 'prefix' in used: + #print "SET_ORDER() Setting parse order to prefix" + prefix = None + self.parse_order = self.prefix_order + for key in self.accept_keywords: + #print "SET_ORDER() '"+key+"'" + if '-' in key: + #print "SET_ORDER()found prefix keyword :", key + if self.arch in key: + prefix = key.split('-')[1] + #print "prefix =", prefix + self.prefix = prefix + self.keyword = '-'.join([self.arch, prefix]) + else: + #print "SET_ORDER() Setting parse order to normal" + self.parse_order = self.normal_order + self.keyword = self.arch + #print "SET_ORDER() completed: prefix =", self.prefix, ", keyword =", \ + # self.keyword, "parse order =",self.parse_order + #print + diff --git a/pym/gentoolkit/analyse/output.py b/pym/gentoolkit/analyse/output.py new file mode 100644 index 0000000..4844eb0 --- /dev/null +++ b/pym/gentoolkit/analyse/output.py @@ -0,0 +1,213 @@ +#!/usr/bin/python +# +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# + +"""Provides various output classes and functions for +both screen and file output +""" + +from __future__ import print_function + +import gentoolkit +from gentoolkit import pprinter as pp +from gentoolkit.textwrap_ import TextWrapper +from gentoolkit.cpv import split_cpv + + +def nl(lines=1): + """small utility function to print blank lines + + @type lines: integer + @param lines: optional number of blank lines to print + default = 1 + """ + print(('\n' * lines)) + +class AnalysisPrinter(object): + """Printing functions""" + def __init__(self, target, verbose=True, references=None): + """@param references: list of accepted keywords or + the system use flags + """ + self.references = references + self.set_target(target, verbose) + + def set_target(self, target, verbose=True): + if target in ["use"]: + if verbose: + self.print_fn = self.print_use_verbose + else: + self.print_fn = self.print_use_quiet + elif target in ["keywords"]: + if verbose: + self.print_fn = self.print_keyword_verbose + else: + self.print_fn = self.print_keyword_quiet + + def __call__(self, key, active, pkgs): + self._format_key(key, active, pkgs) + + def _format_key(self, key, active, pkgs): + """Determines the stats for key, formats it and + calls the pre-determined print function + """ + occurred = str(len(pkgs)) + if active in ["-", "~"]: + _key = active + key + else: + _key = key + if _key in self.references: + default = "default" + else: + default = "......." + count = ' '*(5-len(occurred)) + occurred + pkgs.sort() + self.print_fn(key, active, default, count, pkgs) + + @staticmethod + def print_use_verbose(key, active, default, count, pkgs): + """Verbosely prints a set of use flag info. including the pkgs + using them. + """ + _pkgs = pkgs[:] + if active in ["+", "-"]: + _key = pp.useflag((active+key), active=="+") + else: + _key = (" " + key) + cpv = _pkgs.pop(0) + print(_key,'.'*(35-len(key)), default, pp.number(count), pp.cpv(cpv)) + while _pkgs: + cpv = _pkgs.pop(0) + print(' '*52 + pp.cpv(cpv)) + + # W0613: *Unused argument %r* + # pylint: disable-msg=W0613 + @staticmethod + def print_use_quiet(key, active, default, count, pkgs): + """Quietly prints a subset set of USE flag info.. + """ + if active in ["+", "-"]: + _key = pp.useflag((active+key), active=="+") + else: + _key = (" " + key) + print(_key,'.'*(35-len(key)), default, pp.number(count)) + + @staticmethod + def print_keyword_verbose(key, stability, default, count, pkgs): + """Verbosely prints a set of keywords info. including the pkgs + using them. + """ + _pkgs = pkgs[:] + _key = (pp.keyword((stability+key),stable=(stability==" "), + hard_masked=stability=="-")) + cpv = _pkgs.pop(0) + print(_key,'.'*(20-len(key)), default, pp.number(count), pp.cpv(cpv)) + while _pkgs: + cpv = _pkgs.pop(0) + print(' '*37 + pp.cpv(cpv)) + + # W0613: *Unused argument %r* + # pylint: disable-msg=W0613 + @staticmethod + def print_keyword_quiet(key, stability, default, count, pkgs): + """Quietly prints a subset set of USE flag info.. + """ + _key = (pp.keyword((stability+key), stable=(stability==" "), + hard_masked=stability=="-")) + print(_key,'.'*(20-len(key)), default, pp.number(count)) + + +class RebuildPrinter(object): + """Output functions""" + def __init__(self, target, pretend=True, exact=False): + """@param references: list of accepted keywords or + the system use flags + """ + self.target = target + self.set_target(target) + self.pretend = pretend + if pretend: + self.twrap = TextWrapper(width=gentoolkit.CONFIG['termWidth']) + self.spacer = ' ' + self.init_indent = len(self.spacer) + else: + self.spacer = '' + self.exact = exact + self.data = {} + + + def set_target(self, target): + if target in ["use"]: + self.print_fn = self.print_use + self.use_lines = [self.header()] + elif target in ["keywords"]: + self.print_fn = self.print_keyword + elif target in ["unmask"]: + self.print_fn = self.print_mask + + + def __call__(self, key, values): + self._format_key(key, values) + + + def _format_key(self, key, values): + """Determines the stats for key, formats it and + calls the pre-determined print function + """ + if self.exact: + _key = "=" + key + else: + parts = split_cpv(key) + _key = '/'.join(parts[:2]) + values.sort() + self.data[_key] = values + self.print_fn( _key, values) + + def _format_values(self, key, values): + """Format entry values ie. USE flags, keywords,... + + @type key: str + @param key: a pre-formatted cpv + @type values: list of pre-formatted strings + @param values: ['flag1', 'flag2',...] + @rtype: str + @return: formatted options string + """ + + result = [] + self.twrap.initial_indent = pp.cpv(key+" ") + self.twrap.subsequent_indent = " " * (len(key)+1) + result.append(self.twrap.fill(values)) + return '\n'.join(result) + + def print_use(self, key, values): + """Prints a USE flag string. + """ + if self.pretend: + flags = [] + for flag in values: + flags.append(pp.useflag(flag, (flag[0] != '-'))) + print(self._format_values(self.spacer+key, ' '.join(flags))) + else: + line = ' '.join([key, ' '.join(values)]) + self.use_lines.append(line) + + + def print_keyword(self): + pass + + + def print_unmask(self): + pass + + def header(self): + """Generates a file header + """ + + h=("# This package.%s file was generated by " + %self.target + + "gentoolkit's 'analyse rebuild' module\n" + ) + return h diff --git a/pym/gentoolkit/analyse/rebuild.py b/pym/gentoolkit/analyse/rebuild.py new file mode 100644 index 0000000..31c00a1 --- /dev/null +++ b/pym/gentoolkit/analyse/rebuild.py @@ -0,0 +1,215 @@ +#!/usr/bin/python +# +# Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com> +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# + + +"""Provides a rebuild file of USE flags or keywords used and by +what packages according to the Installed package database""" + +from __future__ import print_function + + +# Move to Imports section after Python 2.6 is stable + + +import sys + +from portage import os + +import gentoolkit +from gentoolkit.dbapi import PORTDB, VARDB +from gentoolkit.analyse.base import ModuleBase +from gentoolkit import pprinter as pp +from gentoolkit.analyse.lib import (get_installed_use, get_flags, + abs_flag, abs_list, FlagAnalyzer) +from gentoolkit.analyse.output import RebuildPrinter + +import portage + + +def cpv_all_diff_use( + cpvs=None, + system_flags=None, + # override-able for testing + _get_flags=get_flags, + _get_used=get_installed_use + ): + if cpvs is None: + cpvs = VARDB.cpv_all() + cpvs.sort() + data = {} + # pass them in to override for tests + flags = FlagAnalyzer(system_flags, + _get_flags=_get_flags, + _get_used=get_installed_use + ) + for cpv in cpvs: + plus, minus, unset = flags.analyse_cpv(cpv) + for flag in minus: + plus.add("-"+flag) + if len(plus): + data[cpv] = list(plus) + return data + + +class Rebuild(ModuleBase): + """Installed db analysis tool to query the installed databse + and produce/output stats for USE flags or keywords/mask. + The 'rebuild' action output is in the form suitable for file type output + to create a new package.use, package.keywords, package.unmask + type files in the event of needing to rebuild the + /etc/portage/* user configs + """ + def __init__(self): + ModuleBase.__init__(self) + self.module_name = "rebuild" + self.options = { + "use": False, + "keywords": False, + "unmask": False, + "verbose": False, + "quiet": False, + "exact": False, + "pretend": False, + #"unset": False + } + self.module_opts = { + "-p": ("pretend", "boolean", True), + "--pretend": ("pretend", "boolean", True), + "-e": ("exact", "boolean", True), + "--exact": ("exact", "boolean", True), + "-v": ("verbose", "boolean", True), + "--verbose": ("verbose", "boolean", True), + } + self.formatted_options = [ + (" -h, --help", "Outputs this useage message"), + (" -p, --pretend", "Does not actually create the files."), + (" ", "It directs the outputs to the screen"), + (" -e, --exact", "will atomize the package with a"), + (" ", "leading '=' and include the version") + ] + self.formatted_args = [ + (" use", + "causes the action to analyse the installed packages USE flags"), + (" keywords", + "causes the action to analyse the installed packages keywords"), + (" unmask", + "causes the action to analyse the installed packages " + \ + "current mask status") + ] + self.short_opts = "hepv" + self.long_opts = ("help", "exact", "pretend", "verbose") + self.need_queries = True + self.arg_spec = "TargetSpec" + self.arg_options = ['use', 'keywords', 'unmask'] + self.arg_option = False + + + + def run(self, input_args, quiet=False): + """runs the module + + @param input_args: input arguments to be parsed + """ + self.options['quiet'] = quiet + query = self.main_setup(input_args) + query = self.validate_query(query) + if query in ["use"]: + self.rebuild_use() + elif query in ["keywords"]: + self.rebuild_keywords() + elif query in ["mask"]: + self.rebuild_mask() + + + def rebuild_use(self): + if not self.options["quiet"]: + print() + print(" -- Scanning installed packages for USE flag settings that") + print(" do not match the default settings") + system_use = portage.settings["USE"].split() + output = RebuildPrinter("use", self.options["pretend"], self.options["exact"]) + pkgs = cpv_all_diff_use(system_flags=system_use) + pkg_count = len(pkgs) + if self.options["verbose"]: + print() + print((pp.emph(" -- Found ") + pp.number(str(pkg_count)) + + pp.emph(" packages that need entries"))) + #print pp.emph(" package.use to maintain their current setting") + if pkgs: + pkg_keys = list(pkgs.keys()) + pkg_keys.sort() + #print len(pkgs) + if self.options["pretend"] and not self.options["quiet"]: + print() + print(pp.globaloption( + " -- These are the installed packages & flags " + + "that were detected")) + print(pp.globaloption(" to need flag settings other " + + "than the defaults.")) + print() + elif not self.options["quiet"]: + print(" -- preparing pkgs for file entries") + flag_count = 0 + unique_flags = set() + for pkg in pkg_keys: + if self.options['verbose']: + flag_count += len(pkgs[pkg]) + unique_flags.update(abs_list(pkgs[pkg])) + output(pkg, pkgs[pkg]) + if self.options['verbose']: + message = (pp.emph(" ") + + pp.number(str(len(unique_flags))) + + pp.emph(" unique flags\n") + " " + + pp.number(str(flag_count))+ + pp.emph(" flag entries\n") + " " + + pp.number(str(pkg_count)) + + pp.emph(" different packages")) + print() + print(pp.globaloption(" -- Totals")) + print(message) + #print + #unique = list(unique_flags) + #unique.sort() + #print unique + if not self.options["pretend"]: + filepath = os.path.expanduser('~/package.use.test') + self.save_file(filepath, output.use_lines) + + def rebuild_keywords(self): + print("Module action not yet available") + print() + + def rebuild_mask(self): + print("Module action not yet available") + print() + + + def save_file(self, filepath, data): + """Writes the data to the file determined by filepath + + @param filepath: string. eg. '/path/to/filename' + @param data: list of lines to write to filepath + """ + if not self.options["quiet"]: + print(' - Saving file: %s' %filepath) + #print "Just kidding :) I haven't codded it yet" + with open(filepath, "w") as output: + output.write('\n'.join(data)) + print(" - Done") + + +def main(input_args): + """Common starting method by the analyse master + unless all modules are converted to this class method. + + @param input_args: input args as supplied by equery master module. + """ + query_module = Rebuild() + query_module.run(input_args, gentoolkit.CONFIG['quiet']) + +# vim: set ts=4 sw=4 tw=79: + diff --git a/pym/gentoolkit/atom.py b/pym/gentoolkit/atom.py index d32a20b..eb4a358 100644 --- a/pym/gentoolkit/atom.py +++ b/pym/gentoolkit/atom.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2009-2010 Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -102,6 +102,9 @@ class Atom(portage.dep.Atom, CPV): #return cmp(self.repo_name, other.repo_name) return True + def __hash__(self): + return hash(self.atom) + def __ne__(self, other): return not self == other @@ -133,12 +136,16 @@ class Atom(portage.dep.Atom, CPV): # return c if self.slot != other.slot: + if self.slot is None: + return False + elif other.slot is None: + return True return self.slot < other.slot - this_use = None + this_use = [] if self.use is not None: this_use = sorted(self.use.tokens) - that_use = None + that_use = [] if other.use is not None: that_use = sorted(other.use.tokens) if this_use != that_use: diff --git a/pym/gentoolkit/base.py b/pym/gentoolkit/base.py new file mode 100644 index 0000000..9819390 --- /dev/null +++ b/pym/gentoolkit/base.py @@ -0,0 +1,151 @@ +# Copyright(c) 2009, Gentoo Foundation +# +# Copyright 2010 Brian Dolbec <brian.dolbec@gmail.com> +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# +# $Header: $ + +"""Analyse Base Module class to hold common module operation functions +""" + +from __future__ import print_function + +__docformat__ = 'epytext' + + +import errno +import sys +import time +from getopt import gnu_getopt, GetoptError + +from portage import os + +import gentoolkit +from gentoolkit import errors +#from gentoolkit.textwrap_ import TextWrapper +import gentoolkit.pprinter as pp +from gentoolkit.formatters import format_options + + +GLOBAL_OPTIONS = ( + (" -h, --help", "display this help message"), + (" -q, --quiet", "minimal output"), + (" -C, --no-color", "turn off colors"), + (" -N, --no-pipe", "turn off pipe detection"), + (" -V, --version", "display version info") +) + + +def initialize_configuration(): + """Setup the standard equery config""" + + # Get terminal size + term_width = pp.output.get_term_size()[1] + if term_width == -1: + # get_term_size() failed. Set a sane default width: + term_width = 80 + # Terminal size, minus a 1-char margin for text wrapping + gentoolkit.CONFIG['termWidth'] = term_width - 1 + # Guess color output + if (gentoolkit.CONFIG['color'] == -1 and (not sys.stdout.isatty() or + os.getenv("NOCOLOR") in ("yes", "true")) or gentoolkit.CONFIG['color'] == 0): + pp.output.nocolor() + gentoolkit.CONFIG['verbose'] = not gentoolkit.CONFIG['piping'] + + +def split_arguments(args): + """Separate module name from module arguments""" + + return args.pop(0), args + + +def main_usage(module_info): + """Return the main usage message for analyse""" + return "%(usage)s %(product)s [%(g_opts)s] %(mod_name)s [%(mod_opts)s]" % { + 'usage': pp.emph("Usage:"), + 'product': pp.productname(module_info["__productname__"]), + 'g_opts': pp.globaloption("global-options"), + 'mod_name': pp.command("module-name"), + 'mod_opts': pp.localoption("module-options") + } + + +def print_version(module_info): + """Print the version of this tool to the console.""" + + print("%(product)s (%(version)s) - %(docstring)s" % { + "product": pp.productname(module_info["__productname__"]), + "version": module_info["__version__"], + "docstring": module_info["__doc__"] + }) + + +def print_help(module_info, formatted_options=None, with_description=True): + """Print description, usage and a detailed help message. + + @param with_description (bool): Option to print module's __doc__ or not + """ + + if with_description: + print() + print(module_info["__doc__"]) + print() + print(main_usage(module_info)) + print() + print(pp.globaloption("global options")) + print(format_options(GLOBAL_OPTIONS)) + print() + if formatted_options: + print(pp.command("modules") + " (" + pp.command("short name") + ")") + print(format_options(formatted_options)) + else: + print("Error: calling function did not supply formatted options") + print() + + +def parse_global_options(global_opts, args, module_info, formatted_options): + """Parse global input args and return True if we should display help for + the called module, else False (or display help and exit from here). + """ + + need_help = False + opts = (opt[0] for opt in global_opts) + for opt in opts: + if opt in ('-h', '--help'): + if args: + need_help = True + else: + print_help( module_info, formatted_options) + sys.exit(0) + elif opt in ('-q','--quiet'): + gentoolkit.CONFIG['quiet'] = True + elif opt in ('-C', '--no-color', '--nocolor'): + gentoolkit.CONFIG['color'] = 0 + pp.output.nocolor() + elif opt in ('-N', '--no-pipe'): + gentoolkit.CONFIG['piping'] = False + elif opt in ('-V', '--version'): + print_version(module_info) + sys.exit(0) + elif opt in ('--debug'): + gentoolkit.CONFIG['debug'] = True + return need_help + + +def mod_usage(mod_name="module", arg="pkgspec", optional=False): + """Provide a consistant usage message to the calling module. + + @type arg: string + @param arg: what kind of argument the module takes (pkgspec, filename, etc) + @type optional: bool + @param optional: is the argument optional? + """ + + return "%(usage)s: %(mod_name)s [%(opts)s] %(arg)s" % { + 'usage': pp.emph("Usage"), + 'mod_name': pp.command(mod_name), + 'opts': pp.localoption("options"), + 'arg': ("[%s]" % pp.emph(arg)) if optional else pp.emph(arg) + } + diff --git a/pym/gentoolkit/cpv.py b/pym/gentoolkit/cpv.py index f390e45..9f2c303 100644 --- a/pym/gentoolkit/cpv.py +++ b/pym/gentoolkit/cpv.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2009-2010 Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -8,13 +8,17 @@ """Provides attributes and methods for a category/package-version string.""" -__all__ = ('CPV',) +__all__ = ( + 'CPV', + 'compare_strs', + 'split_cpv' +) # ======= # Imports # ======= -from portage.versions import catpkgsplit, vercmp +from portage.versions import catpkgsplit, vercmp, pkgcmp from gentoolkit import errors @@ -64,6 +68,9 @@ class CPV(object): return False return self.cpv == other.cpv + def __hash__(self): + return hash(self.cpv) + def __ne__(self, other): return not self == other @@ -81,11 +88,7 @@ class CPV(object): # FIXME: this cmp() hack is for vercmp not using -1,0,1 # See bug 266493; this was fixed in portage-2.2_rc31 #return vercmp(self.fullversion, other.fullversion) - result = cmp(vercmp(self.fullversion, other.fullversion), 0) - if result == -1: - return True - else: - return False + return vercmp(self.fullversion, other.fullversion) < 0 def __gt__(self, other): if not isinstance(other, self.__class__): @@ -119,6 +122,26 @@ class CPV(object): # Functions # ========= +def compare_strs(pkg1, pkg2): + """Similar to the builtin cmp, but for package strings. Usually called + as: package_list.sort(cpv.compare_strs) + + An alternative is to use the CPV descriptor from gentoolkit.cpv: + >>> cpvs = sorted(CPV(x) for x in package_list) + + @see: >>> help(cmp) + """ + + pkg1 = catpkgsplit(pkg1) + pkg2 = catpkgsplit(pkg2) + if pkg1[0] != pkg2[0]: + return -1 if pkg1[0] < pkg2[0] else 1 + elif pkg1[1] != pkg2[1]: + return -1 if pkg1[1] < pkg2[1] else 1 + else: + return pkgcmp(pkg1[1:], pkg2[1:]) + + def split_cpv(cpv): """Split a cpv into category, name, version and revision. diff --git a/pym/gentoolkit/dbapi.py b/pym/gentoolkit/dbapi.py index fae1b6f..2866214 100644 --- a/pym/gentoolkit/dbapi.py +++ b/pym/gentoolkit/dbapi.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2009-2010 Gentoo Foundation +# Copyright 2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # $Header$ diff --git a/pym/gentoolkit/dependencies.py b/pym/gentoolkit/dependencies.py index 2d5e28b..fda28f7 100644 --- a/pym/gentoolkit/dependencies.py +++ b/pym/gentoolkit/dependencies.py @@ -1,4 +1,4 @@ -# Copyright 2009-2010 Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -19,8 +19,9 @@ from portage.dep import paren_reduce from gentoolkit import errors from gentoolkit.atom import Atom from gentoolkit.cpv import CPV -from gentoolkit.helpers import find_best_match, uniqify +from gentoolkit.helpers import uniqify from gentoolkit.dbapi import PORTDB, VARDB +from gentoolkit.query import Query # ======= # Classes @@ -84,7 +85,7 @@ class Dependencies(CPV): try: return self.parser(self.environment(('DEPEND',))[0]) - except portage.exception.InvalidPackageName, err: + except portage.exception.InvalidPackageName as err: raise errors.GentoolkitInvalidCPV(err) def get_pdepend(self): @@ -92,7 +93,7 @@ class Dependencies(CPV): try: return self.parser(self.environment(('PDEPEND',))[0]) - except portage.exception.InvalidPackageName, err: + except portage.exception.InvalidPackageName as err: raise errors.GentoolkitInvalidCPV(err) def get_rdepend(self): @@ -100,7 +101,7 @@ class Dependencies(CPV): try: return self.parser(self.environment(('RDEPEND',))[0]) - except portage.exception.InvalidPackageName, err: + except portage.exception.InvalidPackageName as err: raise errors.GentoolkitInvalidCPV(err) def get_all_depends(self): @@ -109,7 +110,7 @@ class Dependencies(CPV): env_vars = ('DEPEND', 'PDEPEND', 'RDEPEND') try: return self.parser(' '.join(self.environment(env_vars))) - except portage.exception.InvalidPackageName, err: + except portage.exception.InvalidPackageName as err: raise errors.GentoolkitInvalidCPV(err) def graph_depends( @@ -151,7 +152,7 @@ class Dependencies(CPV): try: pkgdep = depcache[dep.atom] except KeyError: - pkgdep = find_best_match(dep.atom) + pkgdep = Query(dep.atom).find_best() depcache[dep.atom] = pkgdep if pkgdep and pkgdep.cpv in seen: continue diff --git a/pym/gentoolkit/deprecated/helpers.py b/pym/gentoolkit/deprecated/helpers.py index 68514d6..bd5d1bd 100644 --- a/pym/gentoolkit/deprecated/helpers.py +++ b/pym/gentoolkit/deprecated/helpers.py @@ -1,12 +1,14 @@ #!/usr/bin/python2 # # Copyright(c) 2004, Karl Trygve Kalleberg <karltk@gentoo.org> -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # # $Header$ +from __future__ import print_function + import warnings import portage @@ -29,7 +31,7 @@ def find_packages(search_key, masked=False): t = portage.db["/"]["porttree"].dbapi.match(search_key) t += portage.db["/"]["vartree"].dbapi.match(search_key) # catch the "amgigous package" Exception - except ValueError, e: + except ValueError as e: if isinstance(e[0],list): t = [] for cp in e[0]: @@ -41,8 +43,8 @@ def find_packages(search_key, masked=False): t += portage.db["/"]["vartree"].dbapi.match(cp) else: raise ValueError(e) - except portage_exception.InvalidAtom, e: - print warn("Invalid Atom: '%s'" % str(e)) + except portage_exception.InvalidAtom as e: + print(warn("Invalid Atom: '%s'" % str(e))) return [] # Make the list of packages unique t = unique_array(t) @@ -56,15 +58,15 @@ def find_installed_packages(search_key, masked=False): try: t = portage.db["/"]["vartree"].dbapi.match(search_key) # catch the "amgigous package" Exception - except ValueError, e: + except ValueError as e: if isinstance(e[0],list): t = [] for cp in e[0]: t += portage.db["/"]["vartree"].dbapi.match(cp) else: raise ValueError(e) - except portage_exception.InvalidAtom, e: - print warn("Invalid Atom: '%s'" % str(e)) + except portage_exception.InvalidAtom as e: + print(warn("Invalid Atom: '%s'" % str(e))) return [] return [Package(x) for x in t] @@ -118,7 +120,7 @@ def find_all_installed_packages(prefilter=None): DeprecationWarning) t = vartree.dbapi.cpv_all() if prefilter: - t = filter(prefilter,t) + t = list(filter(prefilter,t)) return [Package(x) for x in t] def find_all_uninstalled_packages(prefilter=None): @@ -136,7 +138,7 @@ def find_all_packages(prefilter=None): t = porttree.dbapi.cp_all() t += vartree.dbapi.cp_all() if prefilter: - t = filter(prefilter,t) + t = list(filter(prefilter,t)) t = unique_array(t) t2 = [] for x in t: @@ -173,4 +175,4 @@ def split_package_name(name): # return pkglist if __name__ == "__main__": - print "This module is for import only" + print("This module is for import only") diff --git a/pym/gentoolkit/equery/__init__.py b/pym/gentoolkit/equery/__init__.py index 5833f29..2bee8d9 100644 --- a/pym/gentoolkit/equery/__init__.py +++ b/pym/gentoolkit/equery/__init__.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -6,8 +6,10 @@ """Gentoo package query tool""" +from __future__ import print_function + # Move to Imports section after Python 2.6 is stable -from __future__ import with_statement + __all__ = ( 'format_options', @@ -23,12 +25,12 @@ __version__ = "svn" # ======= import errno -import os import sys import time from getopt import getopt, GetoptError import portage +from portage import os import gentoolkit from gentoolkit import CONFIG @@ -72,20 +74,20 @@ def print_help(with_description=True): """ if with_description: - print __doc__ - print main_usage() - print - print pp.globaloption("global options") - print format_options(( + print(__doc__) + print(main_usage()) + print() + print(pp.globaloption("global options")) + print(format_options(( (" -h, --help", "display this help message"), (" -q, --quiet", "minimal output"), (" -C, --no-color", "turn off colors"), (" -N, --no-pipe", "turn off pipe detection"), (" -V, --version", "display version info") - )) - print - print pp.command("modules") + " (" + pp.command("short name") + ")" - print format_options(( + ))) + print() + print(pp.command("modules") + " (" + pp.command("short name") + ")") + print(format_options(( (" (b)elongs", "list what package FILES belong to"), (" (c)hanges", "list changelog entries for ATOM"), (" chec(k)", "verify checksums and timestamps for PKG"), @@ -98,7 +100,7 @@ def print_help(with_description=True): (" (s)ize", "display total size of all files owned by PKG"), (" (u)ses", "display USE flags for PKG"), (" (w)hich", "print full path to ebuild for PKG") - )) + ))) def expand_module_name(module_name): @@ -269,7 +271,6 @@ def parse_global_options(global_opts, args): pp.output.nocolor() elif opt in ('-N', '--no-pipe'): CONFIG['piping'] = False - CONFIG['verbose'] = True elif opt in ('-V', '--version'): print_version() sys.exit(0) @@ -282,11 +283,11 @@ def parse_global_options(global_opts, args): def print_version(): """Print the version of this tool to the console.""" - print "%(product)s (%(version)s) - %(docstring)s" % { + print("%(product)s (%(version)s) - %(docstring)s" % { "product": pp.productname(__productname__), "version": __version__, "docstring": __doc__ - } + }) def split_arguments(args): @@ -307,7 +308,7 @@ def main(): try: global_opts, args = getopt(sys.argv[1:], short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Global %s" % err)) print_help(with_description=False) sys.exit(2) @@ -341,9 +342,9 @@ def main(): expanded_module_name, globals(), locals(), [], -1 ) loaded_module.main(module_args) - except portage.exception.AmbiguousPackageName, err: + except portage.exception.AmbiguousPackageName as err: raise errors.GentoolkitAmbiguousPackage(err) - except IOError, err: + except IOError as err: if err.errno != errno.EPIPE: raise diff --git a/pym/gentoolkit/equery/belongs.py b/pym/gentoolkit/equery/belongs.py index 3845b9d..b426c3a 100644 --- a/pym/gentoolkit/equery/belongs.py +++ b/pym/gentoolkit/equery/belongs.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -10,6 +10,8 @@ Note: Normally, only one package will own a file. If multiple packages own the same file, it usually constitutes a problem, and should be reported. """ +from __future__ import print_function + __docformat__ = 'epytext' # ======= @@ -57,19 +59,19 @@ class BelongsPrinter(object): def print_quiet(self, pkg, cfile): "Format for minimal output." if self.name_only: - name = pkg.cpv.cp + name = pkg.cp else: name = str(pkg.cpv) - print name + print(name) def print_verbose(self, pkg, cfile): "Format for full output." file_str = pp.path(format_filetype(cfile, pkg.parsed_contents()[cfile])) if self.name_only: - name = pkg.cpv.cp + name = pkg.cp else: name = str(pkg.cpv) - print pp.cpv(name), "(" + file_str + ")" + print(pp.cpv(name), "(" + file_str + ")") # ========= @@ -88,7 +90,7 @@ def parse_module_options(module_opts): if opt == '--earlyout': sys.stderr.write(pp.warn("Use of --earlyout is deprecated.")) sys.stderr.write(pp.warn("Please use --early-out.")) - print + print() QUERY_OPTS['earlyOut'] = True elif opt in ('-f', '--full-regex'): QUERY_OPTS['fullRegex'] = True @@ -104,17 +106,17 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print - print mod_usage(mod_name="belongs", arg="filename") - print - print pp.command("options") - print format_options(( + print(__doc__.strip()) + print() + print(mod_usage(mod_name="belongs", arg="filename")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -f, --full-regex", "supplied query is a regex" ), (" -e, --early-out", "stop when first match is found"), (" -n, --name-only", "don't print the version") - )) + ))) def main(input_args): @@ -126,9 +128,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -139,7 +141,7 @@ def main(input_args): sys.exit(2) if CONFIG['verbose']: - print " * Searching for %s ... " % (pp.regexpquery(",".join(queries))) + print(" * Searching for %s ... " % (pp.regexpquery(",".join(queries)))) printer_fn = BelongsPrinter( verbose=CONFIG['verbose'], name_only=QUERY_OPTS['nameOnly'] diff --git a/pym/gentoolkit/equery/changes.py b/pym/gentoolkit/equery/changes.py index 8adf246..d860704 100644 --- a/pym/gentoolkit/equery/changes.py +++ b/pym/gentoolkit/equery/changes.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 or higher # @@ -6,8 +6,10 @@ """Displays the ChangeLog entry for the latest installable version of an atom""" +from __future__ import print_function + # Move to Imports sections when Python 2.6 is stable -from __future__ import with_statement + __docformat__ = 'epytext' @@ -15,15 +17,17 @@ __docformat__ = 'epytext' # Imports # ======= -import os import sys from getopt import gnu_getopt, GetoptError +from portage import os + import gentoolkit.pprinter as pp from gentoolkit import errors from gentoolkit.atom import Atom from gentoolkit.equery import format_options, mod_usage -from gentoolkit.helpers import ChangeLog, find_best_match, find_packages +from gentoolkit.helpers import ChangeLog +from gentoolkit.query import Query # ======= # Globals @@ -49,19 +53,19 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print - print mod_usage(mod_name="changes") - print - print pp.emph("examples") + print(__doc__.strip()) + print() + print(mod_usage(mod_name="changes")) + print() + print(pp.emph("examples")) print (" c portage # show latest visible " "version's entry") - print " c portage --full --limit=3 # show 3 latest entries" - print " c '=sys-apps/portage-2.1.6*' # use atom syntax" - print " c portage --from=2.2_rc20 --to=2.2_rc30 # use version ranges" - print - print pp.command("options") - print format_options(( + print(" c portage --full --limit=3 # show 3 latest entries") + print(" c '=sys-apps/portage-2.1.6*' # use atom syntax") + print(" c portage --from=2.2_rc60 --to=2.2_rc70 # use version ranges") + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -l, --latest", "display only the latest ChangeLog entry"), (" -f, --full", "display the full ChangeLog"), @@ -69,33 +73,7 @@ def print_help(with_description=True): "limit the number of entries displayed (with --full)"), (" --from=VER", "set which version to display from"), (" --to=VER", "set which version to display to"), - )) - - -def get_match(query): - """Find a valid package from which to get the ChangeLog path. - - @raise GentoolkitNoMatches: if no matches found - """ - - match = matches = None - match = find_best_match(query) - - if not match: - matches = find_packages(query, include_masked=True) - else: - matches = [match] - - if not matches: - raise errors.GentoolkitNoMatches(query) - - return matches[0] - - -def is_ranged(atom): - """Return True if an atom string appears to be ranged, else False.""" - - return atom.startswith(('~', '<', '>')) or atom.endswith('*') + ))) def parse_module_options(module_opts): @@ -126,9 +104,9 @@ def print_entries(entries): for i, entry in enumerate(entries): # , start=1): in py2.6 i += 1 if i < len_entries: - print entry + print(entry) else: - print entry.strip() + print(entry.strip()) def set_limit(posarg): @@ -142,7 +120,7 @@ def set_limit(posarg): else: err = "Module option --limit requires integer (got '%s')" sys.stderr.write(pp.error(err % posarg)) - print + print() print_help(with_description=False) sys.exit(2) @@ -155,9 +133,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -168,11 +146,11 @@ def main(input_args): sys.exit(2) first_run = True - for query in queries: + for query in (Query(x) for x in queries): if not first_run: - print + print() - match = get_match(query) + match = query.find_best() changelog_path = os.path.join(match.package_path(), 'ChangeLog') changelog = ChangeLog(changelog_path) @@ -183,7 +161,7 @@ def main(input_args): if (QUERY_OPTS['onlyLatest'] or ( changelog.entries and not changelog.indexed_entries )): - print changelog.latest.strip() + print(changelog.latest.strip()) else: end = QUERY_OPTS['limit'] or len(changelog.indexed_entries) if QUERY_OPTS['to'] or QUERY_OPTS['from']: @@ -197,7 +175,10 @@ def main(input_args): print_entries(changelog.full[:end]) else: # Raises GentoolkitInvalidAtom here if invalid - atom = Atom(query) if is_ranged(query) else '=' + str(match.cpv) + if query.is_ranged(): + atom = Atom(str(query)) + else: + atom = '=' + str(match.cpv) print_entries(changelog.entries_matching_atom(atom)[:end]) first_run = False diff --git a/pym/gentoolkit/equery/check.py b/pym/gentoolkit/equery/check.py index 5969804..7353b31 100644 --- a/pym/gentoolkit/equery/check.py +++ b/pym/gentoolkit/equery/check.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -6,40 +6,39 @@ """Checks timestamps and MD5 sums for files owned by a given installed package""" +from __future__ import print_function + __docformat__ = 'epytext' # ======= # Imports # ======= -import os import sys from functools import partial from getopt import gnu_getopt, GetoptError import portage.checksum as checksum +from portage import os import gentoolkit.pprinter as pp from gentoolkit import errors from gentoolkit.equery import format_options, mod_usage, CONFIG -from gentoolkit.helpers import do_lookup +from gentoolkit.query import Query # ======= # Globals # ======= QUERY_OPTS = { - "includeInstalled": True, - "includeOverlayTree": False, - "includePortTree": False, - "checkMD5sum": True, - "checkTimestamp" : True, - "isRegex": False, - "onlyFailures": False, - "printMatchInfo": False, - "showSummary" : True, - "showPassedFiles" : False, - "showFailedFiles" : True + "in_installed": True, + "in_porttree": False, + "in_overlay": False, + "check_MD5sum": True, + "check_timestamp" : True, + "is_regex": False, + "only_failures": False, + "show_progress": False, } # ======= @@ -179,8 +178,8 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print + print(__doc__.strip()) + print() # Deprecation warning added by djanderson, 12/2008 depwarning = ( @@ -191,16 +190,16 @@ def print_help(with_description=True): ) for line in depwarning: sys.stderr.write(pp.warn(line)) - print + print() - print mod_usage(mod_name="check") - print - print pp.command("options") - print format_options(( + print(mod_usage(mod_name="check")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -f, --full-regex", "query is a regular expression"), (" -o, --only-failures", "only display packages that do not pass"), - )) + ))) def checks_printer(cpv, data, verbose=True, only_failures=False): @@ -214,10 +213,10 @@ def checks_printer(cpv, data, verbose=True, only_failures=False): else: if verbose: if not cpv in seen: - print "* Checking %s ..." % (pp.emph(str(cpv))) + print("* Checking %s ..." % (pp.emph(str(cpv)))) seen.append(cpv) else: - print "%s:" % cpv, + print("%s:" % cpv, end=' ') if verbose: for err in errs: @@ -227,9 +226,9 @@ def checks_printer(cpv, data, verbose=True, only_failures=False): n_passed = pp.number(str(n_passed)) n_checked = pp.number(str(n_checked)) info = " %(n_passed)s out of %(n_checked)s files passed" - print info % locals() + print(info % locals()) else: - print "failed(%s)" % n_failed + print("failed(%s)" % n_failed) def parse_module_options(module_opts): @@ -241,9 +240,9 @@ def parse_module_options(module_opts): print_help() sys.exit(0) elif opt in ('-f', '--full-regex'): - QUERY_OPTS['isRegex'] = True + QUERY_OPTS['is_regex'] = True elif opt in ('-o', '--only-failures'): - QUERY_OPTS['onlyFailures'] = True + QUERY_OPTS['only_failures'] = True def main(input_args): @@ -254,9 +253,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -267,11 +266,11 @@ def main(input_args): sys.exit(2) first_run = True - for query in queries: + for query in (Query(x, QUERY_OPTS['is_regex']) for x in queries): if not first_run: - print + print() - matches = do_lookup(query, QUERY_OPTS) + matches = query.smart_find(**QUERY_OPTS) if not matches: raise errors.GentoolkitNoMatches(query, in_installed=True) @@ -281,7 +280,7 @@ def main(input_args): printer = partial( checks_printer, verbose=CONFIG['verbose'], - only_failures=QUERY_OPTS['onlyFailures'] + only_failures=QUERY_OPTS['only_failures'] ) check = VerifyContents(printer_fn=printer) check(matches) diff --git a/pym/gentoolkit/equery/depends.py b/pym/gentoolkit/equery/depends.py index a1061fc..4fccaaa 100644 --- a/pym/gentoolkit/equery/depends.py +++ b/pym/gentoolkit/equery/depends.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -6,6 +6,8 @@ """List all packages that depend on a atom given query""" +from __future__ import print_function + __docformat__ = 'epytext' # ======= @@ -18,8 +20,8 @@ from getopt import gnu_getopt, GetoptError import gentoolkit.pprinter as pp from gentoolkit.dependencies import Dependencies from gentoolkit.equery import format_options, mod_usage, CONFIG -from gentoolkit.helpers import (get_cpvs, get_installed_cpvs, - compare_package_strings) +from gentoolkit.helpers import get_cpvs, get_installed_cpvs +from gentoolkit.cpv import CPV # ======= # Globals @@ -51,7 +53,7 @@ class DependPrinter(object): """Verbosely prints a set of dep strings.""" sep = ' ? ' if (depatom and use_conditional) else '' - print indent + pp.cpv(cpv), "(" + use_conditional + sep + depatom + ")" + print(indent + pp.cpv(cpv), "(" + use_conditional + sep + depatom + ")") # W0613: *Unused argument %r* # pylint: disable-msg=W0613 @@ -59,7 +61,7 @@ class DependPrinter(object): def print_quiet(indent, cpv, use_conditional, depatom): """Quietly prints a subset set of dep strings.""" - print indent + pp.cpv(cpv) + print(indent + pp.cpv(cpv)) def format_depend(self, dep, dep_is_displayed): """Format a dependency for printing. @@ -104,19 +106,19 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print - print mod_usage(mod_name="depends") - print - print pp.command("options") - print format_options(( + print(__doc__.strip()) + print() + print(mod_usage(mod_name="depends")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -a, --all-packages", "include dependencies that are not installed (slow)"), (" -D, --indirect", "search both direct and indirect dependencies"), (" --depth=N", "limit indirect dependency tree to specified depth") - )) + ))) def parse_module_options(module_opts): @@ -138,7 +140,7 @@ def parse_module_options(module_opts): else: err = "Module option --depth requires integer (got '%s')" sys.stdout.write(pp.error(err % posarg)) - print + print() print_help(with_description=False) sys.exit(2) QUERY_OPTS["maxDepth"] = depth @@ -151,9 +153,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -171,7 +173,7 @@ def main(input_args): first_run = True for query in queries: if not first_run: - print + print() pkg = Dependencies(query) if QUERY_OPTS['includeMasked']: @@ -180,9 +182,9 @@ def main(input_args): pkggetter = get_installed_cpvs if CONFIG['verbose']: - print " * These packages depend on %s:" % pp.emph(str(pkg.cpv)) + print(" * These packages depend on %s:" % pp.emph(pkg.cpv)) pkg.graph_reverse_depends( - pkgset=sorted(pkggetter(), cmp=compare_package_strings), + pkgset=sorted(pkggetter(), key = CPV), max_depth=QUERY_OPTS["maxDepth"], only_direct=QUERY_OPTS["onlyDirect"], printer_fn=dep_print diff --git a/pym/gentoolkit/equery/depgraph.py b/pym/gentoolkit/equery/depgraph.py index 9c7e346..53f0b5b 100644 --- a/pym/gentoolkit/equery/depgraph.py +++ b/pym/gentoolkit/equery/depgraph.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -6,6 +6,8 @@ """Display a direct dependency graph for a given package""" +from __future__ import print_function + __docformat__ = 'epytext' # ======= @@ -16,10 +18,13 @@ import sys from functools import partial from getopt import gnu_getopt, GetoptError +import portage + import gentoolkit.pprinter as pp from gentoolkit import errors from gentoolkit.equery import format_options, mod_usage, CONFIG -from gentoolkit.helpers import do_lookup +from gentoolkit.keyword import determine_keyword +from gentoolkit.query import Query # ======= # Globals @@ -27,16 +32,15 @@ from gentoolkit.helpers import do_lookup QUERY_OPTS = { "depth": 1, - "noAtom": False, - "noIndent": False, - "noUseflags": False, - "includeInstalled": True, - "includePortTree": True, - "includeOverlayTree": True, - "includeMasked": True, - "isRegex": False, - "matchExact": True, - "printMatchInfo": (not CONFIG['quiet']) + "no_atom": False, + "no_indent": False, + "no_useflags": False, + "no_mask": False, + "in_installed": True, + "in_porttree": True, + "in_overlay": True, + "include_masked": True, + "show_progress": (not CONFIG['quiet']) } # ========= @@ -51,20 +55,21 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print - print "Default depth is set to 1 (direct only). Use --depth=0 for no max." - print - print mod_usage(mod_name="depgraph") - print - print pp.command("options") - print format_options(( + print(__doc__.strip()) + print() + print("Default depth is set to 1 (direct only). Use --depth=0 for no max.") + print() + print(mod_usage(mod_name="depgraph")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -A, --no-atom", "do not show dependency atom"), + (" -M, --no-mask", "do not show masking status"), (" -U, --no-useflags", "do not show USE flags"), (" -l, --linear", "do not format the graph by indenting dependencies"), (" --depth=N", "limit dependency graph to specified depth") - )) + ))) def parse_module_options(module_opts): @@ -77,18 +82,20 @@ def parse_module_options(module_opts): print_help() sys.exit(0) if opt in ('-A', '--no-atom'): - QUERY_OPTS["noAtom"] = True + QUERY_OPTS["no_atom"] = True if opt in ('-U', '--no-useflags'): - QUERY_OPTS["noUseflags"] = True + QUERY_OPTS["no_useflags"] = True + if opt in ('-M', '--no-mask'): + QUERY_OPTS["no_mask"] = True if opt in ('-l', '--linear'): - QUERY_OPTS["noIndent"] = True + QUERY_OPTS["no_indent"] = True if opt in ('--depth'): if posarg.isdigit(): depth = int(posarg) else: err = "Module option --depth requires integer (got '%s')" sys.stderr.write(pp.error(err % posarg)) - print + print() print_help(with_description=False) sys.exit(2) QUERY_OPTS["depth"] = depth @@ -101,7 +108,8 @@ def depgraph_printer( no_use=False, no_atom=False, no_indent=False, - initial_pkg=False + initial_pkg=False, + no_mask=False ): """Display L{gentoolkit.dependencies.Dependencies.graph_depends} results. @@ -124,29 +132,44 @@ def depgraph_printer( indent = '' if no_indent or initial_pkg else ' ' + (' ' * depth) decorator = '[%3d] ' % depth if no_indent else '`-- ' use = '' + atom = '' + mask = '' try: - atom = '' if no_atom else ' (%s)' % dep.atom + if not no_atom: + if dep.operator == '=*': + atom += ' (=%s*)' % dep.cpv + else: + atom += ' (%s%s)' % (dep.operator, dep.cpv) if not no_use and dep is not None and dep.use: use = ' [%s]' % ' '.join( pp.useflag(x, enabled=True) for x in dep.use.tokens ) except AttributeError: # 'NoneType' object has no attribute 'atom' - atom = '' + pass + if pkg and not no_mask: + mask = pkg.mask_status() + if not mask: + mask = [determine_keyword(portage.settings["ARCH"], + portage.settings["ACCEPT_KEYWORDS"], + pkg.environment('KEYWORDS'))] + mask = pp.masking(mask) try: - print ''.join((indent, decorator, pp.cpv(str(pkg.cpv)), atom, use)) + print(' '.join((indent, decorator, pp.cpv(str(pkg.cpv)), atom, mask, use))) except AttributeError: # 'NoneType' object has no attribute 'cpv' - print ''.join((indent, decorator, "(no match for %r)" % dep.atom)) + print(''.join((indent, decorator, "(no match for %r)" % dep.atom))) def make_depgraph(pkg, printer_fn): """Create and display depgraph for each package.""" if CONFIG['verbose']: - print " * direct dependency graph for %s:" % pp.cpv(str(pkg.cpv)) + print() # blank line improves readability & package version separation + print(" * " + pp.subsection("dependency graph for ") + pp.cpv(str(pkg.cpv))) else: - print "%s:" % str(pkg.cpv) + print() + print("%s:" % pkg.cpv) # Print out the first package printer_fn(0, pkg, None, initial_pkg=True) @@ -163,20 +186,20 @@ def make_depgraph(pkg, printer_fn): n_packages = pp.number(str(len(deps))) max_seen = pp.number(str(max(x[0] for x in deps))) info = "[ %s stats: packages (%s), max depth (%s) ]" - print info % (pkgname, n_packages, max_seen) + print(info % (pkgname, n_packages, max_seen)) def main(input_args): """Parse input and run the program""" - short_opts = "hAUl" - long_opts = ('help', 'no-atom', 'no-useflags', 'depth=') + short_opts = "hAMUl" + long_opts = ('help', 'no-atom', 'no-useflags', 'no-mask', 'depth=') try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -191,28 +214,32 @@ def main(input_args): # first_run = True - for query in queries: + for query in (Query(x) for x in queries): if not first_run: - print + print() - matches = do_lookup(query, QUERY_OPTS) + matches = query.smart_find(**QUERY_OPTS) if not matches: raise errors.GentoolkitNoMatches(query) + matches.sort() + if CONFIG['verbose']: printer = partial( depgraph_printer, - no_atom=QUERY_OPTS['noAtom'], - no_indent=QUERY_OPTS['noIndent'], - no_use=QUERY_OPTS['noUseflags'] + no_atom=QUERY_OPTS['no_atom'], + no_indent=QUERY_OPTS['no_indent'], + no_use=QUERY_OPTS['no_useflags'], + no_mask=QUERY_OPTS['no_mask'] ) else: printer = partial( depgraph_printer, no_atom=True, no_indent=True, - no_use=True + no_use=True, + no_mask=True ) for pkg in matches: diff --git a/pym/gentoolkit/equery/files.py b/pym/gentoolkit/equery/files.py index f0c40ee..ac2dc13 100644 --- a/pym/gentoolkit/equery/files.py +++ b/pym/gentoolkit/equery/files.py @@ -1,10 +1,12 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # # $Header: $ -"""List files owned by a given package""" +"""List files owned by a given package.""" + +from __future__ import print_function __docformat__ = 'epytext' @@ -12,35 +14,32 @@ __docformat__ = 'epytext' # Imports # ======= -import os import sys from getopt import gnu_getopt, GetoptError import portage +from portage import os import gentoolkit.pprinter as pp from gentoolkit.equery import (format_filetype, format_options, mod_usage, CONFIG) -from gentoolkit.helpers import do_lookup +from gentoolkit.query import Query # ======= # Globals # ======= QUERY_OPTS = { - "categoryFilter": None, - "includeInstalled": True, - "includePortTree": False, - "includeOverlayTree": False, - "includeMasked": True, - "isRegex": False, - "matchExact": True, - "outputTree": False, - "printMatchInfo": (not CONFIG['quiet']), - "showType": False, - "showTimestamp": False, - "showMD5": False, - "typeFilter": None + "in_installed": True, + "in_porttree": False, + "in_overlay": False, + "include_masked": True, + "output_tree": False, + "show_progress": (not CONFIG['quiet']), + "show_type": False, + "show_timestamp": False, + "show_MD5": False, + "type_filter": None } FILTER_RULES = ( @@ -59,12 +58,12 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print - print mod_usage(mod_name="files") - print - print pp.command("options") - print format_options(( + print(__doc__.strip()) + print() + print(mod_usage(mod_name="files")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -m, --md5sum", "include MD5 sum in output"), (" -s, --timestamp", "include timestamp in output"), @@ -73,8 +72,8 @@ def print_help(with_description=True): (" -f, --filter=RULES", "filter output by file type"), (" RULES", "a comma-separated list (no spaces); choose from:") - )) - print " " * 24, ', '.join(pp.emph(x) for x in FILTER_RULES) + ))) + print(" " * 24, ', '.join(pp.emph(x) for x in FILTER_RULES)) # R0912: *Too many branches (%s/%s)* @@ -87,12 +86,12 @@ def display_files(contents): @param contents: {'path': ['filetype', ...], ...} """ - filenames = contents.keys() + filenames = list(contents.keys()) filenames.sort() last = [] for name in filenames: - if QUERY_OPTS["outputTree"]: + if QUERY_OPTS["output_tree"]: dirdepth = name.count('/') indent = " " if dirdepth == 2: @@ -104,7 +103,7 @@ def display_files(contents): if contents[name][0] == "dir": if len(last) == 0: last = basename - print pp.path(indent + basename[0]) + print(pp.path(indent + basename[0])) continue for i, directory in enumerate(basename): try: @@ -114,22 +113,22 @@ def display_files(contents): pass last = basename if len(last) == 1: - print pp.path(indent + last[0]) + print(pp.path(indent + last[0])) continue - print pp.path(indent + "> /" + last[-1]) + print(pp.path(indent + "> /" + last[-1])) elif contents[name][0] == "sym": - print pp.path(indent + "+"), - print pp.path_symlink(basename[-1] + " -> " + contents[name][2]) + print(pp.path(indent + "+"), end=' ') + print(pp.path_symlink(basename[-1] + " -> " + contents[name][2])) else: - print pp.path(indent + "+ ") + basename[-1] + print(pp.path(indent + "+ ") + basename[-1]) else: - print format_filetype( + print(format_filetype( name, contents[name], - show_type=QUERY_OPTS["showType"], - show_md5=QUERY_OPTS["showMD5"], - show_timestamp=QUERY_OPTS["showTimestamp"] - ) + show_type=QUERY_OPTS["show_type"], + show_md5=QUERY_OPTS["show_MD5"], + show_timestamp=QUERY_OPTS["show_timestamp"] + )) def filter_by_doc(contents, content_filter): @@ -209,8 +208,8 @@ def filter_contents(contents): @return: contents with unrequested filetypes stripped """ - if QUERY_OPTS['typeFilter']: - content_filter = QUERY_OPTS['typeFilter'] + if QUERY_OPTS['type_filter']: + content_filter = QUERY_OPTS['type_filter'] else: return contents @@ -242,16 +241,14 @@ def parse_module_options(module_opts): if opt in ('-h', '--help'): print_help() sys.exit(0) - elif opt in ('-e', '--exact-name'): - QUERY_OPTS["matchExact"] = True elif opt in ('-m', '--md5sum'): - QUERY_OPTS["showMD5"] = True + QUERY_OPTS["show_MD5"] = True elif opt in ('-s', '--timestamp'): - QUERY_OPTS["showTimestamp"] = True + QUERY_OPTS["show_timestamp"] = True elif opt in ('-t', '--type'): - QUERY_OPTS["showType"] = True + QUERY_OPTS["show_type"] = True elif opt in ('--tree'): - QUERY_OPTS["outputTree"] = True + QUERY_OPTS["output_tree"] = True elif opt in ('-f', '--filter'): f_split = posarg.split(',') content_filter.extend(x.lstrip('=') for x in f_split) @@ -260,10 +257,10 @@ def parse_module_options(module_opts): sys.stderr.write( pp.error("Invalid filter rule '%s'" % rule) ) - print + print() print_help(with_description=False) sys.exit(2) - QUERY_OPTS["typeFilter"] = content_filter + QUERY_OPTS["type_filter"] = content_filter def main(input_args): @@ -276,9 +273,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -289,8 +286,8 @@ def main(input_args): sys.exit(2) # Turn off filtering for tree output - if QUERY_OPTS["outputTree"]: - QUERY_OPTS["typeFilter"] = None + if QUERY_OPTS["output_tree"]: + QUERY_OPTS["type_filter"] = None # # Output files @@ -299,18 +296,20 @@ def main(input_args): first_run = True for query in queries: if not first_run: - print + print() - matches = do_lookup(query, QUERY_OPTS) + matches = Query(query).smart_find(**QUERY_OPTS) if not matches: sys.stderr.write( pp.error("No matching packages found for %s" % query) ) + matches.sort() + for pkg in matches: if CONFIG['verbose']: - print " * Contents of %s:" % pp.cpv(str(pkg.cpv)) + print(" * Contents of %s:" % pp.cpv(str(pkg.cpv))) contents = pkg.parsed_contents() display_files(filter_contents(contents)) diff --git a/pym/gentoolkit/equery/hasuse.py b/pym/gentoolkit/equery/hasuse.py index 8d51013..88c9da1 100644 --- a/pym/gentoolkit/equery/hasuse.py +++ b/pym/gentoolkit/equery/hasuse.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 or higher # @@ -6,6 +6,8 @@ """List all installed packages that have a given USE flag""" +from __future__ import print_function + __docformat__ = 'epytext' # ======= @@ -18,21 +20,20 @@ from getopt import gnu_getopt, GetoptError import gentoolkit.pprinter as pp from gentoolkit import errors from gentoolkit.equery import format_options, mod_usage, CONFIG -from gentoolkit.helpers import do_lookup -from gentoolkit.package import PackageFormatter +from gentoolkit.package import PackageFormatter, FORMAT_TMPL_VARS +from gentoolkit.query import Query # ======= # Globals # ======= QUERY_OPTS = { - "categoryFilter": None, - "includeInstalled": True, - "includePortTree": False, - "includeOverlayTree": False, - "includeMasked": True, - "isRegex": False, # Necessary for do_lookup, don't change - "printMatchInfo": False + "in_installed": True, + "in_porttree": False, + "in_overlay": False, + "include_masked": True, + "show_progress": False, + "package_format": None } # ========= @@ -47,18 +48,22 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print - print mod_usage(mod_name="hasuse", arg="USE-flag") - print - print pp.command("options") - print format_options(( + print(__doc__.strip()) + print() + print(mod_usage(mod_name="hasuse", arg="USE-flag")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -I, --exclude-installed", "exclude installed packages from search path"), (" -o, --overlay-tree", "include overlays in search path"), - (" -p, --portage-tree", "include entire portage tree in search path") - )) + (" -p, --portage-tree", "include entire portage tree in search path"), + (" -F, --format=TMPL", "specify a custom output format"), + (" TMPL", + "a format template using (see man page):") + ))) + print(" " * 24, ', '.join(pp.emph(x) for x in FORMAT_TMPL_VARS)) def display_useflags(query, pkg): @@ -74,24 +79,32 @@ def display_useflags(query, pkg): return if CONFIG['verbose']: - fmt_pkg = PackageFormatter(pkg, do_format=True) + pkgstr = PackageFormatter( + pkg, + do_format=True, + custom_format=QUERY_OPTS["package_format"] + ) else: - fmt_pkg = PackageFormatter(pkg, do_format=False) - - if (QUERY_OPTS["includeInstalled"] and - not QUERY_OPTS["includePortTree"] and - not QUERY_OPTS["includeOverlayTree"]): - if not 'I' in fmt_pkg.location: + pkgstr = PackageFormatter( + pkg, + do_format=False, + custom_format=QUERY_OPTS["package_format"] + ) + + if (QUERY_OPTS["in_installed"] and + not QUERY_OPTS["in_porttree"] and + not QUERY_OPTS["in_overlay"]): + if not 'I' in pkgstr.location: return - if (QUERY_OPTS["includePortTree"] and - not QUERY_OPTS["includeOverlayTree"]): - if not 'P' in fmt_pkg.location: + if (QUERY_OPTS["in_porttree"] and + not QUERY_OPTS["in_overlay"]): + if not 'P' in pkgstr.location: return - if (QUERY_OPTS["includeOverlayTree"] and - not QUERY_OPTS["includePortTree"]): - if not 'O' in fmt_pkg.location: + if (QUERY_OPTS["in_overlay"] and + not QUERY_OPTS["in_porttree"]): + if not 'O' in pkgstr.location: return - print fmt_pkg + print(pkgstr) @@ -100,31 +113,34 @@ def parse_module_options(module_opts): # Parse module options opts = (x[0] for x in module_opts) - for opt in opts: + posargs = (x[1] for x in module_opts) + for opt, posarg in zip(opts, posargs): if opt in ('-h', '--help'): print_help() sys.exit(0) elif opt in ('-I', '--exclue-installed'): - QUERY_OPTS['includeInstalled'] = False + QUERY_OPTS['in_installed'] = False elif opt in ('-p', '--portage-tree'): - QUERY_OPTS['includePortTree'] = True + QUERY_OPTS['in_porttree'] = True elif opt in ('-o', '--overlay-tree'): - QUERY_OPTS['includeOverlayTree'] = True + QUERY_OPTS['in_overlay'] = True + elif opt in ('-F', '--format'): + QUERY_OPTS["package_format"] = posarg def main(input_args): """Parse input and run the program""" - short_opts = "hiIpo" # -i was option for default action + short_opts = "hiIpoF:" # -i was option for default action # --installed is no longer needed, kept for compatibility (djanderson '09) long_opts = ('help', 'installed', 'exclude-installed', 'portage-tree', - 'overlay-tree') + 'overlay-tree', 'format=') try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -134,7 +150,7 @@ def main(input_args): print_help() sys.exit(2) - matches = do_lookup("*", QUERY_OPTS) + matches = Query("*").smart_find(**QUERY_OPTS) matches.sort() # @@ -144,10 +160,10 @@ def main(input_args): first_run = True for query in queries: if not first_run: - print + print() if CONFIG['verbose']: - print " * Searching for USE flag %s ... " % pp.emph(query) + print(" * Searching for USE flag %s ... " % pp.emph(query)) for pkg in matches: display_useflags(query, pkg) diff --git a/pym/gentoolkit/equery/list_.py b/pym/gentoolkit/equery/list_.py index 3de8355..32c7dc3 100644 --- a/pym/gentoolkit/equery/list_.py +++ b/pym/gentoolkit/equery/list_.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 or higher # @@ -6,6 +6,8 @@ """List installed packages matching the query pattern""" +from __future__ import print_function + __docformat__ = 'epytext' # ======= @@ -18,8 +20,9 @@ from getopt import gnu_getopt, GetoptError import gentoolkit import gentoolkit.pprinter as pp from gentoolkit.equery import format_options, mod_usage, CONFIG -from gentoolkit.helpers import do_lookup, get_installed_cpvs -from gentoolkit.package import Package, PackageFormatter +from gentoolkit.helpers import get_installed_cpvs +from gentoolkit.package import PackageFormatter, FORMAT_TMPL_VARS +from gentoolkit.query import Query # ======= # Globals @@ -27,13 +30,13 @@ from gentoolkit.package import Package, PackageFormatter QUERY_OPTS = { "duplicates": False, - "includeInstalled": True, - "includePortTree": False, - "includeOverlayTree": False, - "includeMasked": True, - "includeMaskReason": False, - "isRegex": False, - "printMatchInfo": (not CONFIG['quiet']) + "in_installed": True, + "in_porttree": False, + "in_overlay": False, + "include_mask_reason": False, + "is_regex": False, + "show_progress": (not CONFIG['quiet']), + "package_format": None } # ========= @@ -48,8 +51,8 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print + print(__doc__.strip()) + print() # Deprecation warning added by djanderson, 12/2008 depwarning = ( @@ -60,12 +63,12 @@ def print_help(with_description=True): ) for line in depwarning: sys.stderr.write(pp.warn(line)) - print + print() - print mod_usage(mod_name="list") - print - print pp.command("options") - print format_options(( + print(mod_usage(mod_name="list")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -d, --duplicates", "list only installed duplicate packages"), (" -f, --full-regex", "query is a regular expression"), @@ -73,8 +76,12 @@ def print_help(with_description=True): (" -I, --exclude-installed", "exclude installed packages from output"), (" -o, --overlay-tree", "list packages in overlays"), - (" -p, --portage-tree", "list packages in the main portage tree") - )) + (" -p, --portage-tree", "list packages in the main portage tree"), + (" -F, --format=TMPL", "specify a custom output format"), + (" TMPL", + "a format template using (see man page):") + ))) + print(" " * 24, ', '.join(pp.emph(x) for x in FORMAT_TMPL_VARS)) def get_duplicates(matches): @@ -83,10 +90,10 @@ def get_duplicates(matches): dups = {} result = [] for pkg in matches: - if pkg.cpv.cp in dups: - dups[pkg.cpv.cp].append(pkg) + if pkg.cp in dups: + dups[pkg.cp].append(pkg) else: - dups[pkg.cpv.cp] = [pkg] + dups[pkg.cp] = [pkg] for cpv in dups.values(): if len(cpv) > 1: @@ -105,43 +112,45 @@ def parse_module_options(module_opts): print_help() sys.exit(0) elif opt in ('-I', '--exclude-installed'): - QUERY_OPTS['includeInstalled'] = False + QUERY_OPTS['in_installed'] = False elif opt in ('-p', '--portage-tree'): - QUERY_OPTS['includePortTree'] = True + QUERY_OPTS['in_porttree'] = True elif opt in ('-o', '--overlay-tree'): - QUERY_OPTS['includeOverlayTree'] = True + QUERY_OPTS['in_overlay'] = True elif opt in ('-f', '--full-regex'): - QUERY_OPTS['isRegex'] = True + QUERY_OPTS['is_regex'] = True elif opt in ('-m', '--mask-reason'): - QUERY_OPTS['includeMaskReason'] = True + QUERY_OPTS['include_mask_reason'] = True elif opt in ('-e', '--exact-name'): sys.stderr.write(pp.warn("-e, --exact-name is now default.")) sys.stderr.write( pp.warn("Use globbing to simulate the old behavior.") ) - print + print() elif opt in ('-d', '--duplicates'): QUERY_OPTS['duplicates'] = True + elif opt in ('-F', '--format'): + QUERY_OPTS["package_format"] = posarg def main(input_args): """Parse input and run the program""" - short_opts = "hdefiImop" # -i, -e were options for default actions + short_opts = "hdefiImopF:" # -i, -e were options for default actions # 04/09: djanderson # --all is no longer needed. Kept for compatibility. # --installed is no longer needed. Kept for compatibility. # --exact-name is no longer needed. Kept for compatibility. long_opts = ('help', 'all', 'installed', 'exclude-installed', - 'mask-reason', 'portage-tree', 'overlay-tree', 'full-regex', 'exact-name', - 'duplicates') + 'mask-reason', 'portage-tree', 'overlay-tree', 'format=', 'full-regex', + 'exact-name', 'duplicates') try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -149,21 +158,21 @@ def main(input_args): # Only search installed packages when listing duplicate packages if QUERY_OPTS["duplicates"]: - QUERY_OPTS["includeInstalled"] = True - QUERY_OPTS["includePortTree"] = False - QUERY_OPTS["includeOverlayTree"] = False - QUERY_OPTS["includeMaskReason"] = False + QUERY_OPTS["in_installed"] = True + QUERY_OPTS["in_porttree"] = False + QUERY_OPTS["in_overlay"] = False + QUERY_OPTS["include_mask_reason"] = False if not queries: print_help() sys.exit(2) first_run = True - for query in queries: + for query in (Query(x, QUERY_OPTS['is_regex']) for x in queries): if not first_run: - print + print() - matches = do_lookup(query, QUERY_OPTS) + matches = query.smart_find(**QUERY_OPTS) # Find duplicate packages if QUERY_OPTS["duplicates"]: @@ -177,26 +186,34 @@ def main(input_args): for pkg in matches: if CONFIG['verbose']: - pkgstr = PackageFormatter(pkg, do_format=True) + pkgstr = PackageFormatter( + pkg, + do_format=True, + custom_format=QUERY_OPTS["package_format"] + ) else: - pkgstr = PackageFormatter(pkg, do_format=False) - - if (QUERY_OPTS["includeInstalled"] and - not QUERY_OPTS["includePortTree"] and - not QUERY_OPTS["includeOverlayTree"]): + pkgstr = PackageFormatter( + pkg, + do_format=False, + custom_format=QUERY_OPTS["package_format"] + ) + + if (QUERY_OPTS["in_installed"] and + not QUERY_OPTS["in_porttree"] and + not QUERY_OPTS["in_overlay"]): if not 'I' in pkgstr.location: continue - if (QUERY_OPTS["includePortTree"] and - not QUERY_OPTS["includeOverlayTree"]): + if (QUERY_OPTS["in_porttree"] and + not QUERY_OPTS["in_overlay"]): if not 'P' in pkgstr.location: continue - if (QUERY_OPTS["includeOverlayTree"] and - not QUERY_OPTS["includePortTree"]): + if (QUERY_OPTS["in_overlay"] and + not QUERY_OPTS["in_porttree"]): if not 'O' in pkgstr.location: continue - print pkgstr + print(pkgstr) - if QUERY_OPTS["includeMaskReason"]: + if QUERY_OPTS["include_mask_reason"]: ms_int, ms_orig = pkgstr.format_mask_status() if not ms_int > 2: # ms_int is a number representation of mask level. @@ -207,17 +224,17 @@ def main(input_args): # Package not on system or not masked continue elif not any(mask_reason): - print " * No mask reason given" + print(" * No mask reason given") else: status = ', '.join(ms_orig) explanation = mask_reason[0] mask_location = mask_reason[1] - print " * Masked by %r" % status - print " * %s:" % mask_location - print '\n'.join( + print(" * Masked by %r" % status) + print(" * %s:" % mask_location) + print('\n'.join( [' * %s' % line.lstrip(' #') for line in explanation.splitlines()] - ) + )) first_run = False diff --git a/pym/gentoolkit/equery/meta.py b/pym/gentoolkit/equery/meta.py index 19c23a6..e1ca794 100644 --- a/pym/gentoolkit/equery/meta.py +++ b/pym/gentoolkit/equery/meta.py @@ -1,13 +1,12 @@ -# Copyright 2009-2010 Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 or higher # # $Header: $ -"""Display metadata about a given package""" +"""Display metadata about a given package.""" -# Move to Imports section after Python-2.6 is stable -from __future__ import with_statement +from __future__ import print_function __docformat__ = 'epytext' @@ -15,16 +14,18 @@ __docformat__ = 'epytext' # Imports # ======= -import os import re import sys from getopt import gnu_getopt, GetoptError +from portage import os + import gentoolkit.pprinter as pp from gentoolkit import errors from gentoolkit.equery import format_options, mod_usage, CONFIG -from gentoolkit.helpers import find_packages, print_sequence, print_file +from gentoolkit.helpers import print_sequence, print_file from gentoolkit.textwrap_ import TextWrapper +from gentoolkit.query import Query # ======= # Globals @@ -57,13 +58,13 @@ def print_help(with_description=True, with_usage=True): """ if with_description: - print __doc__.strip() - print + print(__doc__.strip()) + print() if with_usage: - print mod_usage(mod_name="meta") - print - print pp.command("options") - print format_options(( + print(mod_usage(mod_name="meta")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -d, --description", "show an extended package description"), (" -H, --herd", "show the herd(s) for the package"), @@ -72,7 +73,7 @@ def print_help(with_description=True, with_usage=True): (" -u, --useflags", "show per-package USE flag descriptions"), (" -U, --upstream", "show package's upstream information"), (" -x, --xml", "show the plain metadata.xml file") - )) + ))) def filter_keywords(matches): @@ -244,15 +245,12 @@ def format_keywords_line(pkg, fmtd_keywords, slot, verstr_len): # R0912: *Too many branches (%s/%s)* # pylint: disable-msg=R0912 -def call_format_functions(matches): +def call_format_functions(best_match, matches): """Call information gathering functions and display the results.""" - # Choose a good package to reference metadata from - ref_pkg = get_reference_pkg(matches) - if CONFIG['verbose']: - repo = ref_pkg.repo_name() - print " * %s [%s]" % (pp.cpv(ref_pkg.cp), pp.section(repo)) + repo = best_match.repo_name() + print(" * %s [%s]" % (pp.cpv(best_match.cp), pp.section(repo))) got_opts = False if any(QUERY_OPTS.values()): @@ -260,26 +258,26 @@ def call_format_functions(matches): got_opts = True if QUERY_OPTS["herd"] or not got_opts: - herds = format_herds(ref_pkg.metadata.herds(include_email=True)) + herds = format_herds(best_match.metadata.herds(include_email=True)) if QUERY_OPTS["herd"]: print_sequence(format_list(herds)) else: for herd in herds: - print format_line(herd, "Herd: ", " " * 13) + print(format_line(herd, "Herd: ", " " * 13)) if QUERY_OPTS["maintainer"] or not got_opts: - maints = format_maintainers(ref_pkg.metadata.maintainers()) + maints = format_maintainers(best_match.metadata.maintainers()) if QUERY_OPTS["maintainer"]: print_sequence(format_list(maints)) else: if not maints: - print format_line([], "Maintainer: ", " " * 13) + print(format_line([], "Maintainer: ", " " * 13)) else: for maint in maints: - print format_line(maint, "Maintainer: ", " " * 13) + print(format_line(maint, "Maintainer: ", " " * 13)) if QUERY_OPTS["upstream"] or not got_opts: - upstream = format_upstream(ref_pkg.metadata.upstream()) + upstream = format_upstream(best_match.metadata.upstream()) if QUERY_OPTS["upstream"]: upstream = format_list(upstream) else: @@ -287,8 +285,8 @@ def call_format_functions(matches): print_sequence(upstream) if not got_opts: - pkg_loc = ref_pkg.package_path() - print format_line(pkg_loc, "Location: ", " " * 13) + pkg_loc = best_match.package_path() + print(format_line(pkg_loc, "Location: ", " " * 13)) if QUERY_OPTS["keywords"] or not got_opts: # Get {<Package 'dev-libs/glib-2.20.5'>: [u'ia64', u'm68k', ...], ...} @@ -302,21 +300,21 @@ def call_format_functions(matches): match, fmtd_keywords, slot, verstr_len ) if QUERY_OPTS["keywords"]: - print keywords_line + print(keywords_line) else: indent = " " * (16 + verstr_len) - print format_line(keywords_line, "Keywords: ", indent) + print(format_line(keywords_line, "Keywords: ", indent)) if QUERY_OPTS["description"]: - desc = ref_pkg.metadata.descriptions() + desc = best_match.metadata.descriptions() print_sequence(format_list(desc)) if QUERY_OPTS["useflags"]: - useflags = format_useflags(ref_pkg.metadata.use()) + useflags = format_useflags(best_match.metadata.use()) print_sequence(format_list(useflags)) if QUERY_OPTS["xml"]: - print_file(os.path.join(ref_pkg.package_path(), 'metadata.xml')) + print_file(os.path.join(best_match.package_path(), 'metadata.xml')) def format_line(line, first="", subsequent="", force_quiet=False): @@ -418,19 +416,6 @@ def format_list(lst, first="", subsequent="", force_quiet=False): return result -def get_reference_pkg(matches): - """Find a package in the Portage tree to reference.""" - - pkg = None - rev_matches = list(reversed(matches)) - while rev_matches: - pkg = rev_matches.pop() - if not pkg.is_overlay(): - break - - return pkg - - def parse_module_options(module_opts): """Parse module options and update QUERY_OPTS""" @@ -464,9 +449,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -478,16 +463,17 @@ def main(input_args): sys.exit(2) first_run = True - for query in queries: - matches = find_packages(query, include_masked=True) + for query in (Query(x) for x in queries): + best_match = query.find_best() + matches = query.find(include_masked=True) if not matches: raise errors.GentoolkitNoMatches(query) if not first_run: - print + print() matches.sort() - call_format_functions(matches) + call_format_functions(best_match, matches) first_run = False diff --git a/pym/gentoolkit/equery/size.py b/pym/gentoolkit/equery/size.py index 4d2a686..27aa8ec 100644 --- a/pym/gentoolkit/equery/size.py +++ b/pym/gentoolkit/equery/size.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -6,6 +6,8 @@ """Print total size of files contained in a given package""" +from __future__ import print_function + __docformat__ = 'epytext' # ======= @@ -17,21 +19,20 @@ from getopt import gnu_getopt, GetoptError import gentoolkit.pprinter as pp from gentoolkit.equery import format_options, mod_usage, CONFIG -from gentoolkit.helpers import do_lookup +from gentoolkit.query import Query # ======= # Globals # ======= QUERY_OPTS = { - "includeInstalled": True, - "includePortTree": False, - "includeOverlayTree": False, - "includeMasked": True, - "isRegex": False, - "matchExact": False, - "printMatchInfo": False, - "sizeInBytes": False + "in_installed": True, + "in_porttree": False, + "in_overlay": False, + "include_masked": True, + "is_regex": False, + "show_progress": False, + "size_in_bytes": False } # ========= @@ -46,8 +47,8 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print + print(__doc__.strip()) + print() # Deprecation warning added by djanderson, 12/2008 depwarning = ( @@ -58,16 +59,16 @@ def print_help(with_description=True): ) for line in depwarning: sys.stderr.write(pp.warn(line)) - print + print() - print mod_usage(mod_name="size") - print - print pp.command("options") - print format_options(( + print(mod_usage(mod_name="size")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -b, --bytes", "report size in bytes"), (" -f, --full-regex", "query is a regular expression") - )) + ))) def display_size(match_set): @@ -81,22 +82,22 @@ def display_size(match_set): size, files, uncounted = pkg.size() if CONFIG['verbose']: - print " * %s" % pp.cpv(str(pkg.cpv)) - print "Total files : %s".rjust(25) % pp.number(str(files)) + print(" * %s" % pp.cpv(str(pkg.cpv))) + print("Total files : %s".rjust(25) % pp.number(str(files))) if uncounted: - print ("Inaccessible files : %s".rjust(25) % - pp.number(str(uncounted))) + print(("Inaccessible files : %s".rjust(25) % + pp.number(str(uncounted)))) - if QUERY_OPTS["sizeInBytes"]: + if QUERY_OPTS["size_in_bytes"]: size_str = pp.number(str(size)) else: size_str = "%s %s" % format_bytes(size) - print "Total size : %s".rjust(25) % size_str + print("Total size : %s".rjust(25) % size_str) else: info = "%s: total(%d), inaccessible(%d), size(%s)" - print info % (str(pkg.cpv), files, uncounted, size) + print(info % (str(pkg.cpv), files, uncounted, size)) def format_bytes(bytes_, precision=2): @@ -108,10 +109,10 @@ def format_bytes(bytes_, precision=2): """ labels = ( - (1<<40L, 'TiB'), - (1<<30L, 'GiB'), - (1<<20L, 'MiB'), - (1<<10L, 'KiB'), + (1<<40, 'TiB'), + (1<<30, 'GiB'), + (1<<20, 'MiB'), + (1<<10, 'KiB'), (1, 'bytes') ) @@ -144,14 +145,14 @@ def parse_module_options(module_opts): print_help() sys.exit(0) elif opt in ('-b', '--bytes'): - QUERY_OPTS["sizeInBytes"] = True + QUERY_OPTS["size_in_bytes"] = True elif opt in ('-e', '--exact-name'): sys.stderr.write(pp.warn("-e, --exact-name is now default.")) warning = pp.warn("Use globbing to simulate the old behavior.") sys.stderr.write(warning) - print + print() elif opt in ('-f', '--full-regex'): - QUERY_OPTS['isRegex'] = True + QUERY_OPTS['is_regex'] = True def main(input_args): @@ -164,9 +165,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -177,15 +178,17 @@ def main(input_args): sys.exit(2) first_run = True - for query in queries: + for query in (Query(x, QUERY_OPTS['is_regex']) for x in queries): if not first_run: - print + print() - matches = do_lookup(query, QUERY_OPTS) + matches = query.smart_find(**QUERY_OPTS) if not matches: sys.stderr.write(pp.error("No package found matching %s" % query)) + matches.sort() + display_size(matches) first_run = False diff --git a/pym/gentoolkit/equery/uses.py b/pym/gentoolkit/equery/uses.py index 8358d49..19aab6e 100644 --- a/pym/gentoolkit/equery/uses.py +++ b/pym/gentoolkit/equery/uses.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -6,8 +6,10 @@ """Display USE flags for a given package""" +from __future__ import print_function + # Move to imports section when Python 2.6 is stable -from __future__ import with_statement + __docformat__ = 'epytext' @@ -15,19 +17,19 @@ __docformat__ = 'epytext' # Imports # ======= -import os import sys from functools import partial from getopt import gnu_getopt, GetoptError from glob import glob -from portage import settings +from portage import os, settings import gentoolkit.pprinter as pp from gentoolkit import errors from gentoolkit.equery import format_options, mod_usage, CONFIG -from gentoolkit.helpers import find_best_match, find_packages, uniqify +from gentoolkit.helpers import uniqify from gentoolkit.textwrap_ import TextWrapper +from gentoolkit.query import Query # ======= # Globals @@ -47,15 +49,15 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print - print mod_usage(mod_name=__name__.split('.')[-1]) - print - print pp.command("options") - print format_options(( + print(__doc__.strip()) + print() + print(mod_usage(mod_name=__name__.split('.')[-1])) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -a, --all", "include all package versions") - )) + ))) def display_useflags(output): @@ -98,22 +100,22 @@ def display_useflags(output): restrict = "(%s %s)" % (pp.emph("Restricted to"), pp.cpv(restrict)) twrap.initial_indent = flag_name - print twrap.fill(restrict) + print(twrap.fill(restrict)) if desc: twrap.initial_indent = twrap.subsequent_indent - print twrap.fill(desc) + print(twrap.fill(desc)) else: - print " : <unknown>" + print(" : <unknown>") else: if desc: twrap.initial_indent = flag_name desc = twrap.fill(desc) - print desc + print(desc) else: twrap.initial_indent = flag_name - print twrap.fill("<unknown>") + print(twrap.fill("<unknown>")) else: - print markers[in_makeconf] + flag + print(markers[in_makeconf] + flag) def get_global_useflags(): @@ -165,24 +167,6 @@ def get_global_useflags(): return global_usedesc -def get_matches(query): - """Get packages matching query.""" - - if not QUERY_OPTS["allVersions"]: - matches = [find_best_match(query)] - if None in matches: - matches = find_packages(query, include_masked=False) - if matches: - matches.sort() - else: - matches = find_packages(query, include_masked=True) - - if not matches: - raise errors.GentoolkitNoMatches(query) - - return matches - - def get_output_descriptions(pkg, global_usedesc): """Prepare descriptions and usage information for each USE flag.""" @@ -251,10 +235,10 @@ def parse_module_options(module_opts): def print_legend(): """Print a legend to explain the output format.""" - print "[ Legend : %s - flag is set in make.conf ]" % pp.emph("U") - print "[ : %s - package is installed with flag ]" % pp.emph("I") - print "[ Colors : %s, %s ]" % ( - pp.useflag("set", enabled=True), pp.useflag("unset", enabled=False)) + print("[ Legend : %s - flag is set in make.conf ]" % pp.emph("U")) + print("[ : %s - package is installed with flag ]" % pp.emph("I")) + print("[ Colors : %s, %s ]" % ( + pp.useflag("set", enabled=True), pp.useflag("unset", enabled=False))) def main(input_args): @@ -265,9 +249,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -283,14 +267,18 @@ def main(input_args): first_run = True legend_printed = False - for query in queries: + for query in (Query(x) for x in queries): if not first_run: - print + print() - if CONFIG['verbose']: - print " * Searching for %s ..." % pp.pkgquery(query) + if QUERY_OPTS["allVersions"]: + matches = query.find(include_masked=True) + else: + matches = [query.find_best()] + + if not matches: + raise errors.GentoolkitNoMatches(query) - matches = get_matches(query) matches.sort() global_usedesc = get_global_useflags() @@ -302,9 +290,9 @@ def main(input_args): if not legend_printed: print_legend() legend_printed = True - print (" * Found these USE flags for %s:" % - pp.cpv(str(pkg.cpv))) - print pp.emph(" U I") + print((" * Found these USE flags for %s:" % + pp.cpv(str(pkg.cpv)))) + print(pp.emph(" U I")) display_useflags(output) else: if CONFIG['verbose']: diff --git a/pym/gentoolkit/equery/which.py b/pym/gentoolkit/equery/which.py index be4f5e8..104d057 100644 --- a/pym/gentoolkit/equery/which.py +++ b/pym/gentoolkit/equery/which.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -8,26 +8,29 @@ configuration """ +from __future__ import print_function + __docformat__ = 'epytext' # ======= # Imports # ======= -import os import sys from getopt import gnu_getopt, GetoptError +from portage import os + import gentoolkit.pprinter as pp from gentoolkit import errors from gentoolkit.equery import format_options, mod_usage -from gentoolkit.helpers import find_packages +from gentoolkit.query import Query # ======= # Globals # ======= -QUERY_OPTS = {"includeMasked": False} +QUERY_OPTS = {"include_masked": False} # ========= # Functions @@ -41,15 +44,15 @@ def print_help(with_description=True): """ if with_description: - print __doc__.strip() - print - print mod_usage(mod_name="which") - print - print pp.command("options") - print format_options(( + print(__doc__.strip()) + print() + print(mod_usage(mod_name="which")) + print() + print(pp.command("options")) + print(format_options(( (" -h, --help", "display this help message"), (" -m, --include-masked", "return highest version ebuild available") - )) + ))) def parse_module_options(module_opts): @@ -61,7 +64,7 @@ def parse_module_options(module_opts): print_help() sys.exit(0) elif opt in ('-m', '--include-masked'): - QUERY_OPTS['includeMasked'] = True + QUERY_OPTS['include_masked'] = True def main(input_args): @@ -72,9 +75,9 @@ def main(input_args): try: module_opts, queries = gnu_getopt(input_args, short_opts, long_opts) - except GetoptError, err: + except GetoptError as err: sys.stderr.write(pp.error("Module %s" % err)) - print + print() print_help(with_description=False) sys.exit(2) @@ -84,17 +87,19 @@ def main(input_args): print_help() sys.exit(2) - for query in queries: - - matches = find_packages(query, QUERY_OPTS['includeMasked']) + for query in (Query(x) for x in queries): + matches = query.find( + include_masked=QUERY_OPTS['include_masked'], + in_installed=False + ) if matches: pkg = sorted(matches).pop() ebuild_path = pkg.ebuild_path() if ebuild_path: - print os.path.normpath(ebuild_path) + print(os.path.normpath(ebuild_path)) else: sys.stderr.write( - pp.warn("No ebuilds to satisfy %s" % pkg.cpv.name) + pp.warn("No ebuilds to satisfy %s" % pkg.cpv) ) else: raise errors.GentoolkitNoMatches(query) diff --git a/pym/gentoolkit/errors.py b/pym/gentoolkit/errors.py index 9bcc5f9..c5a88de 100644 --- a/pym/gentoolkit/errors.py +++ b/pym/gentoolkit/errors.py @@ -1,4 +1,4 @@ -# Copyright(c) 2004-2010, Gentoo Foundation +# Copyright(c) 2004-2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 or later @@ -14,7 +14,9 @@ __all__ = ( 'GentoolkitInvalidCPV', 'GentoolkitInvalidRegex', 'GentoolkitInvalidVersion', - 'GentoolkitNoMatches' + 'GentoolkitNoMatches', + 'GentoolkitSetNotFound', + 'GentoolkitUnknownKeyword' ) # ========== @@ -55,6 +57,15 @@ class GentoolkitInvalidAtom(GentoolkitException): return "Invalid atom: '%s'" % self.atom +class GentoolkitSetNotFound(GentoolkitException): + """Got unknown set.""" + def __init__(self, setname): + self.setname = setname + + def __str__(self): + return "Unknown set: '%s'" % self.setname + + class GentoolkitInvalidCategory(GentoolkitException): """The category was not listed in portage.settings.categories.""" def __init__(self, category): @@ -111,4 +122,16 @@ class GentoolkitNoMatches(GentoolkitException): return "No %spackages matching '%s'" % (inst, self.query) +class GentoolkitUnknownKeyword(GentoolkitException): + """No packages were found matching the search query.""" + def __init__(self, query, keywords, use): + self.query = query + self.keywords = keywords + self.use = use + + def __str__(self): + return ("Unable to determine the install keyword for:\n" + + "'%s', KEYWORDS = '%s'\nUSE flags = '%s'" + % (self.query, self.keywords, self.use)) + # vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/formatters.py b/pym/gentoolkit/formatters.py new file mode 100644 index 0000000..f79fa72 --- /dev/null +++ b/pym/gentoolkit/formatters.py @@ -0,0 +1,101 @@ +#!/usr/bin/python +# +# Copyright 2004 Karl Trygve Kalleberg <karltk@gentoo.org> +# Copyright(c) 2010, Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# +# $Header$ + +import errno +import sys +import time + +from portage import os + +import gentoolkit +from gentoolkit.textwrap_ import TextWrapper +import gentoolkit.pprinter as pp + + +def format_options(options): + """Format module options. + + @type options: list + @param options: [('option 1', 'description 1'), ('option 2', 'des... )] + @rtype: str + @return: formatted options string + """ + + result = [] + twrap = TextWrapper(width=gentoolkit.CONFIG['termWidth']) + opts = (x[0] for x in options) + descs = (x[1] for x in options) + for opt, desc in zip(opts, descs): + twrap.initial_indent = pp.emph(opt.ljust(25)) + twrap.subsequent_indent = " " * 25 + result.append(twrap.fill(desc)) + return '\n'.join(result) + + +def format_filetype(path, fdesc, show_type=False, show_md5=False, + show_timestamp=False): + """Format a path for printing. + + @type path: str + @param path: the path + @type fdesc: list + @param fdesc: [file_type, timestamp, MD5 sum/symlink target] + file_type is one of dev, dir, obj, sym. + If file_type is dir, there is no timestamp or MD5 sum. + If file_type is sym, fdesc[2] is the target of the symlink. + @type show_type: bool + @param show_type: if True, prepend the file's type to the formatted string + @type show_md5: bool + @param show_md5: if True, append MD5 sum to the formatted string + @type show_timestamp: bool + @param show_timestamp: if True, append time-of-creation after pathname + @rtype: str + @return: formatted pathname with optional added information + """ + + ftype = fpath = stamp = md5sum = "" + if fdesc[0] == "obj": + ftype = "file" + fpath = path + stamp = format_timestamp(fdesc[1]) + md5sum = fdesc[2] + elif fdesc[0] == "dir": + ftype = "dir" + fpath = pp.path(path) + elif fdesc[0] == "sym": + ftype = "sym" + stamp = format_timestamp(fdesc[1]) + tgt = fdesc[2].split()[0] + if CONFIG["piping"]: + fpath = path + else: + fpath = pp.path_symlink(path + " -> " + tgt) + elif fdesc[0] == "dev": + ftype = "dev" + fpath = path + else: + sys.stderr.write( + pp.error("%s has unknown type: %s" % (path, fdesc[0])) + ) + result = "" + if show_type: + result += "%4s " % ftype + result += fpath + if show_timestamp: + result += " " + stamp + if show_md5: + result += " " + md5sum + return result + + + +def format_timestamp(timestamp): + """Format a timestamp into, e.g., '2009-01-31 21:19:44' format""" + + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(timestamp))) + diff --git a/pym/gentoolkit/glsa/__init__.py b/pym/gentoolkit/glsa/__init__.py index 11b7dbe..ab8dcdd 100644 --- a/pym/gentoolkit/glsa/__init__.py +++ b/pym/gentoolkit/glsa/__init__.py @@ -11,16 +11,22 @@ # - getting GLSAs from http/ftp servers (not really useful without the fixed ebuilds) # - GPG signing/verification (until key policy is clear) +from __future__ import unicode_literals + __author__ = "Marius Mauch <genone@gentoo.org>" -import os + import sys -import urllib +try: + from urllib import urlopen +except ImportError: + from urllib.request import urlopen import codecs import re import operator import xml.dom.minidom -from StringIO import StringIO +from io import StringIO +from functools import reduce if sys.version_info[0:2] < (2,3): raise NotImplementedError("Python versions below 2.3 have broken XML code " \ @@ -32,6 +38,8 @@ except ImportError: sys.path.insert(0, "/usr/lib/portage/pym") import portage +from portage import os + # Note: the space for rgt and rlt is important !! opMapping = {"le": "<=", "lt": "<", "eq": "=", "gt": ">", "ge": ">=", "rge": ">=~", "rle": "<=~", "rgt": " >~", "rlt": " <~"} @@ -512,7 +520,7 @@ class Glsa: myurl = "file://"+self.nr else: myurl = repository + self.config["GLSA_PREFIX"] + str(self.nr) + self.config["GLSA_SUFFIX"] - self.parse(urllib.urlopen(myurl)) + self.parse(urlopen(myurl)) return None def parse(self, myfile): @@ -544,16 +552,17 @@ class Glsa: self.synopsis = getText(myroot.getElementsByTagName("synopsis")[0], format="strip") self.announced = format_date(getText(myroot.getElementsByTagName("announced")[0], format="strip")) - count = 1 # Support both formats of revised: # <revised>December 30, 2007: 02</revised> # <revised count="2">2007-12-30</revised> revisedEl = myroot.getElementsByTagName("revised")[0] self.revised = getText(revisedEl, format="strip") - if (revisedEl.attributes.has_key("count")): - count = revisedEl.getAttribute("count") - elif (self.revised.find(":") >= 0): - (self.revised, count) = self.revised.split(":") + count = revisedEl.attributes.get("count") + if count is None: + if self.revised.find(":") >= 0: + (self.revised, count) = self.revised.split(":") + else: + count = 1 self.revised = format_date(self.revised) @@ -589,7 +598,7 @@ class Glsa: self.packages = {} for p in self.affected.getElementsByTagName("package"): name = p.getAttribute("name") - if not self.packages.has_key(name): + if name not in self.packages: self.packages[name] = [] tmp = {} tmp["arch"] = p.getAttribute("arch") @@ -638,15 +647,15 @@ class Glsa: pass if len(self.bugs) > 0: outstream.write("\nRelated bugs: ") - outstream.write(", ".join(self.bugs)) - outstream.write("\n") + outstream.write(", ".join(self.bugs)) + outstream.write("\n") if self.background: outstream.write("\n"+wrap(self.background, width, caption="Background: ")) outstream.write("\n"+wrap(self.description, width, caption="Description: ")) outstream.write("\n"+wrap(self.impact_text, width, caption="Impact: ")) outstream.write("\n"+wrap(self.workaround, width, caption="Workaround: ")) outstream.write("\n"+wrap(self.resolution, width, caption="Resolution: ")) - myreferences = " ".join(r.replace(" ", SPACE_ESCAPE)+NEWLINE_ESCAPE for r in self.references) + myreferences = " ".join(r.replace(" ", SPACE_ESCAPE)+NEWLINE_ESCAPE for r in self.references) outstream.write("\n"+wrap(myreferences, width, caption="References: ")) outstream.write("\n") diff --git a/pym/gentoolkit/helpers.py b/pym/gentoolkit/helpers.py index 1f25666..0aad2a7 100644 --- a/pym/gentoolkit/helpers.py +++ b/pym/gentoolkit/helpers.py @@ -1,35 +1,25 @@ -# Copyright 2009-2010 Gentoo Foundation +# Copyright(c) 2009-2010, Gentoo Foundation # # Licensed under the GNU General Public License, v2 or higher # # $Header$ -"""Improved versions of the original helpers functions. +"""Miscellaneous helper functions and classes. -As a convention, functions ending in '_packages' or '_match{es}' return -Package objects, while functions ending in 'cpvs' return a sequence of strings. -Functions starting with 'get_' return a set of packages by default and can be -filtered, while functions starting with 'find_' return nothing unless the -query matches one or more packages. +@note: find_* functions that previously lived here have moved to + the query module, where they are called as: Query('portage').find_*(). """ -# Move to Imports section after Python 2.6 is stable -from __future__ import with_statement +from __future__ import print_function __all__ = ( 'ChangeLog', 'FileOwner', - 'compare_package_strings', - 'do_lookup', - 'find_best_match', - 'find_installed_packages', - 'find_packages', 'get_cpvs', 'get_installed_cpvs', 'get_uninstalled_cpvs', 'uniqify', - 'uses_globbing', - 'split_cpv' + 'walk' ) __docformat__ = 'epytext' @@ -37,17 +27,13 @@ __docformat__ = 'epytext' # Imports # ======= -import fnmatch -import os import re from functools import partial from itertools import chain -import portage -from portage.versions import catpkgsplit, pkgcmp +from portage import os, _unicode_decode, _encodings from gentoolkit import pprinter as pp -from gentoolkit import CONFIG from gentoolkit import errors from gentoolkit.atom import Atom from gentoolkit.cpv import CPV @@ -164,6 +150,7 @@ class ChangeLog(object): if from_restriction and not from_restriction.match(i): continue if to_restriction and not to_restriction.match(i): + # TODO: is it safe to break here? continue result.append(entry) @@ -262,7 +249,7 @@ class FileOwner(object): query_re_string = self._prepare_search_regex(queries) try: query_re = re.compile(query_re_string) - except (TypeError, re.error), err: + except (TypeError, re.error) as err: raise errors.GentoolkitInvalidRegex(err) use_match = False @@ -312,6 +299,27 @@ class FileOwner(object): return results @staticmethod + def expand_abspaths(paths): + """Expand any relative paths (./file) to their absolute paths. + + @type paths: list + @param paths: file path strs + @rtype: list + @return: the original list with any relative paths expanded + @raise AttributeError: if paths does not have attribute 'extend' + """ + + osp = os.path + expanded_paths = [] + for p in paths: + if p.startswith('./'): + expanded_paths.append(osp.abspath(p)) + else: + expanded_paths.append(p) + + return expanded_paths + + @staticmethod def extend_realpaths(paths): """Extend a list of paths with the realpaths for any symlinks. @@ -339,6 +347,7 @@ class FileOwner(object): result = [] # Trim trailing and multiple slashes from queries slashes = re.compile('/+') + queries = self.expand_abspaths(queries) queries = self.extend_realpaths(queries) for query in queries: query = slashes.sub('/', query).rstrip('/') @@ -354,201 +363,6 @@ class FileOwner(object): # Functions # ========= -def compare_package_strings(pkg1, pkg2): - """Similar to the builtin cmp, but for package strings. Usually called - as: package_list.sort(compare_package_strings) - - An alternative is to use the CPV descriptor from gentoolkit.cpv: - >>> cpvs = sorted(CPV(x) for x in package_list) - - @see: >>> help(cmp) - """ - - pkg1 = catpkgsplit(pkg1) - pkg2 = catpkgsplit(pkg2) - if pkg1[0] != pkg2[0]: - return cmp(pkg1[0], pkg2[0]) - elif pkg1[1] != pkg2[1]: - return cmp(pkg1[1], pkg2[1]) - else: - return pkgcmp(pkg1[1:], pkg2[1:]) - - -def do_lookup(query, query_opts): - """A high-level wrapper around gentoolkit package-finder functions. - - @type query: str - @param query: pkg, cat/pkg, pkg-ver, cat/pkg-ver, atom, glob or regex - @type query_opts: dict - @param query_opts: user-configurable options from the calling module - Currently supported options are: - - includeInstalled = bool - includePortTree = bool - includeOverlayTree = bool - isRegex = bool - printMatchInfo = bool # Print info about the search - - @rtype: list - @return: Package objects matching query - """ - - if query_opts["includeInstalled"]: - if query_opts["includePortTree"] or query_opts["includeOverlayTree"]: - simple_package_finder = partial(find_packages, include_masked=True) - complex_package_finder = get_cpvs - else: - simple_package_finder = find_installed_packages - complex_package_finder = get_installed_cpvs - elif query_opts["includePortTree"] or query_opts["includeOverlayTree"]: - simple_package_finder = partial(find_packages, include_masked=True) - complex_package_finder = get_uninstalled_cpvs - else: - raise errors.GentoolkitFatalError( - "Not searching in installed, Portage tree, or overlay. " - "Nothing to do." - ) - - is_simple_query = True - if query_opts["isRegex"] or uses_globbing(query): - is_simple_query = False - - if is_simple_query: - matches = _do_simple_lookup(query, simple_package_finder, query_opts) - else: - matches = _do_complex_lookup(query, complex_package_finder, query_opts) - - return matches - - -def _do_complex_lookup(query, package_finder, query_opts): - """Find matches for a query which is a regex or includes globbing.""" - - # FIXME: Remove when lazyimport supports objects: - from gentoolkit.package import Package - - result = [] - - if query_opts["printMatchInfo"] and not CONFIG["piping"]: - print_query_info(query, query_opts) - - cat = split_cpv(query)[0] - - pre_filter = [] - # The "get_" functions can pre-filter against the whole package key, - # but since we allow globbing now, we run into issues like: - # >>> portage.dep.dep_getkey("sys-apps/portage-*") - # 'sys-apps/portage-' - # So the only way to guarantee we don't overrun the key is to - # prefilter by cat only. - if cat: - if query_opts["isRegex"]: - cat_re = cat - else: - cat_re = fnmatch.translate(cat) - # [::-1] reverses a sequence, so we're emulating an ".rreplace()" - # except we have to put our "new" string on backwards - cat_re = cat_re[::-1].replace('$', '*./', 1)[::-1] - predicate = lambda x: re.match(cat_re, x) - pre_filter = package_finder(predicate=predicate) - - # Post-filter - if query_opts["isRegex"]: - predicate = lambda x: re.search(query, x) - else: - if cat: - query_re = fnmatch.translate(query) - else: - query_re = fnmatch.translate("*/%s" % query) - predicate = lambda x: re.search(query_re, x) - if pre_filter: - result = [x for x in pre_filter if predicate(x)] - else: - result = package_finder(predicate=predicate) - - return [Package(x) for x in result] - - -def _do_simple_lookup(query, package_finder, query_opts): - """Find matches for a query which is an atom or string.""" - - result = [] - - if query_opts["printMatchInfo"] and CONFIG['verbose']: - print_query_info(query, query_opts) - - result = package_finder(query) - if not query_opts["includeInstalled"]: - result = [x for x in result if not x.is_installed()] - - return result - - -def find_best_match(query): - """Return the highest unmasked version of a package matching query. - - @type query: str - @param query: can be of the form: pkg, pkg-ver, cat/pkg, cat/pkg-ver, atom - @rtype: str or None - @raise portage.exception.InvalidAtom: if query is not valid input - """ - # FIXME: Remove when lazyimport supports objects: - from gentoolkit.package import Package - - try: - match = PORTDB.xmatch("bestmatch-visible", query) - except portage.exception.InvalidAtom, err: - raise errors.GentoolkitInvalidAtom(err) - - return Package(match) if match else None - - -def find_installed_packages(query): - """Return a list of Package objects that matched the search key.""" - # FIXME: Remove when lazyimport supports objects: - from gentoolkit.package import Package - - try: - matches = VARDB.match(query) - # catch the ambiguous package Exception - except portage.exception.AmbiguousPackageName, err: - matches = [] - for pkgkey in err[0]: - matches.extend(VARDB.match(pkgkey)) - except portage.exception.InvalidAtom, err: - raise errors.GentoolkitInvalidAtom(err) - - return [Package(x) for x in matches] - - -def find_packages(query, include_masked=False): - """Returns a list of Package objects that matched the query. - - @type query: str - @param query: can be of the form: pkg, pkg-ver, cat/pkg, cat/pkg-ver, atom - @type include_masked: bool - @param include_masked: include masked packages - @rtype: list - @return: matching Package objects - """ - # FIXME: Remove when lazyimport supports objects: - from gentoolkit.package import Package - - if not query: - return [] - - try: - if include_masked: - matches = PORTDB.xmatch("match-all", query) - else: - matches = PORTDB.match(query) - matches.extend(VARDB.match(query)) - except portage.exception.InvalidAtom, err: - raise errors.GentoolkitInvalidAtom(str(err)) - - return [Package(x) for x in set(matches)] - - def get_cpvs(predicate=None, include_installed=True): """Get all packages in the Portage tree and overlays. Optionally apply a predicate. @@ -578,16 +392,18 @@ def get_cpvs(predicate=None, include_installed=True): all_cps = PORTDB.cp_all() all_cpvs = chain.from_iterable(PORTDB.cp_list(x) for x in all_cps) - all_installed_cpvs = get_installed_cpvs(predicate) + all_installed_cpvs = set(get_installed_cpvs(predicate)) if include_installed: - for cpv in chain(all_cpvs, all_installed_cpvs): + for cpv in all_cpvs: + if cpv in all_installed_cpvs: + all_installed_cpvs.remove(cpv) + yield cpv + for cpv in all_installed_cpvs: yield cpv else: - # Consume the smaller pkg set: - installed_cpvs = set(all_installed_cpvs) for cpv in all_cpvs: - if cpv not in installed_cpvs: + if cpv not in all_installed_cpvs: yield cpv @@ -615,67 +431,19 @@ def get_installed_cpvs(predicate=None): yield cpv -def print_query_info(query, query_opts): - """Print info about the query to the screen.""" - - cat, pkg = split_cpv(query)[:2] - if cat and not query_opts["isRegex"]: - cat_str = "in %s " % pp.emph(cat.lstrip('><=~!')) - else: - cat_str = "" - - if query_opts["isRegex"]: - pkg_str = query - else: - pkg_str = pkg - - print " * Searching for %s %s..." % (pp.emph(pkg_str), cat_str) - - def print_file(path): """Display the contents of a file.""" with open(path) as open_file: lines = open_file.read() - print lines.strip() + print(lines.strip()) def print_sequence(seq): """Print every item of a sequence.""" for item in seq: - print item - - -def split_cpv(query): - """Split a cpv into category, name, version and revision. - - @type query: str - @param query: pkg, cat/pkg, pkg-ver, cat/pkg-ver, atom or regex - @rtype: tuple - @return: (category, pkg_name, version, revision) - Each tuple element is a string or empty string (""). - """ - - result = catpkgsplit(query) - - if result: - result = list(result) - if result[0] == 'null': - result[0] = '' - if result[3] == 'r0': - result[3] = '' - else: - result = query.split("/") - if len(result) == 1: - result = ['', query, '', ''] - else: - result = result + ['', ''] - - if len(result) != 4: - raise errors.GentoolkitInvalidPackageName(query) - - return tuple(result) + print(item) def uniqify(seq, preserve_order=True): @@ -690,20 +458,19 @@ def uniqify(seq, preserve_order=True): return result -def uses_globbing(query): - """Check the query to see if it is using globbing. - - @type query: str - @param query: user input package query - @rtype: bool - @return: True if query uses globbing, else False - """ - - if set('!*?[]').intersection(query): - # Is query an atom such as '=sys-apps/portage-2.2*'? - if query[0] != '=': - return True - - return False +def walk(top, topdown = True, onerror = None, followlinks = False): + """Variant of os.walk that always returns unicode filenames""" + for root, dirs, files in os.walk(top, topdown, onerror, followlinks): + root = _unicode_decode(root, _encodings["fs"], errors = "strict") + dirs = [ + _unicode_decode(x, _encodings["fs"], errors = "strict") + for x in dirs + ] + files = [ + _unicode_decode(x, _encodings["fs"], errors = "strict") + for x in files + ] + # XXX: in contrast with os.walk we ignore modifications to dirs here + yield root, dirs, files # vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/keyword.py b/pym/gentoolkit/keyword.py new file mode 100644 index 0000000..d40ab42 --- /dev/null +++ b/pym/gentoolkit/keyword.py @@ -0,0 +1,105 @@ +#!/usr/bin/python +# +# Copyright(c) 2004-2010, Gentoo Foundation +# +# Licensed under the GNU General Public License, v2 +# +# $Header$ + +"""Provides common methods on Gentoo GLEP 53 keywords. + +http://www.gentoo.org/proj/en/glep/glep-0053.html +""" + +__all__ = ( + 'Keyword', + 'compare_strs' +) + +# ======= +# Imports +# ======= + + +# ======= +# Classes +# ======= + +class Keyword(object): + """Provides common methods on a GLEP 53 keyword.""" + + def __init__(self, keyword): + self.keyword = keyword + + def __str__(self): + return self.keyword + + def __repr__(self): + return "<Keyword {0.keyword!r}>".format(self) + +# ========= +# Functions +# ========= + +def compare_strs(kw1, kw2): + """Similar to the builtin cmp, but for keyword strings. Usually called + as: keyword_list.sort(keyword.compare_strs) + + An alternative is to use the Keyword descriptor directly: + >>> kwds = sorted(Keyword(x) for x in keyword_list) + + @see: >>> help(cmp) + """ + + pass + + +def reduce_keywords(keywords): + """Reduce a list of keywords to a unique set of stable keywords. + + Example usage: + >>> reduce_keywords(['~amd64', 'x86', '~x86']) + set(['amd64', 'x86']) + + @type keywords: array + @rtype: set + """ + return set(x.lstrip('~') for x in keywords) + + +abs_keywords = reduce_keywords + + +# FIXME: this is unclear +# dj, how about 'deduce_keyword' +# I was trying to avoid a 2nd use of determine_keyword name (in analyse.lib) +# but that one is a little different and not suitable for this task. +def determine_keyword(arch, accepted, keywords): + """Determine a keyword from matching a dep's KEYWORDS + list against the ARCH & ACCEPT_KEYWORDS provided. + + @type arch: string + @param arch: portage.settings["ARCH"] + @type accepted: string + @param accepted: portage.settings["ACCEPT_KEYWORDS"] + @type keywords: string + @param keywords: the pkg ebuilds keywords + """ + if not keywords: + return '' + keys = keywords.split() + if arch in keys: + return arch + keyworded = "~" + arch + if keyworded in keys: + return keyworded + match = list(set(accepted.split(" ")).intersection(keys)) + if len(match) > 1: + if arch in match: + return arch + if keyworded in match: + return keyworded + return 'unknown' + if match: + return match[0] + return 'unknown' diff --git a/pym/gentoolkit/metadata.py b/pym/gentoolkit/metadata.py index 93538b3..7765bcb 100644 --- a/pym/gentoolkit/metadata.py +++ b/pym/gentoolkit/metadata.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2009-2010 Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -34,7 +34,7 @@ """ # Move to Imports section after Python-2.6 is stable -from __future__ import with_statement + __all__ = ('MetaData',) __docformat__ = 'epytext' @@ -44,10 +44,9 @@ __docformat__ = 'epytext' # ======= import re -import os import xml.etree.cElementTree as etree -from portage import settings +from portage import os, settings # ======= # Classes diff --git a/pym/gentoolkit/package.py b/pym/gentoolkit/package.py index fb68965..89d06b1 100644 --- a/pym/gentoolkit/package.py +++ b/pym/gentoolkit/package.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -# Copyright 2004, Karl Trygve Kalleberg <karltk@gentoo.org> -# Copyright 2004-2010 Gentoo Foundation +# Copyright(c) 2004, Karl Trygve Kalleberg <karltk@gentoo.org> +# Copyright(c) 2004-2010, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -26,24 +26,34 @@ Example usage: __all__ = ( 'Package', - 'PackageFormatter' + 'PackageFormatter', + 'FORMAT_TMPL_VARS' ) # ======= +# Globals +# ======= + +FORMAT_TMPL_VARS = ( + '$location', '$mask', '$cp', '$cpv', '$category', '$name', '$version', '$revision', + '$fullversion', '$slot', '$repo' +) + +# ======= # Imports # ======= -import os +from string import Template import portage -from portage import settings +from portage import os, settings +from portage.util import LazyItemsDict import gentoolkit.pprinter as pp from gentoolkit import errors from gentoolkit.cpv import CPV from gentoolkit.dbapi import PORTDB, VARDB -from gentoolkit.dependencies import Dependencies -from gentoolkit.metadata import MetaData +from gentoolkit.keyword import determine_keyword # ======= # Classes @@ -86,6 +96,8 @@ class Package(CPV): def metadata(self): """Instantiate a L{gentoolkit.metadata.MetaData} object here.""" + from gentoolkit.metadata import MetaData + if self._metadata is None: metadata_path = os.path.join( self.package_path(), 'metadata.xml' @@ -112,6 +124,8 @@ class Package(CPV): def deps(self): """Instantiate a L{gentoolkit.dependencies.Dependencies} object here.""" + from gentoolkit.dependencies import Dependencies + if self._deps is None: self._deps = Dependencies(self.cpv) @@ -151,7 +165,7 @@ class Package(CPV): """ got_string = False - if isinstance(envvars, basestring): + if isinstance(envvars, str): got_string = True envvars = (envvars,) if prefer_vdb: @@ -274,7 +288,7 @@ class Package(CPV): @param fallback: if the repo_name file does not exist, return the repository name from the path @rtype: str - @return: output of the repository metadata file, which stores the + @return: output of the repository metadata file, which stores the repo_name variable, or try to get the name of the repo from the path. @raise GentoolkitFatalError: if fallback is False and repo_name is @@ -310,21 +324,29 @@ class Package(CPV): """ seen = set() - content_stats = (os.lstat(x) for x in self.parsed_contents()) - # Remove hardlinks by checking for duplicate inodes. Bug #301026. - unique_file_stats = (x for x in content_stats if x.st_ino not in seen - and not seen.add(x.st_ino)) - size = n_uncounted = n_files = 0 - for st in unique_file_stats: + size = n_files = n_uncounted = 0 + for f in self.parsed_contents(): + try: + st = os.lstat(f) + except OSError: + pass + + # Remove hardlinks by checking for duplicate inodes. Bug #301026. + file_inode = st.st_ino + if file_inode in seen: + pass + seen.add(file_inode) + try: size += st.st_size n_files += 1 except OSError: n_uncounted += 1 + return (size, n_files, n_uncounted) def is_installed(self): - """Returns True if this package is installed (merged)""" + """Returns True if this package is installed (merged).""" return self.dblink.exists() @@ -339,9 +361,10 @@ class Package(CPV): return (tree and tree != self._portdir_path) def is_masked(self): - """Returns true if this package is masked against installation. - Note: We blindly assume that the package actually exists on disk - somewhere.""" + """Returns True if this package is masked against installation. + + @note: We blindly assume that the package actually exists on disk. + """ unmasked = PORTDB.xmatch("match-visible", self.cpv) return self.cpv not in unmasked @@ -366,40 +389,85 @@ class PackageFormatter(object): @type pkg: L{gentoolkit.package.Package} @param pkg: package to format - @type format: L{bool} - @param format: Whether to format the package name or not. - Essentially C{format} should be set to False when piping or when + @type do_format: bool + @param do_format: Whether to format the package name or not. + Essentially C{do_format} should be set to False when piping or when quiet output is desired. If C{do_format} is False, only the location attribute will be created to save time. """ - def __init__(self, pkg, do_format=True): - self.pkg = pkg + _tmpl_verbose = "[$location] [$mask] $cpv:$slot" + _tmpl_quiet = "$cpv:$slot" + + def __init__(self, pkg, do_format=True, custom_format=None, fill_sizes = None): + self._pkg = None self.do_format = do_format - self.location = self.format_package_location() or '' + self._str = None + self._location = None + if not custom_format: + if do_format: + custom_format = self._tmpl_verbose + else: + custom_format = self._tmpl_quiet + self.tmpl = Template(custom_format) + self.format_vars = LazyItemsDict() + self.pkg = pkg + if fill_sizes: + self.fill_sizes = fill_sizes + else: + self.fill_sizes = { + 'cpv': 50, + 'keyword': 10, + 'mask': 10, + } + def __repr__(self): return "<%s %s @%#8x>" % (self.__class__.__name__, self.pkg, id(self)) def __str__(self): - if self.do_format: - maskmodes = [' ', ' ~', ' -', 'M ', 'M~', 'M-', '??'] - maskmode = maskmodes[self.format_mask_status()[0]] - return "[%(location)s] [%(mask)s] %(package)s:%(slot)s" % { - 'location': self.location, - 'mask': pp.keyword( - maskmode, - stable=not maskmode.strip(), - hard_masked=set(('M', '?', '-')).intersection(maskmode) - ), - 'package': pp.cpv(str(self.pkg.cpv)), - 'slot': pp.slot(self.pkg.environment("SLOT")) - } - else: - return str(self.pkg.cpv) + if self._str is None: + self._str = self.tmpl.safe_substitute(self.format_vars) + return self._str + + @property + def location(self): + if self._location is None: + self._location = self.format_package_location() + return self._location + + @property + def pkg(self): + """Package to format""" + return self._pkg + + @pkg.setter + def pkg(self, value): + if self._pkg == value: + return + self._pkg = value + self._location = None + + fmt_vars = self.format_vars + self.format_vars.clear() + fmt_vars.addLazySingleton("location", + lambda: getattr(self, "location")) + fmt_vars.addLazySingleton("mask", self.format_mask) + fmt_vars.addLazySingleton("mask2", self.format_mask_status2) + fmt_vars.addLazySingleton("cpv", self.format_cpv) + fmt_vars.addLazySingleton("cpv_fill", self.format_cpv, fill=True) + fmt_vars.addLazySingleton("cp", self.format_cpv, "cp") + fmt_vars.addLazySingleton("category", self.format_cpv, "category") + fmt_vars.addLazySingleton("name", self.format_cpv, "name") + fmt_vars.addLazySingleton("version", self.format_cpv, "version") + fmt_vars.addLazySingleton("revision", self.format_cpv, "revision") + fmt_vars.addLazySingleton("fullversion", self.format_cpv, + "fullversion") + fmt_vars.addLazySingleton("slot", self.format_slot) + fmt_vars.addLazySingleton("repo", self.pkg.repo_name) def format_package_location(self): - """Get the install status (in /var/db/?) and origin (from and overlay + """Get the install status (in /var/db/?) and origin (from an overlay and the Portage tree?). @rtype: str @@ -457,5 +525,49 @@ class PackageFormatter(object): return (result, masking_status) + def format_mask_status2(self): + """Get the mask status of a given package. + """ + mask = self.pkg.mask_status() + if mask: + return pp.masking(mask) + else: + arch = self.pkg.settings("ARCH") + keywords = self.pkg.environment('KEYWORDS') + mask = [determine_keyword(arch, + portage.settings["ACCEPT_KEYWORDS"], + keywords)] + return pp.masking(mask) + + def format_mask(self): + maskmodes = [' ', ' ~', ' -', 'M ', 'M~', 'M-', '??'] + maskmode = maskmodes[self.format_mask_status()[0]] + return pp.keyword( + maskmode, + stable=not maskmode.strip(), + hard_masked=set(('M', '?', '-')).intersection(maskmode) + ) + + def format_cpv(self, attr = None, fill=False): + if attr is None: + value = self.pkg.cpv + else: + value = getattr(self.pkg, attr) + if self.do_format: + if fill: + trail = '.'*(self.fill_sizes['cpv']-len(value)) + return pp.cpv(value) + trail + else: + return pp.cpv(value) + else: + return value + + def format_slot(self): + value = self.pkg.environment("SLOT") + if self.do_format: + return pp.slot(value) + else: + return value + # vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/pprinter.py b/pym/gentoolkit/pprinter.py index c070a0f..2c60299 100644 --- a/pym/gentoolkit/pprinter.py +++ b/pym/gentoolkit/pprinter.py @@ -1,7 +1,7 @@ #!/usr/bin/python # # Copyright 2004 Karl Trygve Kalleberg <karltk@gentoo.org> -# Copyright 2004-2010 Gentoo Foundation +# Copyright 2004-2009 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 # # $Header$ @@ -37,6 +37,7 @@ __all__ = ( import sys import portage.output as output +from portage import archlist # ========= # Functions @@ -124,6 +125,23 @@ def keyword(string, stable=True, hard_masked=False): # keyword masked: return output.blue(string) +def masking(mask): + """Returns a 'masked by' string.""" + if 'package.mask' in mask or 'profile' in mask: + # use porthole wrap style to help clarify meaning + return output.red("M["+mask[0]+"]") + if mask is not []: + for status in mask: + if 'keyword' in status: + # keyword masked | " [missing keyword] " <=looks better + return output.blue("["+status+"]") + if status in archlist: + return output.green(status) + if 'unknown' in status: + return output.yellow(status) + return output.red(status) + return '' + def warn(string): """Returns a warning string.""" return "!!! " + string + "\n" diff --git a/pym/gentoolkit/query.py b/pym/gentoolkit/query.py index 4802bc1..110337e 100644 --- a/pym/gentoolkit/query.py +++ b/pym/gentoolkit/query.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright 2004-2010, Gentoo Foundation +# Copyright(c) 2004-2010, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # @@ -16,19 +16,339 @@ __all__ = ( # Imports # ======= +import fnmatch +import re +from functools import partial + +import portage + +from gentoolkit import CONFIG +from gentoolkit import errors +from gentoolkit import helpers +from gentoolkit import pprinter as pp from gentoolkit.cpv import CPV +from gentoolkit.dbapi import PORTDB, VARDB +from gentoolkit.package import Package +from gentoolkit.sets import get_set_atoms, SETPREFIX #from gentoolkit.helpers import * # ======= # Classes # ======= -class Query(CPV): +class Query(object): """Provides common methods on a package query.""" - def __init__(self, cpv): - if isinstance(cpv, CPV): - self.cpv = cpv + def __init__(self, query, is_regex=False): + """Create query object. + + @type is_regex: bool + @param is_regex: query is a regular expression + """ + + # Separate repository + repository = None + if query.count(':') == 2: + query, repository = query.rsplit(':', 1) + self.query = query.rstrip(':') # Don't leave dangling colon + self.repo_filter = repository + self.is_regex = is_regex + self.query_type = self._get_query_type() + + def __repr__(self): + rx = '' + if self.is_regex: + rx = ' regex' + repo = '' + if self.repo_filter: + repo = ' in %s' % self.repo_filter + return "<%s%s %r%s>" % (self.__class__.__name__, rx, self.query, repo) + + def __str__(self): + return self.query + + def print_summary(self): + """Print a summary of the query.""" + + cpv = CPV(self.query) + cat, pkg = cpv.category, cpv.name + cpv.fullversion + if cat and not self.is_regex: + cat_str = "in %s " % pp.emph(cat.lstrip('><=~!')) + else: + cat_str = "" + + if self.is_regex: + pkg_str = pp.emph(self.query) + else: + pkg_str = pp.emph(pkg) + + repo = '' + if self.repo_filter is not None: + repo = ' %s' % pp.section(self.repo_filter) + + print(" * Searching%s for %s %s..." % (repo, pkg_str, cat_str)) + + def smart_find( + self, + in_installed=True, + in_porttree=True, + in_overlay=True, + include_masked=True, + show_progress=True, + **kwargs + ): + """A high-level wrapper around gentoolkit package-finder functions. + + @type in_installed: bool + @param in_installed: search for query in VARDB + @type in_porttree: bool + @param in_porttree: search for query in PORTDB + @type in_overlay: bool + @param in_overlay: search for query in overlays + @type show_progress: bool + @param show_progress: output search progress + @rtype: list + @return: Package objects matching query + """ + + if in_installed: + if in_porttree or in_overlay: + simple_package_finder = partial( + self.find, + include_masked=include_masked + ) + complex_package_finder = helpers.get_cpvs + else: + simple_package_finder = self.find_installed + complex_package_finder = helpers.get_installed_cpvs + elif in_porttree or in_overlay: + simple_package_finder = partial( + helpers.find_packages, + include_masked=include_masked + ) + complex_package_finder = helpers.get_uninstalled_cpvs + else: + raise errors.GentoolkitFatalError( + "Not searching in installed, Portage tree, or overlay. " + "Nothing to do." + ) + + if self.query_type == "set": + self.package_finder = simple_package_finder + matches = self._do_set_lookup(show_progress=show_progress) + elif self.query_type == "simple": + self.package_finder = simple_package_finder + matches = self._do_simple_lookup( + in_installed=in_installed, + show_progress=show_progress + ) + else: + self.package_finder = complex_package_finder + matches = self._do_complex_lookup(show_progress=show_progress) + + if self.repo_filter is not None: + matches = self._filter_by_repository(matches) + + return matches + + def find(self, in_installed=True, include_masked=True): + """Returns a list of Package objects that matched the query. + + @rtype: list + @return: matching Package objects + """ + + if not self.query: + return [] + + try: + if include_masked: + matches = PORTDB.xmatch("match-all", self.query) + else: + matches = PORTDB.match(self.query) + if in_installed: + matches.extend(VARDB.match(self.query)) + except portage.exception.InvalidAtom as err: + raise errors.GentoolkitInvalidAtom(str(err)) + + return [Package(x) for x in set(matches)] + + def find_installed(self): + """Return a list of Package objects that matched the search key.""" + + try: + matches = VARDB.match(self.query) + # catch the ambiguous package Exception + except portage.exception.AmbiguousPackageName as err: + matches = [] + for pkgkey in err[0]: + matches.extend(VARDB.match(pkgkey)) + except portage.exception.InvalidAtom as err: + raise errors.GentoolkitInvalidAtom(err) + + return [Package(x) for x in set(matches)] + + def find_best(self, include_keyworded=True, include_masked=True): + """Returns the "best" version available. + + Order of preference: + highest available stable => + highest available keyworded => + highest available masked + + @rtype: Package object or None + @return: best of up to three options + @raise errors.GentoolkitInvalidAtom: if query is not valid input + """ + + best = keyworded = masked = None + try: + best = PORTDB.xmatch("bestmatch-visible", self.query) + except portage.exception.InvalidAtom as err: + raise errors.GentoolkitInvalidAtom(err) + # xmatch can return an empty string, so checking for None is not enough + if not best: + if not include_keyworded or include_masked: + return None + try: + matches = PORTDB.xmatch("match-all", self.query) + except portage.exception.InvalidAtom as err: + raise errors.GentoolkitInvalidAtom(err) + masked = portage.best(matches) + keywordable = [] + for m in matches: + status = portage.getmaskingstatus(m) + if 'package.mask' not in status or 'profile' not in status: + keywordable.append(m) + if matches: + keyworded = portage.best(keywordable) + else: + return Package(best) + if include_keyworded and keyworded: + return Package(keyworded) + if include_masked and masked: + return Package(masked) + return None + + def uses_globbing(self): + """Check the query to see if it is using globbing. + + @rtype: bool + @return: True if query uses globbing, else False + """ + + if set('!*?[]').intersection(self.query): + # Is query an atom such as '=sys-apps/portage-2.2*'? + if self.query[0] != '=': + return True + + return False + + def is_ranged(self): + """Return True if the query appears to be ranged, else False.""" + + q = self.query + return q.startswith(('~', '<', '>')) or q.endswith('*') + + def _do_simple_lookup(self, in_installed=True, show_progress=True): + """Find matches for a query which is an atom or cpv.""" + + result = [] + + if show_progress and CONFIG['verbose']: + self.print_summary() + + result = self.package_finder() + if not in_installed: + result = [x for x in result if not x.is_installed()] + + return result + + def _do_complex_lookup(self, show_progress=True): + """Find matches for a query which is a regex or includes globbing.""" + + result = [] + + if show_progress and not CONFIG["piping"]: + self.print_summary() + + cat = CPV(self.query).category + + pre_filter = [] + # The "get_" functions can pre-filter against the whole package key, + # but since we allow globbing now, we run into issues like: + # >>> portage.dep.dep_getkey("sys-apps/portage-*") + # 'sys-apps/portage-' + # So the only way to guarantee we don't overrun the key is to + # prefilter by cat only. + if cat: + if self.is_regex: + cat_re = cat + else: + cat_re = fnmatch.translate(cat) + # [::-1] reverses a sequence, so we're emulating an ".rreplace()" + # except we have to put our "new" string on backwards + cat_re = cat_re[::-1].replace('$', '*./', 1)[::-1] + predicate = lambda x: re.match(cat_re, x) + pre_filter = self.package_finder(predicate=predicate) + + # Post-filter + if self.is_regex: + predicate = lambda x: re.search(self.query, x) + else: + if cat: + query_re = fnmatch.translate(self.query) + else: + query_re = fnmatch.translate("*/%s" % self.query) + predicate = lambda x: re.search(query_re, x) + if pre_filter: + result = [x for x in pre_filter if predicate(x)] else: - self.cpv = CPV(cpv) - del cpv + result = self.package_finder(predicate=predicate) + + return [Package(x) for x in result] + + def _do_set_lookup(self, show_progress=True): + """Find matches for a query that is a package set.""" + + if show_progress and not CONFIG["piping"]: + self.print_summary() + + setname = self.query[len(SETPREFIX):] + result = [] + try: + atoms = get_set_atoms(setname) + except errors.GentoolkitSetNotFound: + return result + + q = self.query + for atom in atoms: + self.query = atom + result.extend(self._do_simple_lookup(show_progress=False)) + self.query = q + + return result + + def _filter_by_repository(self, matches): + """Filter out packages which do not belong to self.repo_filter.""" + + result = [] + for match in matches: + repo_name = match.repo_name() + if repo_name == self.repo_filter: + result.append(match) + elif (not repo_name and + self.repo_filter in ('unknown', 'null')): + result.append(match) + + return result + + def _get_query_type(self): + """Determine of what type the query is.""" + + if self.query.startswith(SETPREFIX): + return "set" + elif self.is_regex or self.uses_globbing(): + return "complex" + return "simple" + diff --git a/pym/gentoolkit/sets.py b/pym/gentoolkit/sets.py new file mode 100644 index 0000000..b9c3601 --- /dev/null +++ b/pym/gentoolkit/sets.py @@ -0,0 +1,57 @@ +# Copyright(c) 2010, Gentoo Foundation +# +# Licensed under the GNU General Public License, v2 or higher +# +# $Header$ + +"""Provides access to Portage sets api""" + +__docformat__ = 'epytext' + +import portage +try: + import portage.sets + _sets_available = True + SETPREFIX = portage.sets.SETPREFIX +except ImportError: + _sets_available = False + SETPREFIX = "@" + +from gentoolkit import errors +from gentoolkit.atom import Atom + + +_set_config = None +def _init_set_config(): + global _set_config + if _set_config is None: + _set_config = portage.sets.load_default_config( + portage.settings, portage.db[portage.root]) + +def get_available_sets(): + """Returns all available sets.""" + + if _sets_available: + _init_set_config() + return _set_config.getSets() + return {} + +def get_set_atoms(setname): + """Return atoms belonging to the given set + + @type setname: string + @param setname: Name of the set + @rtype list + @return: List of atoms in the given set + """ + + if _sets_available: + _init_set_config() + try: + return set([Atom(str(x)) + for x in _set_config.getSetAtoms(setname)]) + except portage.sets.PackageSetNotFound: + raise errors.GentoolkitSetNotFound(setname) + raise errors.GentoolkitSetNotFound(setname) + +# vim: set ts=4 sw=4 tw=79: diff --git a/pym/gentoolkit/test/__init__.py b/pym/gentoolkit/test/__init__.py index 901e478..ea0f3c7 100644 --- a/pym/gentoolkit/test/__init__.py +++ b/pym/gentoolkit/test/__init__.py @@ -1,6 +1,25 @@ #!/usr/bin/python -# Copyright 2009-2010 Gentoo Foundation +# Copyright 2009 Gentoo Foundation # # Distributed under the terms of the GNU General Public License v2 # # $Header$ + +__all__ = ['cmp'] + +# py3k doesn't have cmp emulate it in order to keep testing cmp +# in python-2.x +#XXX: not sure if this is the best place for this +try: + cmp = cmp +except NameError: + def cmp(a, b): + if a == b: + return 0 + elif a < b: + return -1 + elif a > b: + return 1 + # just to be safe, __lt__/ __gt__ above should have thrown + # something like this already + raise TypeError("Comparison between onorderable types") diff --git a/pym/gentoolkit/test/equery/__init__.py b/pym/gentoolkit/test/equery/__init__.py index 901e478..94423e9 100644 --- a/pym/gentoolkit/test/equery/__init__.py +++ b/pym/gentoolkit/test/equery/__init__.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2009-2010 Gentoo Foundation +# Copyright 2009 Gentoo Foundation # # Distributed under the terms of the GNU General Public License v2 # diff --git a/pym/gentoolkit/test/equery/test_init.py b/pym/gentoolkit/test/equery/test_init.py index 98e2648..d59fdc2 100644 --- a/pym/gentoolkit/test/equery/test_init.py +++ b/pym/gentoolkit/test/equery/test_init.py @@ -1,5 +1,8 @@ import unittest -from test import test_support +try: + from test import test_support +except ImportError: + from test import support as test_support from gentoolkit import equery diff --git a/pym/gentoolkit/test/test_atom.py b/pym/gentoolkit/test/test_atom.py index 0c5a786..b498545 100644 --- a/pym/gentoolkit/test/test_atom.py +++ b/pym/gentoolkit/test/test_atom.py @@ -1,4 +1,4 @@ -# Copyright(c) 2009-2010, Gentoo Foundation, Inc. +# Copyright(c) 2009, Gentoo Foundation # Copyright: 2006-2008 Brian Harring <ferringb@gmail.com> # # License: GPL2/BSD @@ -6,9 +6,13 @@ # $Header$ import unittest -from test import test_support +try: + from test import test_support +except ImportError: + from test import support as test_support from gentoolkit.atom import * +from gentoolkit.test import cmp """Atom test suite (verbatim) from pkgcore.""" diff --git a/pym/gentoolkit/test/test_cpv.py b/pym/gentoolkit/test/test_cpv.py index 833fd49..3ce4dee 100644 --- a/pym/gentoolkit/test/test_cpv.py +++ b/pym/gentoolkit/test/test_cpv.py @@ -1,15 +1,19 @@ #!/usr/bin/python # -# Copyright(c) 2009-2010, Gentoo Foundation +# Copyright(c) 2009, Gentoo Foundation # # Licensed under the GNU General Public License, v2 # # $Header$ import unittest -from test import test_support +try: + from test import test_support +except ImportError: + from test import support as test_support from gentoolkit.cpv import * +from gentoolkit.test import cmp class TestGentoolkitCPV(unittest.TestCase): @@ -52,6 +56,28 @@ class TestGentoolkitCPV(unittest.TestCase): self.assertEqual2(CPV('cat/pkg-1_rc2'), CPV('cat/pkg-1_rc2')) self.assertNotEqual2(CPV('cat/pkg-2_rc2-r1'), CPV('cat/pkg-2_rc1-r1')) + def test_compare_strs(self): + # Test ordering of package strings, Portage has test for vercmp, + # so just do the rest + version_tests = [ + # different categories + ('sys-apps/portage-2.1.6.8', 'sys-auth/pambase-20080318'), + # different package names + ('sys-apps/pkgcore-0.4.7.15-r1', 'sys-apps/portage-2.1.6.8'), + # different package versions + ('sys-apps/portage-2.1.6.8', 'sys-apps/portage-2.2_rc25') + ] + # Check less than + for vt in version_tests: + self.failUnless(compare_strs(vt[0], vt[1]) == -1) + # Check greater than + for vt in version_tests: + self.failUnless(compare_strs(vt[1], vt[0]) == 1) + # Check equal + vt = ('sys-auth/pambase-20080318', 'sys-auth/pambase-20080318') + self.failUnless(compare_strs(vt[0], vt[1]) == 0) + + def test_main(): test_support.run_unittest(TestGentoolkitCPV) diff --git a/pym/gentoolkit/test/test_helpers.py b/pym/gentoolkit/test/test_helpers.py index 2291efd..5073659 100644 --- a/pym/gentoolkit/test/test_helpers.py +++ b/pym/gentoolkit/test/test_helpers.py @@ -1,8 +1,12 @@ -import os import unittest import warnings -from tempfile import NamedTemporaryFile -from test import test_support +from tempfile import NamedTemporaryFile, mktemp +try: + from test import test_support +except ImportError: + from test import support as test_support + +from portage import os from gentoolkit import helpers @@ -60,6 +64,21 @@ class TestFileOwner(unittest.TestCase): def tearDown(self): pass + def test_expand_abspaths(self): + expand_abspaths = helpers.FileOwner.expand_abspaths + + initial_file_list = ['foo0', '/foo1', '~/foo2', './foo3'] + # This function should only effect foo3, and not ordering: + + final_file_list = [ + 'foo0', + '/foo1', + '~/foo2', + os.path.join(os.getcwd(), os.path.normpath(initial_file_list[3])) + ] + + self.failUnlessEqual(expand_abspaths(initial_file_list), final_file_list) + def test_extend_realpaths(self): extend_realpaths = helpers.FileOwner.extend_realpaths @@ -69,9 +88,9 @@ class TestFileOwner(unittest.TestCase): f3 = NamedTemporaryFile(prefix='equeryunittest') with warnings.catch_warnings(): warnings.simplefilter("ignore") - sym1 = os.tmpnam() + sym1 = mktemp() os.symlink(f1.name, sym1) - sym2 = os.tmpnam() + sym2 = mktemp() os.symlink(f3.name, sym2) # We've created 3 files and 2 symlinks for testing. We're going to pass # in only the first two files and both symlinks. sym1 points to f1. @@ -98,33 +117,6 @@ class TestFileOwner(unittest.TestCase): class TestGentoolkitHelpers(unittest.TestCase): - def test_compare_package_strings(self): - # Test ordering of package strings, Portage has test for vercmp, - # so just do the rest - version_tests = [ - # different categories - ('sys-apps/portage-2.1.6.8', 'sys-auth/pambase-20080318'), - # different package names - ('sys-apps/pkgcore-0.4.7.15-r1', 'sys-apps/portage-2.1.6.8'), - # different package versions - ('sys-apps/portage-2.1.6.8', 'sys-apps/portage-2.2_rc25') - ] - # Check less than - for vt in version_tests: - self.failUnless( - helpers.compare_package_strings(vt[0], vt[1]) == -1 - ) - # Check greater than - for vt in version_tests: - self.failUnless( - helpers.compare_package_strings(vt[1], vt[0]) == 1 - ) - # Check equal - vt = ('sys-auth/pambase-20080318', 'sys-auth/pambase-20080318') - self.failUnless( - helpers.compare_package_strings(vt[0], vt[1]) == 0 - ) - def test_uses_globbing(self): globbing_tests = [ ('sys-apps/portage-2.1.6.13', False), diff --git a/pym/gentoolkit/test/test_keyword.py b/pym/gentoolkit/test/test_keyword.py new file mode 100644 index 0000000..e054d35 --- /dev/null +++ b/pym/gentoolkit/test/test_keyword.py @@ -0,0 +1,43 @@ +import unittest +import warnings +from tempfile import NamedTemporaryFile +try: + from test import test_support +except ImportError: + from test import support as test_support + +from portage import os + +from gentoolkit import keyword + +class TestGentoolkitKeyword(unittest.TestCase): + + def test_compare_strs(self): + compare_strs = keyword.compare_strs + + # Test ordering of keyword strings + version_tests = [ + # different archs + ('amd64', 'x86'), + # stable vs. unstable + ('amd64-linux', '~amd64-linux'), + # different OSes + ('~x86-linux', '~x86-solaris') + ] + # Check less than + for vt in version_tests: + self.failUnless(compare_strs(vt[0], vt[1]) == -1) + # Check greater than + for vt in version_tests: + self.failUnless(compare_strs(vt[1], vt[0]) == 1) + # Check equal + vt = ('~amd64-linux', '~amd64-linux') + self.failUnless(compare_strs(vt[0], vt[1]) == 0) + + +def test_main(): + test_support.run_unittest(TestGentoolkitHelpers2) + + +if __name__ == '__main__': + test_main() diff --git a/pym/gentoolkit/test/test_syntax.py b/pym/gentoolkit/test/test_syntax.py index bb7dcb4..77fa5dc 100644 --- a/pym/gentoolkit/test/test_syntax.py +++ b/pym/gentoolkit/test/test_syntax.py @@ -1,11 +1,14 @@ -import os -import os.path as osp import unittest import py_compile +from portage import os +osp = os.path + +from gentoolkit.helpers import walk + """Does a basic syntax check by compiling all modules. From Portage.""" -pym_dirs = os.walk(osp.dirname(osp.dirname(osp.dirname(__file__)))) +pym_dirs = walk(osp.dirname(osp.dirname(osp.dirname(__file__)))) blacklist_dirs = frozenset(('.svn', 'test')) class TestForSyntaxErrors(unittest.TestCase): diff --git a/pym/gentoolkit/versionmatch.py b/pym/gentoolkit/versionmatch.py index c081de0..914672f 100644 --- a/pym/gentoolkit/versionmatch.py +++ b/pym/gentoolkit/versionmatch.py @@ -1,9 +1,9 @@ #! /usr/bin/python # -# Copyright 2009-2010 Gentoo Foundation +# Copyright(c) 2009 Gentoo Foundation # Licensed under the GNU General Public License, v2 # -# Copyright 2005-2007 Brian Harring <ferringb@gmail.com> +# Copyright: 2005-2007 Brian Harring <ferringb@gmail.com> # License: GPL2/BSD # # $Header$ @@ -32,7 +32,7 @@ class VersionMatch(object): _convert_op2int = {(-1,):"<", (-1, 0): "<=", (0,):"=", (0, 1):">=", (1,):">"} - _convert_int2op = dict([(v, k) for k, v in _convert_op2int.iteritems()]) + _convert_int2op = dict([(v, k) for k, v in _convert_op2int.items()]) def __init__(self, cpv, op='='): """Initialize a VersionMatch instance. @@ -1,14 +1,19 @@ #!/usr/bin/env python -from __future__ import with_statement +from __future__ import print_function + -import os import re import sys import distutils from distutils import core, log from glob import glob +from portage import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'pym')) +from gentoolkit.helpers import walk + __version__ = os.getenv('VERSION', default='9999') cwd = os.getcwd() @@ -41,7 +46,7 @@ class set_version(core.Command): def run(self): ver = 'svn' if __version__ == '9999' else __version__ - print "Setting version to %s" % ver + print("Settings version to %s" % ver) def sub(files, pattern): for f in files: updated_file = [] @@ -83,8 +88,8 @@ def load_test(): packages = [ - '.'.join(root.split(os.sep)[1:]) - for root, dirs, files in os.walk('pym/gentoolkit') + str('.'.join(root.split(os.sep)[1:])) + for root, dirs, files in walk('pym/gentoolkit') if '__init__.py' in files ] |