diff options
author | Matti Picus <matti.picus@gmail.com> | 2020-12-30 08:30:43 +0200 |
---|---|---|
committer | Matti Picus <matti.picus@gmail.com> | 2020-12-30 08:30:43 +0200 |
commit | bc9eed601c9bb966bef6f52e39d33ae7efb5f672 (patch) | |
tree | aa69d3bf873a8b54062becc0d7222c5d77c3f22a /lib-python | |
parent | rename super ambiguous "index" to "attrkind" (diff) | |
download | pypy-bc9eed601c9bb966bef6f52e39d33ae7efb5f672.tar.gz pypy-bc9eed601c9bb966bef6f52e39d33ae7efb5f672.tar.bz2 pypy-bc9eed601c9bb966bef6f52e39d33ae7efb5f672.zip |
add _msvccompiler from python3 and sync msvc-related files. Use _msvccompiler for windows
Diffstat (limited to 'lib-python')
-rw-r--r-- | lib-python/2.7/distutils/_msvccompiler.py | 561 | ||||
-rw-r--r-- | lib-python/2.7/distutils/ccompiler.py | 30 | ||||
-rw-r--r-- | lib-python/2.7/distutils/msvc9compiler.py | 46 | ||||
-rw-r--r-- | lib-python/2.7/distutils/msvccompiler.py | 270 | ||||
-rw-r--r-- | lib-python/2.7/distutils/tests/test_msvccompiler.py | 138 |
5 files changed, 860 insertions, 185 deletions
diff --git a/lib-python/2.7/distutils/_msvccompiler.py b/lib-python/2.7/distutils/_msvccompiler.py new file mode 100644 index 0000000000..c58aa27e27 --- /dev/null +++ b/lib-python/2.7/distutils/_msvccompiler.py @@ -0,0 +1,561 @@ +"""distutils._msvccompiler + +Contains MSVCCompiler, an implementation of the abstract CCompiler class +for Microsoft Visual Studio 2015. + +The module is compatible with VS 2015 and later. You can find legacy support +for older versions in distutils.msvc9compiler and distutils.msvccompiler. +""" + +# Written by Perry Stoll +# hacked by Robin Becker and Thomas Heller to do a better job of +# finding DevStudio (through the registry) +# ported to VS 2005 and VS 2008 by Christian Heimes +# ported to VS 2015 by Steve Dower + +import os +import subprocess +import contextlib +import warnings +import unittest.mock +with contextlib.suppress(ImportError): + import _winreg as winreg + +from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ + CompileError, LibError, LinkError +from distutils.ccompiler import CCompiler, gen_lib_options +from distutils import log +from distutils.util import get_platform + +from itertools import count + +def _find_vc2015(): + try: + key = winreg.OpenKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"Software\Microsoft\VisualStudio\SxS\VC7", + access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY + ) + except OSError: + log.debug("Visual C++ is not registered") + return None, None + + best_version = 0 + best_dir = None + with key: + for i in count(): + try: + v, vc_dir, vt = winreg.EnumValue(key, i) + except OSError: + break + if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir): + try: + version = int(float(v)) + except (ValueError, TypeError): + continue + if version >= 14 and version > best_version: + best_version, best_dir = version, vc_dir + return best_version, best_dir + +def _find_vc2017(): + """Returns "15, path" based on the result of invoking vswhere.exe + If no install is found, returns "None, None" + + The version is returned to avoid unnecessarily changing the function + result. It may be ignored when the path is not None. + + If vswhere.exe is not available, by definition, VS 2017 is not + installed. + """ + root = os.environ.get("ProgramFiles(x86)") or os.environ.get("ProgramFiles") + if not root: + return None, None + + try: + path = subprocess.check_output([ + os.path.join(root, "Microsoft Visual Studio", "Installer", "vswhere.exe"), + "-latest", + "-prerelease", + "-requires", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", "installationPath", + "-products", "*", + ], encoding="mbcs", errors="strict").strip() + except (subprocess.CalledProcessError, OSError, UnicodeDecodeError): + return None, None + + path = os.path.join(path, "VC", "Auxiliary", "Build") + if os.path.isdir(path): + return 15, path + + return None, None + +PLAT_SPEC_TO_RUNTIME = { + 'x86' : 'x86', + 'x86_amd64' : 'x64', + 'x86_arm' : 'arm', + 'x86_arm64' : 'arm64' +} + +def _find_vcvarsall(plat_spec): + # bpo-38597: Removed vcruntime return value + _, best_dir = _find_vc2017() + + if not best_dir: + best_version, best_dir = _find_vc2015() + + if not best_dir: + log.debug("No suitable Visual C++ version found") + return None, None + + vcvarsall = os.path.join(best_dir, "vcvarsall.bat") + if not os.path.isfile(vcvarsall): + log.debug("%s cannot be found", vcvarsall) + return None, None + + return vcvarsall, None + +def _get_vc_env(plat_spec): + if os.getenv("DISTUTILS_USE_SDK"): + return { + key.lower(): value + for key, value in os.environ.items() + } + + vcvarsall, _ = _find_vcvarsall(plat_spec) + if not vcvarsall: + raise DistutilsPlatformError("Unable to find vcvarsall.bat") + + try: + out = subprocess.check_output( + 'cmd /u /c "{}" {} && set'.format(vcvarsall, plat_spec), + stderr=subprocess.STDOUT, + ).decode('utf-16le', errors='replace') + except subprocess.CalledProcessError as exc: + log.error(exc.output) + raise DistutilsPlatformError("Error executing {}" + .format(exc.cmd)) + + env = { + key.lower(): value + for key, _, value in + (line.partition('=') for line in out.splitlines()) + if key and value + } + + return env + +def _find_exe(exe, paths=None): + """Return path to an MSVC executable program. + + Tries to find the program in several places: first, one of the + MSVC program search paths from the registry; next, the directories + in the PATH environment variable. If any of those work, return an + absolute path that is known to exist. If none of them work, just + return the original program name, 'exe'. + """ + if not paths: + paths = os.getenv('path').split(os.pathsep) + for p in paths: + fn = os.path.join(os.path.abspath(p), exe) + if os.path.isfile(fn): + return fn + return exe + +# A map keyed by get_platform() return values to values accepted by +# 'vcvarsall.bat'. Always cross-compile from x86 to work with the +# lighter-weight MSVC installs that do not include native 64-bit tools. +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'x86_amd64', + 'win-arm32' : 'x86_arm', + 'win-arm64' : 'x86_arm64' +} + +class MSVCCompiler(CCompiler) : + """Concrete class that implements an interface to Microsoft Visual C++, + as defined by the CCompiler abstract class.""" + + compiler_type = 'msvc' + + # Just set this so CCompiler's constructor doesn't barf. We currently + # don't use the 'set_executables()' bureaucracy provided by CCompiler, + # as it really isn't necessary for this sort of single-compiler class. + # Would be nice to have a consistent interface with UnixCCompiler, + # though, so it's worth thinking about. + executables = {} + + # Private class data (need to distinguish C from C++ source for compiler) + _c_extensions = ['.c'] + _cpp_extensions = ['.cc', '.cpp', '.cxx'] + _rc_extensions = ['.rc'] + _mc_extensions = ['.mc'] + + # Needed for the filename generation methods provided by the + # base class, CCompiler. + src_extensions = (_c_extensions + _cpp_extensions + + _rc_extensions + _mc_extensions) + res_extension = '.res' + obj_extension = '.obj' + static_lib_extension = '.lib' + shared_lib_extension = '.dll' + static_lib_format = shared_lib_format = '%s%s' + exe_extension = '.exe' + + + def __init__(self, verbose=0, dry_run=0, force=0): + CCompiler.__init__ (self, verbose, dry_run, force) + # target platform (.plat_name is consistent with 'bdist') + self.plat_name = None + self.initialized = False + + def initialize(self, plat_name=None): + # multi-init means we would need to check platform same each time... + assert not self.initialized, "don't init multiple times" + if plat_name is None: + plat_name = get_platform() + # sanity check for platforms to prevent obscure errors later. + if plat_name not in PLAT_TO_VCVARS: + raise DistutilsPlatformError("--plat-name must be one of {}" + .format(tuple(PLAT_TO_VCVARS))) + + # Get the vcvarsall.bat spec for the requested platform. + plat_spec = PLAT_TO_VCVARS[plat_name] + + vc_env = _get_vc_env(plat_spec) + if not vc_env: + raise DistutilsPlatformError("Unable to find a compatible " + "Visual Studio installation.") + + self._paths = vc_env.get('path', '') + paths = self._paths.split(os.pathsep) + self.cc = _find_exe("cl.exe", paths) + self.linker = _find_exe("link.exe", paths) + self.lib = _find_exe("lib.exe", paths) + self.rc = _find_exe("rc.exe", paths) # resource compiler + self.mc = _find_exe("mc.exe", paths) # message compiler + self.mt = _find_exe("mt.exe", paths) # message compiler + + for dir in vc_env.get('include', '').split(os.pathsep): + if dir: + self.add_include_dir(dir.rstrip(os.sep)) + + for dir in vc_env.get('lib', '').split(os.pathsep): + if dir: + self.add_library_dir(dir.rstrip(os.sep)) + + self.preprocess_options = None + # bpo-38597: Always compile with dynamic linking + # Future releases of Python 3.x will include all past + # versions of vcruntime*.dll for compatibility. + self.compile_options = [ + '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG', '/MD' + ] + + self.compile_options_debug = [ + '/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG' + ] + + ldflags = [ + '/nologo', '/INCREMENTAL:NO', '/LTCG' + ] + + ldflags_debug = [ + '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL' + ] + + self.ldflags_exe = [*ldflags, '/MANIFEST:EMBED,ID=1'] + self.ldflags_exe_debug = [*ldflags_debug, '/MANIFEST:EMBED,ID=1'] + self.ldflags_shared = [*ldflags, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] + self.ldflags_shared_debug = [*ldflags_debug, '/DLL', '/MANIFEST:EMBED,ID=2', '/MANIFESTUAC:NO'] + self.ldflags_static = [*ldflags] + self.ldflags_static_debug = [*ldflags_debug] + + self._ldflags = { + (CCompiler.EXECUTABLE, None): self.ldflags_exe, + (CCompiler.EXECUTABLE, False): self.ldflags_exe, + (CCompiler.EXECUTABLE, True): self.ldflags_exe_debug, + (CCompiler.SHARED_OBJECT, None): self.ldflags_shared, + (CCompiler.SHARED_OBJECT, False): self.ldflags_shared, + (CCompiler.SHARED_OBJECT, True): self.ldflags_shared_debug, + (CCompiler.SHARED_LIBRARY, None): self.ldflags_static, + (CCompiler.SHARED_LIBRARY, False): self.ldflags_static, + (CCompiler.SHARED_LIBRARY, True): self.ldflags_static_debug, + } + + self.initialized = True + + # -- Worker methods ------------------------------------------------ + + def object_filenames(self, + source_filenames, + strip_dir=0, + output_dir=''): + ext_map = { + **{ext: self.obj_extension for ext in self.src_extensions}, + **{ext: self.res_extension for ext in self._rc_extensions + self._mc_extensions}, + } + + output_dir = output_dir or '' + + def make_out_path(p): + base, ext = os.path.splitext(p) + if strip_dir: + base = os.path.basename(base) + else: + _, base = os.path.splitdrive(base) + if base.startswith((os.path.sep, os.path.altsep)): + base = base[1:] + try: + # XXX: This may produce absurdly long paths. We should check + # the length of the result and trim base until we fit within + # 260 characters. + return os.path.join(output_dir, base + ext_map[ext]) + except LookupError: + # Better to raise an exception instead of silently continuing + # and later complain about sources and targets having + # different lengths + raise CompileError("Don't know how to compile {}".format(p)) + + return list(map(make_out_path, source_filenames)) + + + def compile(self, sources, + output_dir=None, macros=None, include_dirs=None, debug=0, + extra_preargs=None, extra_postargs=None, depends=None): + + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, + sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info + + compile_opts = extra_preargs or [] + compile_opts.append('/c') + if debug: + compile_opts.extend(self.compile_options_debug) + else: + compile_opts.extend(self.compile_options) + + + add_cpp_opts = False + + for obj in objects: + try: + src, ext = build[obj] + except KeyError: + continue + if debug: + # pass the full pathname to MSVC in debug mode, + # this allows the debugger to find the source file + # without asking the user to browse for it + src = os.path.abspath(src) + + if ext in self._c_extensions: + input_opt = "/Tc" + src + elif ext in self._cpp_extensions: + input_opt = "/Tp" + src + add_cpp_opts = True + elif ext in self._rc_extensions: + # compile .RC to .RES file + input_opt = src + output_opt = "/fo" + obj + try: + self.spawn([self.rc] + pp_opts + [output_opt, input_opt]) + except DistutilsExecError as msg: + raise CompileError(msg) + continue + elif ext in self._mc_extensions: + # Compile .MC to .RC file to .RES file. + # * '-h dir' specifies the directory for the + # generated include file + # * '-r dir' specifies the target directory of the + # generated RC file and the binary message resource + # it includes + # + # For now (since there are no options to change this), + # we use the source-directory for the include file and + # the build directory for the RC file and message + # resources. This works at least for win32all. + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) + try: + # first compile .MC to .RC and .H file + self.spawn([self.mc, '-h', h_dir, '-r', rc_dir, src]) + base, _ = os.path.splitext(os.path.basename (src)) + rc_file = os.path.join(rc_dir, base + '.rc') + # then compile .RC to .RES file + self.spawn([self.rc, "/fo" + obj, rc_file]) + + except DistutilsExecError as msg: + raise CompileError(msg) + continue + else: + # how to handle this file? + raise CompileError("Don't know how to compile {} to {}" + .format(src, obj)) + + args = [self.cc] + compile_opts + pp_opts + if add_cpp_opts: + args.append('/EHsc') + args.append(input_opt) + args.append("/Fo" + obj) + args.extend(extra_postargs) + + try: + self.spawn(args) + except DistutilsExecError as msg: + raise CompileError(msg) + + return objects + + + def create_static_lib(self, + objects, + output_libname, + output_dir=None, + debug=0, + target_lang=None): + + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + output_filename = self.library_filename(output_libname, + output_dir=output_dir) + + if self._need_link(objects, output_filename): + lib_args = objects + ['/OUT:' + output_filename] + if debug: + pass # XXX what goes here? + try: + log.debug('Executing "%s" %s', self.lib, ' '.join(lib_args)) + self.spawn([self.lib] + lib_args) + except DistutilsExecError as msg: + raise LibError(msg) + else: + log.debug("skipping %s (up-to-date)", output_filename) + + + def link(self, + target_desc, + objects, + output_filename, + output_dir=None, + libraries=None, + library_dirs=None, + runtime_library_dirs=None, + export_symbols=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + build_temp=None, + target_lang=None): + + if not self.initialized: + self.initialize() + objects, output_dir = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + libraries, library_dirs, runtime_library_dirs = fixed_args + + if runtime_library_dirs: + self.warn("I don't know what to do with 'runtime_library_dirs': " + + str(runtime_library_dirs)) + + lib_opts = gen_lib_options(self, + library_dirs, runtime_library_dirs, + libraries) + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + ldflags = self._ldflags[target_desc, debug] + + export_opts = ["/EXPORT:" + sym for sym in (export_symbols or [])] + + ld_args = (ldflags + lib_opts + export_opts + + objects + ['/OUT:' + output_filename]) + + # The MSVC linker generates .lib and .exp files, which cannot be + # suppressed by any linker switches. The .lib files may even be + # needed! Make sure they are generated in the temporary build + # directory. Since they have different names for debug and release + # builds, they can go into the same directory. + build_temp = os.path.dirname(objects[0]) + if export_symbols is not None: + (dll_name, dll_ext) = os.path.splitext( + os.path.basename(output_filename)) + implib_file = os.path.join( + build_temp, + self.library_filename(dll_name)) + ld_args.append ('/IMPLIB:' + implib_file) + + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + + output_dir = os.path.dirname(os.path.abspath(output_filename)) + self.mkpath(output_dir) + try: + log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args)) + self.spawn([self.linker] + ld_args) + except DistutilsExecError as msg: + raise LinkError(msg) + else: + log.debug("skipping %s (up-to-date)", output_filename) + + def spawn(self, cmd): + env = dict(os.environ, PATH=self._paths) + with self._fallback_spawn(cmd, env) as fallback: + return super().spawn(cmd, env=env) + return fallback.value + + @contextlib.contextmanager + def _fallback_spawn(self, cmd, env): + """ + Discovered in pypa/distutils#15, some tools monkeypatch the compiler, + so the 'env' kwarg causes a TypeError. Detect this condition and + restore the legacy, unsafe behavior. + """ + bag = type('Bag', (), {})() + try: + yield bag + except TypeError as exc: + if "unexpected keyword argument 'env'" not in str(exc): + raise + else: + return + warnings.warn( + "Fallback spawn triggered. Please update distutils monkeypatch.") + with unittest.mock.patch('os.environ', env): + bag.value = super().spawn(cmd) + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "/LIBPATH:" + dir + + def runtime_library_dir_option(self, dir): + raise DistutilsPlatformError( + "don't know how to set runtime library search path for MSVC") + + def library_option(self, lib): + return self.library_filename(lib) + + def find_library_file(self, dirs, lib, debug=0): + # Prefer a debugging library if found (and requested), but deal + # with it if we don't have one. + if debug: + try_names = [lib + "_d", lib] + else: + try_names = [lib] + for dir in dirs: + for name in try_names: + libfile = os.path.join(dir, self.library_filename(name)) + if os.path.isfile(libfile): + return libfile + else: + # Oops, didn't find it in *any* of 'dirs' + return None diff --git a/lib-python/2.7/distutils/ccompiler.py b/lib-python/2.7/distutils/ccompiler.py index 3a7b5b84e1..40344b692a 100644 --- a/lib-python/2.7/distutils/ccompiler.py +++ b/lib-python/2.7/distutils/ccompiler.py @@ -897,7 +897,6 @@ _default_compilers = ( # on a cygwin built python we can use gcc like an ordinary UNIXish # compiler ('cygwin.*', 'unix'), - ('os2emx', 'emx'), # OS name mappings ('posix', 'unix'), @@ -906,15 +905,14 @@ _default_compilers = ( ) def get_default_compiler(osname=None, platform=None): - """ Determine the default compiler to use for the given platform. + """Determine the default compiler to use for the given platform. - osname should be one of the standard Python OS names (i.e. the - ones returned by os.name) and platform the common value - returned by sys.platform for the platform in question. - - The default values are os.name and sys.platform in case the - parameters are not given. + osname should be one of the standard Python OS names (i.e. the + ones returned by os.name) and platform the common value + returned by sys.platform for the platform in question. + The default values are os.name and sys.platform in case the + parameters are not given. """ if osname is None: osname = os.name @@ -932,7 +930,7 @@ def get_default_compiler(osname=None, platform=None): # is assumed to be in the 'distutils' package.) compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', "standard UNIX-style compiler"), - 'msvc': ('msvccompiler', 'MSVCCompiler', + 'msvc': ('_msvccompiler', 'MSVCCompiler', "Microsoft Visual C++"), 'cygwin': ('cygwinccompiler', 'CygwinCCompiler', "Cygwin port of GNU C Compiler for Win32"), @@ -940,8 +938,6 @@ compiler_class = { 'unix': ('unixccompiler', 'UnixCCompiler', "Mingw32 port of GNU C Compiler for Win32"), 'bcpp': ('bcppcompiler', 'BCPPCompiler', "Borland C++ Compiler"), - 'emx': ('emxccompiler', 'EMXCCompiler', - "EMX port of GNU C Compiler for OS/2"), } def show_compilers(): @@ -984,7 +980,7 @@ def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): msg = "don't know how to compile C/C++ code on platform '%s'" % plat if compiler is not None: msg = msg + " with '%s' compiler" % compiler - raise DistutilsPlatformError, msg + raise DistutilsPlatformError(msg) try: module_name = "distutils." + module_name @@ -992,13 +988,13 @@ def new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0): module = sys.modules[module_name] klass = vars(module)[class_name] except ImportError: - raise DistutilsModuleError, \ + raise DistutilsModuleError( "can't compile C/C++ code: unable to load module '%s'" % \ - module_name + module_name) except KeyError: - raise DistutilsModuleError, \ - ("can't compile C/C++ code: unable to find class '%s' " + - "in module '%s'") % (class_name, module_name) + raise DistutilsModuleError( + "can't compile C/C++ code: unable to find class '%s' " + "in module '%s'" % (class_name, module_name)) # XXX The None is necessary to preserve backwards compatibility # with classes that expect verbose to be the first positional diff --git a/lib-python/2.7/distutils/msvc9compiler.py b/lib-python/2.7/distutils/msvc9compiler.py index 56ec9d3bc8..261fae5fb9 100644 --- a/lib-python/2.7/distutils/msvc9compiler.py +++ b/lib-python/2.7/distutils/msvc9compiler.py @@ -12,30 +12,28 @@ for older versions of VS in distutils.msvccompiler. # finding DevStudio (through the registry) # ported to VS2005 and VS 2008 by Christian Heimes -__revision__ = "$Id$" - import os import subprocess import sys import re -from distutils.errors import (DistutilsExecError, DistutilsPlatformError, - CompileError, LibError, LinkError) +from distutils.errors import DistutilsExecError, DistutilsPlatformError, \ + CompileError, LibError, LinkError from distutils.ccompiler import CCompiler, gen_lib_options from distutils import log from distutils.util import get_platform -import _winreg +import _winreg as winreg -RegOpenKeyEx = _winreg.OpenKeyEx -RegEnumKey = _winreg.EnumKey -RegEnumValue = _winreg.EnumValue -RegError = _winreg.error +RegOpenKeyEx = winreg.OpenKeyEx +RegEnumKey = winreg.EnumKey +RegEnumValue = winreg.EnumValue +RegError = winreg.error -HKEYS = (_winreg.HKEY_USERS, - _winreg.HKEY_CURRENT_USER, - _winreg.HKEY_LOCAL_MACHINE, - _winreg.HKEY_CLASSES_ROOT) +HKEYS = (winreg.HKEY_USERS, + winreg.HKEY_CURRENT_USER, + winreg.HKEY_LOCAL_MACHINE, + winreg.HKEY_CLASSES_ROOT) NATIVE_WIN64 = (sys.platform == 'win32' and sys.maxsize > 2**32) if NATIVE_WIN64: @@ -58,7 +56,6 @@ else: PLAT_TO_VCVARS = { 'win32' : 'x86', 'win-amd64' : 'amd64', - 'win-ia64' : 'ia64', } class Reg: @@ -230,6 +227,7 @@ def find_vcvarsall(version): productdir = Reg.get_value(r"%s\Setup\VC" % vsbase, "productdir") except KeyError: + log.debug("Unable to find productdir in registry") productdir = None # trying Express edition @@ -267,7 +265,7 @@ def query_vcvarsall(version, arch="x86"): """Launch vcvarsall.bat and read the settings from its environment """ vcvarsall = find_vcvarsall(version) - interesting = set(("include", "lib", "libpath", "path")) + interesting = {"include", "lib", "libpath", "path"} result = {} if vcvarsall is None: @@ -356,7 +354,7 @@ class MSVCCompiler(CCompiler) : if plat_name is None: plat_name = get_platform() # sanity check for platforms to prevent obscure errors later. - ok_plats = 'win32', 'win-amd64', 'win-ia64' + ok_plats = 'win32', 'win-amd64' if plat_name not in ok_plats: raise DistutilsPlatformError("--plat-name must be one of %s" % (ok_plats,)) @@ -374,7 +372,6 @@ class MSVCCompiler(CCompiler) : # to cross compile, you use 'x86_amd64'. # On AMD64, 'vcvars32.bat amd64' is a native build env; to cross # compile use 'x86' (ie, it runs the x86 compiler directly) - # No idea how itanium handles this, if at all. if plat_name == get_platform() or plat_name == 'win32': # native build or cross-compile to win32 plat_spec = PLAT_TO_VCVARS[plat_name] @@ -507,7 +504,7 @@ class MSVCCompiler(CCompiler) : try: self.spawn([self.rc] + pp_opts + [output_opt] + [input_opt]) - except DistutilsExecError, msg: + except DistutilsExecError as msg: raise CompileError(msg) continue elif ext in self._mc_extensions: @@ -534,7 +531,7 @@ class MSVCCompiler(CCompiler) : self.spawn([self.rc] + ["/fo" + obj] + [rc_file]) - except DistutilsExecError, msg: + except DistutilsExecError as msg: raise CompileError(msg) continue else: @@ -547,7 +544,7 @@ class MSVCCompiler(CCompiler) : self.spawn([self.cc] + compile_opts + pp_opts + [input_opt, output_opt] + extra_postargs) - except DistutilsExecError, msg: + except DistutilsExecError as msg: raise CompileError(msg) return objects @@ -572,7 +569,7 @@ class MSVCCompiler(CCompiler) : pass # XXX what goes here? try: self.spawn([self.lib] + lib_args) - except DistutilsExecError, msg: + except DistutilsExecError as msg: raise LibError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) @@ -653,7 +650,7 @@ class MSVCCompiler(CCompiler) : self.mkpath(os.path.dirname(output_filename)) try: self.spawn([self.linker] + ld_args) - except DistutilsExecError, msg: + except DistutilsExecError as msg: raise LinkError(msg) # embed the manifest @@ -668,7 +665,7 @@ class MSVCCompiler(CCompiler) : try: self.spawn(['mt.exe', '-nologo', '-manifest', mffilename, out_arg]) - except DistutilsExecError, msg: + except DistutilsExecError as msg: raise LinkError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) @@ -683,7 +680,6 @@ class MSVCCompiler(CCompiler) : temp_manifest = os.path.join( build_temp, os.path.basename(output_filename) + ".manifest") - ld_args.append('/MANIFEST') ld_args.append('/MANIFESTFILE:' + temp_manifest) def manifest_get_embed_info(self, target_desc, ld_args): @@ -730,7 +726,7 @@ class MSVCCompiler(CCompiler) : r"""VC\d{2}\.CRT("|').*?(/>|</assemblyIdentity>)""", re.DOTALL) manifest_buf = re.sub(pattern, "", manifest_buf) - pattern = "<dependentAssembly>\s*</dependentAssembly>" + pattern = r"<dependentAssembly>\s*</dependentAssembly>" manifest_buf = re.sub(pattern, "", manifest_buf) # Now see if any other assemblies are referenced - if not, we # don't want a manifest embedded. diff --git a/lib-python/2.7/distutils/msvccompiler.py b/lib-python/2.7/distutils/msvccompiler.py index 0e69fd368c..6ddf326be6 100644 --- a/lib-python/2.7/distutils/msvccompiler.py +++ b/lib-python/2.7/distutils/msvccompiler.py @@ -8,41 +8,37 @@ for the Microsoft Visual Studio. # hacked by Robin Becker and Thomas Heller to do a better job of # finding DevStudio (through the registry) -__revision__ = "$Id$" - -import sys -import os -import string - -from distutils.errors import (DistutilsExecError, DistutilsPlatformError, - CompileError, LibError, LinkError) -from distutils.ccompiler import CCompiler, gen_lib_options +import sys, os +from distutils.errors import \ + DistutilsExecError, DistutilsPlatformError, \ + CompileError, LibError, LinkError +from distutils.ccompiler import \ + CCompiler, gen_lib_options from distutils import log -_can_read_reg = 0 +_can_read_reg = False try: - import _winreg + import _winreg as winreg - _can_read_reg = 1 - hkey_mod = _winreg + _can_read_reg = True + hkey_mod = winreg - RegOpenKeyEx = _winreg.OpenKeyEx - RegEnumKey = _winreg.EnumKey - RegEnumValue = _winreg.EnumValue - RegError = _winreg.error + RegOpenKeyEx = winreg.OpenKeyEx + RegEnumKey = winreg.EnumKey + RegEnumValue = winreg.EnumValue + RegError = winreg.error except ImportError: try: import win32api import win32con - _can_read_reg = 1 + _can_read_reg = True hkey_mod = win32con RegOpenKeyEx = win32api.RegOpenKeyEx RegEnumKey = win32api.RegEnumKey RegEnumValue = win32api.RegEnumValue RegError = win32api.error - except ImportError: log.info("Warning: Can't read registry to find the " "necessary compiler setting\n" @@ -58,20 +54,19 @@ if _can_read_reg: def read_keys(base, key): """Return list of registry keys.""" - try: handle = RegOpenKeyEx(base, key) except RegError: return None L = [] i = 0 - while 1: + while True: try: k = RegEnumKey(handle, i) except RegError: break L.append(k) - i = i + 1 + i += 1 return L def read_values(base, key): @@ -85,27 +80,26 @@ def read_values(base, key): return None d = {} i = 0 - while 1: + while True: try: name, value, type = RegEnumValue(handle, i) except RegError: break name = name.lower() d[convert_mbcs(name)] = convert_mbcs(value) - i = i + 1 + i += 1 return d def convert_mbcs(s): - enc = getattr(s, "encode", None) - if enc is not None: + dec = getattr(s, "decode", None) + if dec is not None: try: - s = enc("mbcs") + s = dec("mbcs") except UnicodeError: pass return s class MacroExpander: - def __init__(self, version): self.macros = {} self.load_macros(version) @@ -128,9 +122,9 @@ class MacroExpander: self.set_macro("FrameworkSDKDir", net, "sdkinstallrootv1.1") else: self.set_macro("FrameworkSDKDir", net, "sdkinstallroot") - except KeyError: - raise DistutilsPlatformError, \ - ("""Python was built with Visual Studio 2003; + except KeyError as exc: # + raise DistutilsPlatformError( + """Python was built with Visual Studio 2003; extensions must be built with a compiler than can generate compatible binaries. Visual Studio 2003 was not found on this system. If you have Cygwin installed, you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") @@ -147,7 +141,7 @@ you can try compiling with MingW32, by passing "-c mingw32" to setup.py.""") def sub(self, s): for k, v in self.macros.items(): - s = string.replace(s, k, v) + s = s.replace(k, v) return s def get_build_version(): @@ -156,14 +150,16 @@ def get_build_version(): For Python 2.3 and up, the version number is included in sys.version. For earlier versions, assume the compiler is MSVC 6. """ - prefix = "MSC v." - i = string.find(sys.version, prefix) + i = sys.version.find(prefix) if i == -1: return 6 i = i + len(prefix) s, rest = sys.version[i:].split(" ", 1) majorVersion = int(s[:-2]) - 6 + if majorVersion >= 13: + # v13 was skipped and should be v14 + majorVersion += 1 minorVersion = int(s[2:3]) / 10.0 # I don't think paths are affected by minor version in version 6 if majorVersion == 6: @@ -176,14 +172,14 @@ def get_build_version(): def get_build_architecture(): """Return the processor architecture. - Possible results are "Intel", "Itanium", or "AMD64". + Possible results are "Intel" or "AMD64". """ prefix = " bit (" - i = string.find(sys.version, prefix) + i = sys.version.find(prefix) if i == -1: return "Intel" - j = string.find(sys.version, ")", i) + j = sys.version.find(")", i) return sys.version[i+len(prefix):j] def normalize_and_reduce_paths(paths): @@ -201,7 +197,7 @@ def normalize_and_reduce_paths(paths): return reduced_paths -class MSVCCompiler (CCompiler) : +class MSVCCompiler(CCompiler) : """Concrete class that implements an interface to Microsoft Visual C++, as defined by the CCompiler abstract class.""" @@ -231,7 +227,7 @@ class MSVCCompiler (CCompiler) : static_lib_format = shared_lib_format = '%s%s' exe_extension = '.exe' - def __init__ (self, verbose=0, dry_run=0, force=0): + def __init__(self, verbose=0, dry_run=0, force=0): CCompiler.__init__ (self, verbose, dry_run, force) self.__version = get_build_version() self.__arch = get_build_architecture() @@ -262,11 +258,11 @@ class MSVCCompiler (CCompiler) : else: self.__paths = self.get_msvc_paths("path") - if len (self.__paths) == 0: - raise DistutilsPlatformError, \ - ("Python was built with %s, " + if len(self.__paths) == 0: + raise DistutilsPlatformError("Python was built with %s, " "and extensions need to be built with the same " - "version of the compiler, but it isn't installed." % self.__product) + "version of the compiler, but it isn't installed." + % self.__product) self.cc = self.find_exe("cl.exe") self.linker = self.find_exe("link.exe") @@ -278,12 +274,12 @@ class MSVCCompiler (CCompiler) : # extend the MSVC path with the current path try: - for p in string.split(os.environ['path'], ';'): + for p in os.environ['path'].split(';'): self.__paths.append(p) except KeyError: pass self.__paths = normalize_and_reduce_paths(self.__paths) - os.environ['path'] = string.join(self.__paths, ';') + os.environ['path'] = ";".join(self.__paths) self.preprocess_options = None if self.__arch == "Intel": @@ -313,10 +309,10 @@ class MSVCCompiler (CCompiler) : # -- Worker methods ------------------------------------------------ - def object_filenames (self, - source_filenames, - strip_dir=0, - output_dir=''): + def object_filenames(self, + source_filenames, + strip_dir=0, + output_dir=''): # Copied from ccompiler.py, extended to return .res as 'object'-file # for .rc input file if output_dir is None: output_dir = '' @@ -343,17 +339,16 @@ class MSVCCompiler (CCompiler) : base + self.obj_extension)) return obj_names - # object_filenames () - def compile(self, sources, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None): - if not self.initialized: self.initialize() - macros, objects, extra_postargs, pp_opts, build = \ - self._setup_compile(output_dir, macros, include_dirs, sources, - depends, extra_postargs) + if not self.initialized: + self.initialize() + compile_info = self._setup_compile(output_dir, macros, include_dirs, + sources, depends, extra_postargs) + macros, objects, extra_postargs, pp_opts, build = compile_info compile_opts = extra_preargs or [] compile_opts.append ('/c') @@ -382,13 +377,12 @@ class MSVCCompiler (CCompiler) : input_opt = src output_opt = "/fo" + obj try: - self.spawn ([self.rc] + pp_opts + - [output_opt] + [input_opt]) - except DistutilsExecError, msg: - raise CompileError, msg + self.spawn([self.rc] + pp_opts + + [output_opt] + [input_opt]) + except DistutilsExecError as msg: + raise CompileError(msg) continue elif ext in self._mc_extensions: - # Compile .MC to .RC file to .RES file. # * '-h dir' specifies the directory for the # generated include file @@ -400,99 +394,95 @@ class MSVCCompiler (CCompiler) : # we use the source-directory for the include file and # the build directory for the RC file and message # resources. This works at least for win32all. - - h_dir = os.path.dirname (src) - rc_dir = os.path.dirname (obj) + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) try: # first compile .MC to .RC and .H file - self.spawn ([self.mc] + - ['-h', h_dir, '-r', rc_dir] + [src]) + self.spawn([self.mc] + + ['-h', h_dir, '-r', rc_dir] + [src]) base, _ = os.path.splitext (os.path.basename (src)) rc_file = os.path.join (rc_dir, base + '.rc') # then compile .RC to .RES file - self.spawn ([self.rc] + - ["/fo" + obj] + [rc_file]) + self.spawn([self.rc] + + ["/fo" + obj] + [rc_file]) - except DistutilsExecError, msg: - raise CompileError, msg + except DistutilsExecError as msg: + raise CompileError(msg) continue else: # how to handle this file? - raise CompileError ( - "Don't know how to compile %s to %s" % \ - (src, obj)) + raise CompileError("Don't know how to compile %s to %s" + % (src, obj)) output_opt = "/Fo" + obj try: - self.spawn ([self.cc] + compile_opts + pp_opts + - [input_opt, output_opt] + - extra_postargs) - except DistutilsExecError, msg: - raise CompileError, msg + self.spawn([self.cc] + compile_opts + pp_opts + + [input_opt, output_opt] + + extra_postargs) + except DistutilsExecError as msg: + raise CompileError(msg) return objects - # compile () - - def create_static_lib (self, - objects, - output_libname, - output_dir=None, - debug=0, - target_lang=None): + def create_static_lib(self, + objects, + output_libname, + output_dir=None, + debug=0, + target_lang=None): - if not self.initialized: self.initialize() - (objects, output_dir) = self._fix_object_args (objects, output_dir) - output_filename = \ - self.library_filename (output_libname, output_dir=output_dir) + if not self.initialized: + self.initialize() + (objects, output_dir) = self._fix_object_args(objects, output_dir) + output_filename = self.library_filename(output_libname, + output_dir=output_dir) - if self._need_link (objects, output_filename): + if self._need_link(objects, output_filename): lib_args = objects + ['/OUT:' + output_filename] if debug: - pass # XXX what goes here? + pass # XXX what goes here? try: - self.spawn ([self.lib] + lib_args) - except DistutilsExecError, msg: - raise LibError, msg - + self.spawn([self.lib] + lib_args) + except DistutilsExecError as msg: + raise LibError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - # create_static_lib () - - def link (self, - target_desc, - objects, - output_filename, - output_dir=None, - libraries=None, - library_dirs=None, - runtime_library_dirs=None, - export_symbols=None, - debug=0, - extra_preargs=None, - extra_postargs=None, - build_temp=None, - target_lang=None): - - if not self.initialized: self.initialize() - (objects, output_dir) = self._fix_object_args (objects, output_dir) - (libraries, library_dirs, runtime_library_dirs) = \ - self._fix_lib_args (libraries, library_dirs, runtime_library_dirs) + + def link(self, + target_desc, + objects, + output_filename, + output_dir=None, + libraries=None, + library_dirs=None, + runtime_library_dirs=None, + export_symbols=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + build_temp=None, + target_lang=None): + + if not self.initialized: + self.initialize() + (objects, output_dir) = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + (libraries, library_dirs, runtime_library_dirs) = fixed_args if runtime_library_dirs: self.warn ("I don't know what to do with 'runtime_library_dirs': " + str (runtime_library_dirs)) - lib_opts = gen_lib_options (self, - library_dirs, runtime_library_dirs, - libraries) + lib_opts = gen_lib_options(self, + library_dirs, runtime_library_dirs, + libraries) if output_dir is not None: - output_filename = os.path.join (output_dir, output_filename) - - if self._need_link (objects, output_filename): + output_filename = os.path.join(output_dir, output_filename) + if self._need_link(objects, output_filename): if target_desc == CCompiler.EXECUTABLE: if debug: ldflags = self.ldflags_shared_debug[1:] @@ -529,34 +519,32 @@ class MSVCCompiler (CCompiler) : if extra_postargs: ld_args.extend(extra_postargs) - self.mkpath (os.path.dirname (output_filename)) + self.mkpath(os.path.dirname(output_filename)) try: - self.spawn ([self.linker] + ld_args) - except DistutilsExecError, msg: - raise LinkError, msg + self.spawn([self.linker] + ld_args) + except DistutilsExecError as msg: + raise LinkError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - # link () - # -- Miscellaneous methods ----------------------------------------- # These are all used by the 'gen_lib_options() function, in # ccompiler.py. - def library_dir_option (self, dir): + def library_dir_option(self, dir): return "/LIBPATH:" + dir - def runtime_library_dir_option (self, dir): - raise DistutilsPlatformError, \ - "don't know how to set runtime library search path for MSVC++" + def runtime_library_dir_option(self, dir): + raise DistutilsPlatformError( + "don't know how to set runtime library search path for MSVC++") - def library_option (self, lib): - return self.library_filename (lib) + def library_option(self, lib): + return self.library_filename(lib) - def find_library_file (self, dirs, lib, debug=0): + def find_library_file(self, dirs, lib, debug=0): # Prefer a debugging library if found (and requested), but deal # with it if we don't have one. if debug: @@ -572,8 +560,6 @@ class MSVCCompiler (CCompiler) : # Oops, didn't find it in *any* of 'dirs' return None - # find_library_file () - # Helper methods for using the MSVC registry settings def find_exe(self, exe): @@ -585,14 +571,13 @@ class MSVCCompiler (CCompiler) : absolute path that is known to exist. If none of them work, just return the original program name, 'exe'. """ - for p in self.__paths: fn = os.path.join(os.path.abspath(p), exe) if os.path.isfile(fn): return fn # didn't find it; try existing path - for p in string.split(os.environ['Path'],';'): + for p in os.environ['Path'].split(';'): fn = os.path.join(os.path.abspath(p),exe) if os.path.isfile(fn): return fn @@ -605,7 +590,6 @@ class MSVCCompiler (CCompiler) : Return a list of strings. The list will be empty if unable to access the registry or appropriate registry keys not found. """ - if not _can_read_reg: return [] @@ -621,9 +605,9 @@ class MSVCCompiler (CCompiler) : d = read_values(base, key) if d: if self.__version >= 7: - return string.split(self.__macros.sub(d[path]), ";") + return self.__macros.sub(d[path]).split(";") else: - return string.split(d[path], ";") + return d[path].split(";") # MSVC 6 seems to create the registry entries we need only when # the GUI is run. if self.__version == 6: @@ -648,7 +632,7 @@ class MSVCCompiler (CCompiler) : else: p = self.get_msvc_paths(name) if p: - os.environ[name] = string.join(p, ';') + os.environ[name] = ';'.join(p) if get_build_version() >= 8.0: diff --git a/lib-python/2.7/distutils/tests/test_msvccompiler.py b/lib-python/2.7/distutils/tests/test_msvccompiler.py new file mode 100644 index 0000000000..46a51cd0a7 --- /dev/null +++ b/lib-python/2.7/distutils/tests/test_msvccompiler.py @@ -0,0 +1,138 @@ +"""Tests for distutils._msvccompiler.""" +import sys +import unittest +import os +import threading + +from distutils.errors import DistutilsPlatformError +from distutils.tests import support +from test.support import run_unittest + + +SKIP_MESSAGE = (None if sys.platform == "win32" else + "These tests are only for win32") + +@unittest.skipUnless(SKIP_MESSAGE is None, SKIP_MESSAGE) +class msvccompilerTestCase(support.TempdirManager, + unittest.TestCase): + + def test_no_compiler(self): + import distutils._msvccompiler as _msvccompiler + # makes sure query_vcvarsall raises + # a DistutilsPlatformError if the compiler + # is not found + def _find_vcvarsall(plat_spec): + return None, None + + old_find_vcvarsall = _msvccompiler._find_vcvarsall + _msvccompiler._find_vcvarsall = _find_vcvarsall + try: + self.assertRaises(DistutilsPlatformError, + _msvccompiler._get_vc_env, + 'wont find this version') + finally: + _msvccompiler._find_vcvarsall = old_find_vcvarsall + + def test_get_vc_env_unicode(self): + import distutils._msvccompiler as _msvccompiler + + test_var = 'ṰḖṤṪ┅ṼẨṜ' + test_value = '₃⁴₅' + + # Ensure we don't early exit from _get_vc_env + old_distutils_use_sdk = os.environ.pop('DISTUTILS_USE_SDK', None) + os.environ[test_var] = test_value + try: + env = _msvccompiler._get_vc_env('x86') + self.assertIn(test_var.lower(), env) + self.assertEqual(test_value, env[test_var.lower()]) + finally: + os.environ.pop(test_var) + if old_distutils_use_sdk: + os.environ['DISTUTILS_USE_SDK'] = old_distutils_use_sdk + + def test_get_vc2017(self): + import distutils._msvccompiler as _msvccompiler + + # This function cannot be mocked, so pass it if we find VS 2017 + # and mark it skipped if we do not. + version, path = _msvccompiler._find_vc2017() + if version: + self.assertGreaterEqual(version, 15) + self.assertTrue(os.path.isdir(path)) + else: + raise unittest.SkipTest("VS 2017 is not installed") + + def test_get_vc2015(self): + import distutils._msvccompiler as _msvccompiler + + # This function cannot be mocked, so pass it if we find VS 2015 + # and mark it skipped if we do not. + version, path = _msvccompiler._find_vc2015() + if version: + self.assertGreaterEqual(version, 14) + self.assertTrue(os.path.isdir(path)) + else: + raise unittest.SkipTest("VS 2015 is not installed") + + +class CheckThread(threading.Thread): + exc_info = None + + def run(self): + try: + super().run() + except Exception: + self.exc_info = sys.exc_info() + + def __bool__(self): + return not self.exc_info + + +class TestSpawn(unittest.TestCase): + def test_concurrent_safe(self): + """ + Concurrent calls to spawn should have consistent results. + """ + import distutils._msvccompiler as _msvccompiler + compiler = _msvccompiler.MSVCCompiler() + compiler._paths = "expected" + inner_cmd = 'import os; assert os.environ["PATH"] == "expected"' + command = ['python', '-c', inner_cmd] + + threads = [ + CheckThread(target=compiler.spawn, args=[command]) + for n in range(100) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert all(threads) + + def test_concurrent_safe_fallback(self): + """ + If CCompiler.spawn has been monkey-patched without support + for an env, it should still execute. + """ + import distutils._msvccompiler as _msvccompiler + from distutils import ccompiler + compiler = _msvccompiler.MSVCCompiler() + compiler._paths = "expected" + + def CCompiler_spawn(self, cmd): + "A spawn without an env argument." + assert os.environ["PATH"] == "expected" + + with unittest.mock.patch.object( + ccompiler.CCompiler, 'spawn', CCompiler_spawn): + compiler.spawn(["n/a"]) + + assert os.environ.get("PATH") != "expected" + + +def test_suite(): + return unittest.makeSuite(msvccompilerTestCase) + +if __name__ == "__main__": + run_unittest(test_suite()) |