from functools import partial from glob import glob from distutils.util import convert_path import distutils.command.build_py as orig import os import fnmatch import textwrap import io import distutils.errors import itertools import stat import warnings from pathlib import Path from typing import Dict, Iterable, Iterator, List, Optional, Tuple from setuptools._deprecation_warning import SetuptoolsDeprecationWarning from setuptools.extern.more_itertools import unique_everseen def make_writable(target): os.chmod(target, os.stat(target).st_mode | stat.S_IWRITE) class build_py(orig.build_py): """Enhanced 'build_py' command that includes data files with packages The data files are specified via a 'package_data' argument to 'setup()'. See 'setuptools.dist.Distribution' for more details. Also, this version of the 'build_py' command allows you to specify both 'py_modules' and 'packages' in the same setup operation. """ editable_mode: bool = False existing_egg_info_dir: Optional[str] = None #: Private API, internal use only. def finalize_options(self): orig.build_py.finalize_options(self) self.package_data = self.distribution.package_data self.exclude_package_data = self.distribution.exclude_package_data or {} if 'data_files' in self.__dict__: del self.__dict__['data_files'] self.__updated_files = [] def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, link=None, level=1): # Overwrite base class to allow using links if link: infile = str(Path(infile).resolve()) outfile = str(Path(outfile).resolve()) return super().copy_file(infile, outfile, preserve_mode, preserve_times, link, level) def run(self): """Build modules, packages, and copy data files to build directory""" if not (self.py_modules or self.packages) or self.editable_mode: return if self.py_modules: self.build_modules() if self.packages: self.build_packages() self.build_package_data() # Only compile actual .py files, using our base class' idea of what our # output files are. self.byte_compile(orig.build_py.get_outputs(self, include_bytecode=0)) def __getattr__(self, attr): "lazily compute data files" if attr == 'data_files': self.data_files = self._get_data_files() return self.data_files return orig.build_py.__getattr__(self, attr) def build_module(self, module, module_file, package): outfile, copied = orig.build_py.build_module(self, module, module_file, package) if copied: self.__updated_files.append(outfile) return outfile, copied def _get_data_files(self): """Generate list of '(package,src_dir,build_dir,filenames)' tuples""" self.analyze_manifest() return list(map(self._get_pkg_data_files, self.packages or ())) def get_data_files_without_manifest(self): """ Generate list of ``(package,src_dir,build_dir,filenames)`` tuples, but without triggering any attempt to analyze or build the manifest. """ # Prevent eventual errors from unset `manifest_files` # (that would otherwise be set by `analyze_manifest`) self.__dict__.setdefault('manifest_files', {}) return list(map(self._get_pkg_data_files, self.packages or ())) def _get_pkg_data_files(self, package): # Locate package source directory src_dir = self.get_package_dir(package) # Compute package build directory build_dir = os.path.join(*([self.build_lib] + package.split('.'))) # Strip directory from globbed filenames filenames = [ os.path.relpath(file, src_dir) for file in self.find_data_files(package, src_dir) ] return package, src_dir, build_dir, filenames def find_data_files(self, package, src_dir): """Return filenames for package's data files in 'src_dir'""" patterns = self._get_platform_patterns( self.package_data, package, src_dir, ) globs_expanded = map(partial(glob, recursive=True), patterns) # flatten the expanded globs into an iterable of matches globs_matches = itertools.chain.from_iterable(globs_expanded) glob_files = filter(os.path.isfile, globs_matches) files = itertools.chain( self.manifest_files.get(package, []), glob_files, ) return self.exclude_data_files(package, src_dir, files) def get_outputs(self, include_bytecode=1) -> List[str]: """See :class:`setuptools.commands.build.SubCommand`""" if self.editable_mode: return list(self.get_output_mapping().keys()) return super().get_outputs(include_bytecode) def get_output_mapping(self) -> Dict[str, str]: """See :class:`setuptools.commands.build.SubCommand`""" mapping = itertools.chain( self._get_package_data_output_mapping(), self._get_module_mapping(), ) return dict(sorted(mapping, key=lambda x: x[0])) def _get_module_mapping(self) -> Iterator[Tuple[str, str]]: """Iterate over all modules producing (dest, src) pairs.""" for (package, module, module_file) in self.find_all_modules(): package = package.split('.') filename = self.get_module_outfile(self.build_lib, package, module) yield (filename, module_file) def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]: """Iterate over package data producing (dest, src) pairs.""" for package, src_dir, build_dir, filenames in self.data_files: for filename in filenames: target = os.path.join(build_dir, filename) srcfile = os.path.join(src_dir, filename) yield (target, srcfile) def build_package_data(self): """Copy data files into build directory""" for target, srcfile in self._get_package_data_output_mapping(): self.mkpath(os.path.dirname(target)) _outf, _copied = self.copy_file(srcfile, target) make_writable(target) def analyze_manifest(self): self.manifest_files = mf = {} if not self.distribution.include_package_data: return src_dirs = {} for package in self.packages or (): # Locate package source directory src_dirs[assert_relative(self.get_package_dir(package))] = package if ( getattr(self, 'existing_egg_info_dir', None) and Path(self.existing_egg_info_dir, "SOURCES.txt").exists() ): egg_info_dir = self.existing_egg_info_dir manifest = Path(egg_info_dir, "SOURCES.txt") files = manifest.read_text(encoding="utf-8").splitlines() else: self.run_command('egg_info') ei_cmd = self.get_finalized_command('egg_info') egg_info_dir = ei_cmd.egg_info files = ei_cmd.filelist.files check = _IncludePackageDataAbuse() for path in self._filter_build_files(files, egg_info_dir): d, f = os.path.split(assert_relative(path)) prev = None oldf = f while d and d != prev and d not in src_dirs: prev = d d, df = os.path.split(d) f = os.path.join(df, f) if d in src_dirs: if f == oldf: if check.is_module(f): continue # it's a module, not data else: importable = check.importable_subpackage(src_dirs[d], f) if importable: check.warn(importable) mf.setdefault(src_dirs[d], []).append(path) def _filter_build_files(self, files: Iterable[str], egg_info: str) -> Iterator[str]: """ ``build_meta`` may try to create egg_info outside of the project directory, and this can be problematic for certain plugins (reported in issue #3500). Extensions might also include between their sources files created on the ``build_lib`` and ``build_temp`` directories. This function should filter this case of invalid files out. """ build = self.get_finalized_command("build") build_dirs = (egg_info, self.build_lib, build.build_temp, build.build_base) norm_dirs = [os.path.normpath(p) for p in build_dirs if p] for file in files: norm_path = os.path.normpath(file) if not os.path.isabs(file) or all(d not in norm_path for d in norm_dirs): yield file def get_data_files(self): pass # Lazily compute data files in _get_data_files() function. def check_package(self, package, package_dir): """Check namespace packages' __init__ for declare_namespace""" try: return self.packages_checked[package] except KeyError: pass init_py = orig.build_py.check_package(self, package, package_dir) self.packages_checked[package] = init_py if not init_py or not self.distribution.namespace_packages: return init_py for pkg in self.distribution.namespace_packages: if pkg == package or pkg.startswith(package + '.'): break else: return init_py with io.open(init_py, 'rb') as f: contents = f.read() if b'declare_namespace' not in contents: raise distutils.errors.DistutilsError( "Namespace package problem: %s is a namespace package, but " "its\n__init__.py does not call declare_namespace()! Please " 'fix it.\n(See the setuptools manual under ' '"Namespace Packages" for details.)\n"' % (package,) ) return init_py def initialize_options(self): self.packages_checked = {} orig.build_py.initialize_options(self) self.editable_mode = False self.existing_egg_info_dir = None def get_package_dir(self, package): res = orig.build_py.get_package_dir(self, package) if self.distribution.src_root is not None: return os.path.join(self.distribution.src_root, res) return res def exclude_data_files(self, package, src_dir, files): """Filter filenames for package's data files in 'src_dir'""" files = list(files) patterns = self._get_platform_patterns( self.exclude_package_data, package, src_dir, ) match_groups = (fnmatch.filter(files, pattern) for pattern in patterns) # flatten the groups of matches into an iterable of matches matches = itertools.chain.from_iterable(match_groups) bad = set(matches) keepers = (fn for fn in files if fn not in bad) # ditch dupes return list(unique_everseen(keepers)) @staticmethod def _get_platform_patterns(spec, package, src_dir): """ yield platform-specific path patterns (suitable for glob or fn_match) from a glob-based spec (such as self.package_data or self.exclude_package_data) matching package in src_dir. """ raw_patterns = itertools.chain( spec.get('', []), spec.get(package, []), ) return ( # Each pattern has to be converted to a platform-specific path os.path.join(src_dir, convert_path(pattern)) for pattern in raw_patterns ) def assert_relative(path): if not os.path.isabs(path): return path from distutils.errors import DistutilsSetupError msg = ( textwrap.dedent( """ Error: setup script specifies an absolute path: %s setup() arguments must *always* be /-separated paths relative to the setup.py directory, *never* absolute paths. """ ).lstrip() % path ) raise DistutilsSetupError(msg) class _IncludePackageDataAbuse: """Inform users that package or module is included as 'data file'""" MESSAGE = """\ Installing {importable!r} as data is deprecated, please list it in `packages`. !!\n\n ############################ # Package would be ignored # ############################ Python recognizes {importable!r} as an importable package, but it is not listed in the `packages` configuration of setuptools. {importable!r} has been automatically added to the distribution only because it may contain data files, but this behavior is likely to change in future versions of setuptools (and therefore is considered deprecated). Please make sure that {importable!r} is included as a package by using the `packages` configuration field or the proper discovery methods (for example by using `find_namespace_packages(...)`/`find_namespace:` instead of `find_packages(...)`/`find:`). You can read more about "package discovery" and "data files" on setuptools documentation page. \n\n!! """ def __init__(self): self._already_warned = set() def is_module(self, file): return file.endswith(".py") and file[:-len(".py")].isidentifier() def importable_subpackage(self, parent, file): pkg = Path(file).parent parts = list(itertools.takewhile(str.isidentifier, pkg.parts)) if parts: return ".".join([parent, *parts]) return None def warn(self, importable): if importable not in self._already_warned: msg = textwrap.dedent(self.MESSAGE).format(importable=importable) warnings.warn(msg, SetuptoolsDeprecationWarning, stacklevel=2) self._already_warned.add(importable)