diff --git a/README.rst b/README.rst index 7333180..6cb8913 100644 --- a/README.rst +++ b/README.rst @@ -63,6 +63,11 @@ Real life example:: max-line-length = 100 known-modules = my-lib:[mylib.drm,mylib.encryption] +It is also possible to scan host's site-packages directory for installed packages. This feature is +disabled by default, but user can enable it with the ``--scan-host-site-packages`` command line +option. Please note, however, that the location of the site-packages directory will be determined +by the Python version used for flake8 execution. + If you use the ``-r`` flag in your ``requirements.txt`` file with more than one level of recursion (in other words, one file includes another, the included file includes yet another, and so on), add the ``--requirements-max-depth`` option to flake8 (for example, ``--requirements-max-depth=3`` diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index d31437d..729e64a 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -1,6 +1,7 @@ import ast import os import re +import site import sys from collections import namedtuple from functools import wraps @@ -9,6 +10,7 @@ import flake8 import toml from pkg_resources import parse_requirements +from pkg_resources import yield_lines from .modules import KNOWN_3RD_PARTIES from .modules import STDLIB_PY2 @@ -287,6 +289,9 @@ class Flake8Checker(object): for k, v in KNOWN_3RD_PARTIES.items() } + # Host-based mapping for 3rd party modules. + known_host_3rd_parties = {} + # User defined project->modules mapping. known_modules = {} @@ -329,6 +334,16 @@ def add_options(cls, manager): ), **kw ) + manager.add_option( + "--scan-host-site-packages", + action='store_true', + help=( + "Scan host's site-packages directory for 3rd party projects, " + "which provide more than one module or the name of the module" + " is different than the project name itself." + ), + **kw + ) @classmethod def parse_options(cls, options): @@ -341,8 +356,39 @@ def parse_options(cls, options): ] } cls.requirements_max_depth = options.requirements_max_depth + if options.scan_host_site_packages: + cls.discover_host_3rd_party_modules() cls.discover_project_root_dir() + @classmethod + def discover_host_3rd_party_modules(cls): + """Scan host site-packages for 3rd party modules.""" + try: + site_packages_dirs = site.getsitepackagess() + site_packages_dirs.append(site.getusersitepackages()) + except AttributeError as e: + LOG.error("Couldn't get site packages: %s", e) + return + for site_dir in site_packages_dirs: + try: + dir_entries = os.listdir(site_dir) + except IOError: + continue + for egg in (x for x in dir_entries if x.endswith(".egg-info")): + pkg_info_path = os.path.join(site_dir, egg, "PKG-INFO") + modules_path = os.path.join(site_dir, egg, "top_level.txt") + if not os.path.isfile(pkg_info_path): + continue + with open(pkg_info_path) as f: + name = next(iter( + line.split(":")[1].strip() + for line in yield_lines(f.readlines()) + if line.lower().startswith("name:") + ), "") + with open(modules_path) as f: + modules = list(yield_lines(f.readlines())) + cls.known_host_3rd_parties[project2module(name)] = modules + @classmethod def discover_project_root_dir(cls): """Discover project's root directory.""" @@ -478,6 +524,8 @@ def get_mods_3rd_party(self): modules = [project2module(requirement.project_name)] if modules[0] in self.known_3rd_parties: modules = self.known_3rd_parties[modules[0]] + if modules[0] in self.known_host_3rd_parties: + modules = self.known_host_3rd_parties[modules[0]] if modules[0] in self.known_modules: modules = self.known_modules[modules[0]] mods_3rd_party.update(modsplit(x) for x in modules) diff --git a/test/test_checker.py b/test/test_checker.py index 06fb12f..1947134 100644 --- a/test/test_checker.py +++ b/test/test_checker.py @@ -32,10 +32,13 @@ def processing_setup_py(self): return self.filename == "setup.py" +class Flake8Options: + known_modules = "" + requirements_max_depth = 0 + scan_host_site_packages = False + + def check(code, filename="", options=None): - class Flake8Options: - known_modules = "" - requirements_max_depth = 0 if options is None: options = Flake8Options checker.memoize.mem = {} @@ -110,20 +113,18 @@ def test_relative(self): self.assertEqual(len(errors), 0) def test_custom_mapping_parser(self): - class Flake8Options: + class Options(Flake8Options): known_modules = ":[pydrmcodec],mylib:[mylib.drm,mylib.ex]" - requirements_max_depth = 0 - Flake8Checker.parse_options(Flake8Options) + Flake8Checker.parse_options(Options) self.assertEqual( Flake8Checker.known_modules, {"": ["pydrmcodec"], "mylib": ["mylib.drm", "mylib.ex"]}, ) def test_custom_mapping(self): - class Flake8Options: + class Options(Flake8Options): known_modules = "flake8-requires:[flake8req]" - requirements_max_depth = 0 - errors = check("from flake8req import mymodule", options=Flake8Options) + errors = check("from flake8req import mymodule", options=Options) self.assertEqual(len(errors), 0) def test_setup_py(self):