Scan host site-packages directory for 3rd parties

This feature requires flake8 launched with Python >= 2.7 or >= 3.2.

README.rst

@@ -63,6 +63,11 @@ Real life example::
   max-line-length = 100
   known-modules = my-lib:[mylib.drm,mylib.encryption]
 
+It is also possible to scan host's site-packages directory for installed packages. This feature is
+disabled by default, but user can enable it with the ``--scan-host-site-packages`` command line
+option. Please note, however, that the location of the site-packages directory will be determined
+by the Python version used for flake8 execution.
+
 If you use the ``-r`` flag in your ``requirements.txt`` file with more than one level of recursion
 (in other words, one file includes another, the included file includes yet another, and so on),
 add the ``--requirements-max-depth`` option to flake8 (for example, ``--requirements-max-depth=3``

src/flake8_requirements/checker.py

@@ -1,6 +1,7 @@
 import ast
 import os
 import re
+import site
 import sys
 from collections import namedtuple
 from functools import wraps
@@ -9,6 +10,7 @@
 import flake8
 import toml
 from pkg_resources import parse_requirements
+from pkg_resources import yield_lines
 
 from .modules import KNOWN_3RD_PARTIES
 from .modules import STDLIB_PY2
@@ -287,6 +289,9 @@ class Flake8Checker(object):
         for k, v in KNOWN_3RD_PARTIES.items()
     }
 
+    # Host-based mapping for 3rd party modules.
+    known_host_3rd_parties = {}
+
     # User defined project->modules mapping.
     known_modules = {}
 
@@ -329,6 +334,16 @@ def add_options(cls, manager):
             ),
             **kw
         )
+        manager.add_option(
+            "--scan-host-site-packages",
+            action='store_true',
+            help=(
+                "Scan host's site-packages directory for 3rd party projects, "
+                "which provide more than one module or the name of the module"
+                " is different than the project name itself."
+            ),
+            **kw
+        )
 
     @classmethod
     def parse_options(cls, options):
@@ -341,8 +356,39 @@ def parse_options(cls, options):
             ]
         }
         cls.requirements_max_depth = options.requirements_max_depth
+        if options.scan_host_site_packages:
+            cls.discover_host_3rd_party_modules()
         cls.discover_project_root_dir()
 
+    @classmethod
+    def discover_host_3rd_party_modules(cls):
+        """Scan host site-packages for 3rd party modules."""
+        try:
+            site_packages_dirs = site.getsitepackagess()
+            site_packages_dirs.append(site.getusersitepackages())
+        except AttributeError as e:
+            LOG.error("Couldn't get site packages: %s", e)
+            return
+        for site_dir in site_packages_dirs:
+            try:
+                dir_entries = os.listdir(site_dir)
+            except IOError:
+                continue
+            for egg in (x for x in dir_entries if x.endswith(".egg-info")):
+                pkg_info_path = os.path.join(site_dir, egg, "PKG-INFO")
+                modules_path = os.path.join(site_dir, egg, "top_level.txt")
+                if not os.path.isfile(pkg_info_path):
+                    continue
+                with open(pkg_info_path) as f:
+                    name = next(iter(
+                        line.split(":")[1].strip()
+                        for line in yield_lines(f.readlines())
+                        if line.lower().startswith("name:")
+                    ), "")
+                with open(modules_path) as f:
+                    modules = list(yield_lines(f.readlines()))
+                cls.known_host_3rd_parties[project2module(name)] = modules
+
     @classmethod
     def discover_project_root_dir(cls):
         """Discover project's root directory."""
@@ -478,6 +524,8 @@ def get_mods_3rd_party(self):
             modules = [project2module(requirement.project_name)]
             if modules[0] in self.known_3rd_parties:
                 modules = self.known_3rd_parties[modules[0]]
+            if modules[0] in self.known_host_3rd_parties:
+                modules = self.known_host_3rd_parties[modules[0]]
             if modules[0] in self.known_modules:
                 modules = self.known_modules[modules[0]]
             mods_3rd_party.update(modsplit(x) for x in modules)

test/test_checker.py

@@ -32,10 +32,13 @@ def processing_setup_py(self):
         return self.filename == "setup.py"
 
 
+class Flake8Options:
+    known_modules = ""
+    requirements_max_depth = 0
+    scan_host_site_packages = False
+
+
 def check(code, filename="<unknown>", options=None):
-    class Flake8Options:
-        known_modules = ""
-        requirements_max_depth = 0
     if options is None:
         options = Flake8Options
     checker.memoize.mem = {}
@@ -110,20 +113,18 @@ def test_relative(self):
         self.assertEqual(len(errors), 0)
 
     def test_custom_mapping_parser(self):
-        class Flake8Options:
+        class Options(Flake8Options):
             known_modules = ":[pydrmcodec],mylib:[mylib.drm,mylib.ex]"
-            requirements_max_depth = 0
-        Flake8Checker.parse_options(Flake8Options)
+        Flake8Checker.parse_options(Options)
         self.assertEqual(
             Flake8Checker.known_modules,
             {"": ["pydrmcodec"], "mylib": ["mylib.drm", "mylib.ex"]},
         )
 
     def test_custom_mapping(self):
-        class Flake8Options:
+        class Options(Flake8Options):
             known_modules = "flake8-requires:[flake8req]"
-            requirements_max_depth = 0
-        errors = check("from flake8req import mymodule", options=Flake8Options)
+        errors = check("from flake8req import mymodule", options=Options)
         self.assertEqual(len(errors), 0)
 
     def test_setup_py(self):