Cache mods - speed up consecutive execution

Closes arkq#6

src/flake8_requirements/checker.py

@@ -45,6 +45,18 @@ def w(*args, **kw):
 memoize.mem = {}
 
 
+def modsplit(module):
+    """Split module into submodules."""
+    return tuple(module.split("."))
+
+
+def modcmp(lib=(), test=()):
+    """Compare import modules."""
+    if len(lib) > len(test):
+        return False
+    return all(a == b for a, b in zip(lib, test))
+
+
 def project2module(project):
     """Convert project name into a module name."""
     # Name unification in accordance with PEP 426.
@@ -261,8 +273,6 @@ def __init__(self, tree, filename, lines=None):
         self.tree = tree
         self.filename = filename
         self.lines = lines
-        self.requirements = self.get_requirements()
-        self.setup = self.get_setup()
 
     @classmethod
     def add_options(cls, manager):
@@ -338,6 +348,42 @@ def get_requirements(cls):
         return tuple(parse_requirements(cls.resolve_requirement(
             "-r requirements.txt", cls.requirements_max_depth + 1)))
 
+    @classmethod
+    @memoize
+    def get_mods_1st_party(cls):
+        setup = cls.get_setup()
+        mods_1st_party = set()
+        # Get 1st party modules (used for absolute imports).
+        modules = [project2module(setup.keywords.get('name', ""))]
+        if modules[0] in cls.known_modules:
+            modules = cls.known_modules[modules[0]]
+        mods_1st_party.update(modsplit(x) for x in modules)
+        return mods_1st_party
+
+    @memoize
+    def get_mods_3rd_party(cls):
+        setup = cls.get_setup()
+        requirements = cls.get_requirements()
+
+        if setup.redirected:
+            # Use requirements from setup if available.
+            requirements = setup.get_requirements(
+                setup=cls.processing_setup_py,
+                tests=True,
+            )
+
+        mods_3rd_party = set()
+        # Get 3rd party module names based on requirements.
+        for requirement in requirements:
+            modules = [project2module(requirement.project_name)]
+            if modules[0] in KNOWN_3RD_PARTIES:
+                modules = KNOWN_3RD_PARTIES[modules[0]]
+            if modules[0] in cls.known_modules:
+                modules = cls.known_modules[modules[0]]
+            mods_3rd_party.update(modsplit(x) for x in modules)
+
+        return mods_3rd_party
+
     @classmethod
     @memoize
     def get_setup(cls):
@@ -359,55 +405,21 @@ def processing_setup_py(self):
 
     def run(self):
         """Run checker."""
-
-        def split(module):
-            """Split module into submodules."""
-            return tuple(module.split("."))
-
-        def modcmp(lib=(), test=()):
-            """Compare import modules."""
-            if len(lib) > len(test):
-                return False
-            return all(a == b for a, b in zip(lib, test))
-
-        mods_1st_party = set()
-        mods_3rd_party = set()
-
-        # Get 1st party modules (used for absolute imports).
-        modules = [project2module(self.setup.keywords.get('name', ""))]
-        if modules[0] in self.known_modules:
-            modules = self.known_modules[modules[0]]
-        mods_1st_party.update(split(x) for x in modules)
-
-        requirements = self.requirements
-        if self.setup.redirected:
-            # Use requirements from setup if available.
-            requirements = self.setup.get_requirements(
-                setup=self.processing_setup_py,
-                tests=True,
-            )
-
-        # Get 3rd party module names based on requirements.
-        for requirement in requirements:
-            modules = [project2module(requirement.project_name)]
-            if modules[0] in KNOWN_3RD_PARTIES:
-                modules = KNOWN_3RD_PARTIES[modules[0]]
-            if modules[0] in self.known_modules:
-                modules = self.known_modules[modules[0]]
-            mods_3rd_party.update(split(x) for x in modules)
+        mods_1st_party = self.get_mods_1st_party()
+        mods_3rd_party = self.get_mods_3rd_party()
 
         # When processing setup.py file, forcefully add setuptools to the
         # project requirements. Setuptools might be required to build the
         # project, even though it is not listed as a requirement - this
         # package is required to run setup.py, so listing it as a setup
         # requirement would be pointless.
         if self.processing_setup_py:
-            mods_3rd_party.add(split("setuptools"))
+            mods_3rd_party.add(modsplit("setuptools"))
 
         for node in ImportVisitor(self.tree).imports:
-            _mod = split(node.mod)
-            _alt = split(node.alt)
-            if any([_mod[0] == x for x in STDLIB]):
+            _mod = modsplit(node.mod)
+            _alt = modsplit(node.alt)
+            if _mod[0] in STDLIB:
                 continue
             if any([modcmp(x, _mod) or modcmp(x, _alt)
                     for x in mods_1st_party]):

test/test_checker.py

@@ -32,7 +32,14 @@ def processing_setup_py(self):
         return self.filename == "setup.py"
 
 
-def check(code, filename="<unknown>"):
+def check(code, filename="<unknown>", options=None):
+    class Flake8Options:
+        known_modules = ""
+        requirements_max_depth = 0
+    if options is None:
+        options = Flake8Options
+    checker.memoize.mem = {}
+    Flake8Checker.parse_options(options)
     return list(Flake8Checker(ast.parse(code), filename).run())
 
 
@@ -116,13 +123,13 @@ def test_custom_mapping(self):
         class Flake8Options:
             known_modules = "flake8-requires:[flake8req]"
             requirements_max_depth = 0
-        Flake8Checker.parse_options(Flake8Options)
-        errors = check("from flake8req import mymodule")
+        errors = check("from flake8req import mymodule", options=Flake8Options)
         self.assertEqual(len(errors), 0)
 
     def test_setup_py(self):
         errors = check("from setuptools import setup", "setup.py")
         self.assertEqual(len(errors), 0)
+        # mods_3rd_party
         errors = check("from setuptools import setup", "xxx.py")
         self.assertEqual(len(errors), 1)
         self.assertEqual(

test/test_requirements.py

@@ -81,13 +81,11 @@ def test_init_with_no_requirements(self):
 
     def test_init_with_simple_requirements(self):
         content = "foo >= 1.0.0\nbar <= 1.0.0\n"
-        setup_content = ""
 
         with mock.patch("os.path.exists", return_value=True):
             with mock.patch(builtins_open, mock.mock_open()) as m:
                 m.side_effect = (
                     mock.mock_open(read_data=content).return_value,
-                    mock.mock_open(read_data=setup_content).return_value,
                 )
 
                 memoize.mem = {}
@@ -105,35 +103,32 @@ def test_init_with_simple_requirements(self):
     def test_init_with_recursive_requirements_beyond_max_depth(self):
         content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"
         inner_content = "# inner\nbaz\n\nqux\n"
-        setup_content = ""
 
         with mock.patch("os.path.exists", return_value=True):
             with mock.patch(builtins_open, mock.mock_open()) as m:
                 m.side_effect = (
                     mock.mock_open(read_data=content).return_value,
                     mock.mock_open(read_data=inner_content).return_value,
-                    mock.mock_open(read_data=setup_content).return_value,
                 )
 
                 with self.assertRaises(RuntimeError):
                     try:
                         memoize.mem = {}
                         Flake8Checker.requirements_max_depth = 0
-                        Flake8Checker(None, None)
+                        checker = Flake8Checker(None, None)
+                        checker.get_requirements()
                     finally:
                         Flake8Checker.requirements_max_depth = 1
 
     def test_init_with_recursive_requirements(self):
         content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"
         inner_content = "# inner\nbaz\n\nqux\n"
-        setup_content = ""
 
         with mock.patch("os.path.exists", return_value=True):
             with mock.patch(builtins_open, mock.mock_open()) as m:
                 m.side_effect = (
                     mock.mock_open(read_data=content).return_value,
                     mock.mock_open(read_data=inner_content).return_value,
-                    mock.mock_open(read_data=setup_content).return_value,
                 )
 
                 memoize.mem = {}