From 1f21baa017cc63994040af5d03e06619da50ed6d Mon Sep 17 00:00:00 2001 From: avrahamstrax Date: Sun, 29 Dec 2019 14:50:11 +0200 Subject: [PATCH] Cache mods - speed up consecutive execution Closes #6 --- src/flake8_requirements/checker.py | 96 +++++++++++++++++------------- test/test_checker.py | 13 +++- test/test_requirements.py | 9 +-- 3 files changed, 66 insertions(+), 52 deletions(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index c2ab025..fbdc120 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -45,6 +45,18 @@ def w(*args, **kw): memoize.mem = {} +def modsplit(module): + """Split module into submodules.""" + return tuple(module.split(".")) + + +def modcmp(lib=(), test=()): + """Compare import modules.""" + if len(lib) > len(test): + return False + return all(a == b for a, b in zip(lib, test)) + + def project2module(project): """Convert project name into a module name.""" # Name unification in accordance with PEP 426. @@ -261,8 +273,6 @@ def __init__(self, tree, filename, lines=None): self.tree = tree self.filename = filename self.lines = lines - self.requirements = self.get_requirements() - self.setup = self.get_setup() @classmethod def add_options(cls, manager): @@ -338,6 +348,42 @@ def get_requirements(cls): return tuple(parse_requirements(cls.resolve_requirement( "-r requirements.txt", cls.requirements_max_depth + 1))) + @classmethod + @memoize + def get_mods_1st_party(cls): + setup = cls.get_setup() + mods_1st_party = set() + # Get 1st party modules (used for absolute imports). + modules = [project2module(setup.keywords.get('name', ""))] + if modules[0] in cls.known_modules: + modules = cls.known_modules[modules[0]] + mods_1st_party.update(modsplit(x) for x in modules) + return mods_1st_party + + @memoize + def get_mods_3rd_party(cls): + setup = cls.get_setup() + requirements = cls.get_requirements() + + if setup.redirected: + # Use requirements from setup if available. + requirements = setup.get_requirements( + setup=cls.processing_setup_py, + tests=True, + ) + + mods_3rd_party = set() + # Get 3rd party module names based on requirements. + for requirement in requirements: + modules = [project2module(requirement.project_name)] + if modules[0] in KNOWN_3RD_PARTIES: + modules = KNOWN_3RD_PARTIES[modules[0]] + if modules[0] in cls.known_modules: + modules = cls.known_modules[modules[0]] + mods_3rd_party.update(modsplit(x) for x in modules) + + return mods_3rd_party + @classmethod @memoize def get_setup(cls): @@ -359,42 +405,8 @@ def processing_setup_py(self): def run(self): """Run checker.""" - - def split(module): - """Split module into submodules.""" - return tuple(module.split(".")) - - def modcmp(lib=(), test=()): - """Compare import modules.""" - if len(lib) > len(test): - return False - return all(a == b for a, b in zip(lib, test)) - - mods_1st_party = set() - mods_3rd_party = set() - - # Get 1st party modules (used for absolute imports). - modules = [project2module(self.setup.keywords.get('name', ""))] - if modules[0] in self.known_modules: - modules = self.known_modules[modules[0]] - mods_1st_party.update(split(x) for x in modules) - - requirements = self.requirements - if self.setup.redirected: - # Use requirements from setup if available. - requirements = self.setup.get_requirements( - setup=self.processing_setup_py, - tests=True, - ) - - # Get 3rd party module names based on requirements. - for requirement in requirements: - modules = [project2module(requirement.project_name)] - if modules[0] in KNOWN_3RD_PARTIES: - modules = KNOWN_3RD_PARTIES[modules[0]] - if modules[0] in self.known_modules: - modules = self.known_modules[modules[0]] - mods_3rd_party.update(split(x) for x in modules) + mods_1st_party = self.get_mods_1st_party() + mods_3rd_party = self.get_mods_3rd_party() # When processing setup.py file, forcefully add setuptools to the # project requirements. Setuptools might be required to build the @@ -402,12 +414,12 @@ def modcmp(lib=(), test=()): # package is required to run setup.py, so listing it as a setup # requirement would be pointless. if self.processing_setup_py: - mods_3rd_party.add(split("setuptools")) + mods_3rd_party.add(modsplit("setuptools")) for node in ImportVisitor(self.tree).imports: - _mod = split(node.mod) - _alt = split(node.alt) - if any([_mod[0] == x for x in STDLIB]): + _mod = modsplit(node.mod) + _alt = modsplit(node.alt) + if _mod[0] in STDLIB: continue if any([modcmp(x, _mod) or modcmp(x, _alt) for x in mods_1st_party]): diff --git a/test/test_checker.py b/test/test_checker.py index 522a97e..1b4df0f 100644 --- a/test/test_checker.py +++ b/test/test_checker.py @@ -32,7 +32,14 @@ def processing_setup_py(self): return self.filename == "setup.py" -def check(code, filename=""): +def check(code, filename="", options=None): + class Flake8Options: + known_modules = "" + requirements_max_depth = 0 + if options is None: + options = Flake8Options + checker.memoize.mem = {} + Flake8Checker.parse_options(options) return list(Flake8Checker(ast.parse(code), filename).run()) @@ -116,13 +123,13 @@ def test_custom_mapping(self): class Flake8Options: known_modules = "flake8-requires:[flake8req]" requirements_max_depth = 0 - Flake8Checker.parse_options(Flake8Options) - errors = check("from flake8req import mymodule") + errors = check("from flake8req import mymodule", options=Flake8Options) self.assertEqual(len(errors), 0) def test_setup_py(self): errors = check("from setuptools import setup", "setup.py") self.assertEqual(len(errors), 0) + # mods_3rd_party errors = check("from setuptools import setup", "xxx.py") self.assertEqual(len(errors), 1) self.assertEqual( diff --git a/test/test_requirements.py b/test/test_requirements.py index 5c52caf..17a5dc3 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -81,13 +81,11 @@ def test_init_with_no_requirements(self): def test_init_with_simple_requirements(self): content = "foo >= 1.0.0\nbar <= 1.0.0\n" - setup_content = "" with mock.patch("os.path.exists", return_value=True): with mock.patch(builtins_open, mock.mock_open()) as m: m.side_effect = ( mock.mock_open(read_data=content).return_value, - mock.mock_open(read_data=setup_content).return_value, ) memoize.mem = {} @@ -105,35 +103,32 @@ def test_init_with_simple_requirements(self): def test_init_with_recursive_requirements_beyond_max_depth(self): content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n" inner_content = "# inner\nbaz\n\nqux\n" - setup_content = "" with mock.patch("os.path.exists", return_value=True): with mock.patch(builtins_open, mock.mock_open()) as m: m.side_effect = ( mock.mock_open(read_data=content).return_value, mock.mock_open(read_data=inner_content).return_value, - mock.mock_open(read_data=setup_content).return_value, ) with self.assertRaises(RuntimeError): try: memoize.mem = {} Flake8Checker.requirements_max_depth = 0 - Flake8Checker(None, None) + checker = Flake8Checker(None, None) + checker.get_requirements() finally: Flake8Checker.requirements_max_depth = 1 def test_init_with_recursive_requirements(self): content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n" inner_content = "# inner\nbaz\n\nqux\n" - setup_content = "" with mock.patch("os.path.exists", return_value=True): with mock.patch(builtins_open, mock.mock_open()) as m: m.side_effect = ( mock.mock_open(read_data=content).return_value, mock.mock_open(read_data=inner_content).return_value, - mock.mock_open(read_data=setup_content).return_value, ) memoize.mem = {}