From ad47aad49f41b6c2625ff312c38388eee73556cc Mon Sep 17 00:00:00 2001 From: Justin Ludwig Date: Sun, 24 Nov 2019 21:19:25 -0800 Subject: [PATCH] Handle -r flag in requirements.txt Resolve recursive requirements (ie `-r` flag) in requirements.txt when `--requirements-max-depth` flag is specified. Behavior remains the same as before if flag not specified. Closes #5 --- README.rst | 5 + setup.py | 2 +- src/flake8_requirements/checker.py | 49 ++++++++-- test/test_checker.py | 2 + test/test_requirements.py | 143 +++++++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 7 deletions(-) create mode 100644 test/test_requirements.py diff --git a/README.rst b/README.rst index c026cb5..e2434a1 100644 --- a/README.rst +++ b/README.rst @@ -60,3 +60,8 @@ Real life example:: [flake8] max-line-length = 100 known-modules = my-lib:[mylib.drm,mylib.encryption] + +If you use the ``-r`` flag in your ``requirements.txt`` file with more than one level of recursion +(in other words, one file includes another, the included file includes yet another, and so on), +add the ``--requirements-max-depth`` option to flake8 (for example, ``--requirements-max-depth=3`` +to allow three levels of recursion). diff --git a/setup.py b/setup.py index abae3b7..9a20e36 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ def get_abs_path(pathname): "setuptools", ], setup_requires=["pytest-runner"], - tests_require=["pytest"], + tests_require=["mock", "pytest"], entry_points={ 'flake8.extension': [ 'I90 = flake8_requirements:Flake8Checker', diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 31babce..2cd720a 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -14,7 +14,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "1.1.2" +__version__ = "1.2.0" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') @@ -252,6 +252,9 @@ class Flake8Checker(object): # User defined project->modules mapping. known_modules = {} + # max depth to resolve recursive requirements + requirements_max_depth = 1 + def __init__(self, tree, filename, lines=None): """Initialize requirements checker.""" self.tree = tree @@ -277,6 +280,16 @@ def add_options(cls, manager): ), **kw ) + manager.add_option( + "--requirements-max-depth", + type="int", + default=1, + help=( + "Max depth to resolve recursive requirements. Defaults to 1 " + "(one level of recursion allowed)." + ), + **kw + ) @classmethod def parse_options(cls, options): @@ -288,17 +301,41 @@ def parse_options(cls, options): for x in re.split(r"],?", options.known_modules)[:-1] ] } + cls.requirements_max_depth = options.requirements_max_depth @classmethod @memoize def get_requirements(cls): """Get package requirements.""" - try: - with open("requirements.txt") as f: - return tuple(parse_requirements(f.readlines())) - except IOError as e: - LOG.debug("Couldn't open requirements file: %s", e) + if not os.path.exists("requirements.txt"): + LOG.debug("No requirements.txt file") return () + return tuple(parse_requirements(cls.resolve_requirement( + "-r requirements.txt", cls.requirements_max_depth + 1))) + + @classmethod + def resolve_requirement(cls, requirement, max_depth=0): + """Resolves flags like -r in an individual requirement line.""" + requirement = requirement.strip() + + if requirement.startswith("#") or not requirement: + return [] + + if requirement.startswith("-r "): + # Error out if we need to recurse deeper than allowed. + if max_depth <= 0: + msg = "Cannot resolve {}: beyond max depth" + raise RuntimeError(msg.format(requirement.strip())) + + resolved = [] + # Error out if requirements file cannot be opened. + with open(requirement[3:].lstrip()) as f: + for line in f.readlines(): + resolved.extend(cls.resolve_requirement( + line, max_depth - 1)) + return resolved + + return [requirement] @classmethod @memoize diff --git a/test/test_checker.py b/test/test_checker.py index e67f183..522a97e 100644 --- a/test/test_checker.py +++ b/test/test_checker.py @@ -105,6 +105,7 @@ def test_relative(self): def test_custom_mapping_parser(self): class Flake8Options: known_modules = ":[pydrmcodec],mylib:[mylib.drm,mylib.ex]" + requirements_max_depth = 0 Flake8Checker.parse_options(Flake8Options) self.assertEqual( Flake8Checker.known_modules, @@ -114,6 +115,7 @@ class Flake8Options: def test_custom_mapping(self): class Flake8Options: known_modules = "flake8-requires:[flake8req]" + requirements_max_depth = 0 Flake8Checker.parse_options(Flake8Options) errors = check("from flake8req import mymodule") self.assertEqual(len(errors), 0) diff --git a/test/test_requirements.py b/test/test_requirements.py new file mode 100644 index 0000000..3eb2af5 --- /dev/null +++ b/test/test_requirements.py @@ -0,0 +1,143 @@ +from unittest import TestCase + +from pkg_resources import parse_requirements + +from flake8_requirements.checker import Flake8Checker + +try: + from unittest import mock + builtins_open = 'builtins.open' +except ImportError: + import mock + builtins_open = '__builtin__.open' + + +class RequirementsTestCase(TestCase): + + def test_resolve_requirement_with_blank(self): + self.assertEqual(Flake8Checker.resolve_requirement(""), []) + + def test_resolve_requirement_with_comment(self): + self.assertEqual( + Flake8Checker.resolve_requirement("#-r requirements.txt"), + [], + ) + + def test_resolve_requirement_with_simple(self): + self.assertEqual( + Flake8Checker.resolve_requirement("foo >= 1.0.0"), + ["foo >= 1.0.0"], + ) + + def test_resolve_requirement_with_file_beyond_max_depth(self): + with self.assertRaises(RuntimeError): + Flake8Checker.resolve_requirement("-r requirements.txt") + + def test_resolve_requirement_with_file_empty(self): + with mock.patch(builtins_open, mock.mock_open()) as m: + self.assertEqual( + Flake8Checker.resolve_requirement("-r requirements.txt", 1), + [], + ) + m.assert_called_once_with("requirements.txt") + + def test_resolve_requirement_with_file_content(self): + content = "foo >= 1.0.0\nbar <= 1.0.0\n" + with mock.patch(builtins_open, mock.mock_open(read_data=content)): + self.assertEqual( + Flake8Checker.resolve_requirement("-r requirements.txt", 1), + ["foo >= 1.0.0", "bar <= 1.0.0"], + ) + + def test_resolve_requirement_with_file_recursion_beyond_max_depth(self): + content = "-r requirements.txt\n" + with mock.patch(builtins_open, mock.mock_open(read_data=content)): + with self.assertRaises(RuntimeError): + Flake8Checker.resolve_requirement("-r requirements.txt", 1), + + def test_resolve_requirement_with_file_recursion(self): + content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n" + inner_content = "# inner\nbaz\n\nqux\n" + + with mock.patch(builtins_open, mock.mock_open()) as m: + m.side_effect = ( + mock.mock_open(read_data=content).return_value, + mock.mock_open(read_data=inner_content).return_value, + ) + + self.assertEqual( + Flake8Checker.resolve_requirement("-r requirements.txt", 2), + ["foo >= 1.0.0", "baz", "qux", "bar <= 1.0.0"], + ) + + def test_init_with_no_requirements(self): + with mock.patch("os.path.exists", return_value=False) as exists: + with mock.patch(builtins_open, mock.mock_open()) as m: + checker = Flake8Checker(None, None) + self.assertEqual(checker.requirements, ()) + m.assert_called_once_with("setup.py") + exists.assert_called_once_with("requirements.txt") + + def test_init_with_simple_requirements(self): + requirements_content = "foo >= 1.0.0\nbar <= 1.0.0\n" + setup_content = "" + + with mock.patch("os.path.exists", return_value=True): + with mock.patch(builtins_open, mock.mock_open()) as m: + m.side_effect = ( + mock.mock_open(read_data=requirements_content).return_value, + mock.mock_open(read_data=setup_content).return_value, + ) + + checker = Flake8Checker(None, None) + self.assertEqual( + sorted(checker.requirements, key=lambda x: x.project_name), + sorted(parse_requirements([ + "foo >= 1.0.0", + "bar <= 1.0.0", + ]), key=lambda x: x.project_name), + ) + + def test_init_with_recursive_requirements_beyond_max_depth(self): + requirements_content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n" + inner_content = "# inner\nbaz\n\nqux\n" + setup_content = "" + + with mock.patch("os.path.exists", return_value=True): + with mock.patch(builtins_open, mock.mock_open()) as m: + m.side_effect = ( + mock.mock_open(read_data=requirements_content).return_value, + mock.mock_open(read_data=inner_content).return_value, + mock.mock_open(read_data=setup_content).return_value, + ) + + with self.assertRaises(RuntimeError): + try: + Flake8Checker.requirements_max_depth = 0 + Flake8Checker(None, None) + finally: + Flake8Checker.requirements_max_depth = 1 + + def test_init_with_recursive_requirements(self): + requirements_content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n" + inner_content = "# inner\nbaz\n\nqux\n" + setup_content = "" + + with mock.patch("os.path.exists", return_value=True): + with mock.patch(builtins_open, mock.mock_open()) as m: + m.side_effect = ( + mock.mock_open(read_data=requirements_content).return_value, + mock.mock_open(read_data=inner_content).return_value, + mock.mock_open(read_data=setup_content).return_value, + ) + + checker = Flake8Checker(None, None) + self.assertEqual( + sorted(checker.requirements, key=lambda x: x.project_name), + sorted(parse_requirements([ + "foo >= 1.0.0", + "baz", + "qux", + "bar <= 1.0.0", + ]), key=lambda x: x.project_name), + )