Handle -r flag in requirements.txt

Resolve recursive requirements (ie `-r` flag) in requirements.txt when
`--requirements-max-depth` flag is specified. Behavior remains the same
as before if flag not specified.

Closes arkq#5

README.rst

@@ -60,3 +60,8 @@ Real life example::
   [flake8]
   max-line-length = 100
   known-modules = my-lib:[mylib.drm,mylib.encryption]
+
+If you use the ``-r`` flag in your ``requirements.txt`` file with more than one level of recursion
+(in other words, one file includes another, the included file includes yet another, and so on),
+add the ``--requirements-max-depth`` option to flake8 (for example, ``--requirements-max-depth=3``
+to allow three levels of recursion).

setup.py

@@ -31,7 +31,7 @@ def get_abs_path(pathname):
         "setuptools",
     ],
     setup_requires=["pytest-runner"],
-    tests_require=["pytest"],
+    tests_require=["mock", "pytest"],
     entry_points={
         'flake8.extension': [
             'I90 = flake8_requirements:Flake8Checker',

src/flake8_requirements/checker.py

@@ -14,7 +14,7 @@
 from .modules import STDLIB_PY3
 
 # NOTE: Changing this number will alter package version as well.
-__version__ = "1.1.2"
+__version__ = "1.2.0"
 __license__ = "MIT"
 
 LOG = getLogger('flake8.plugin.requirements')
@@ -252,6 +252,9 @@ class Flake8Checker(object):
     # User defined project->modules mapping.
     known_modules = {}
 
+    # max depth to resolve recursive requirements
+    requirements_max_depth = 1
+
     def __init__(self, tree, filename, lines=None):
         """Initialize requirements checker."""
         self.tree = tree
@@ -277,6 +280,16 @@ def add_options(cls, manager):
             ),
             **kw
         )
+        manager.add_option(
+            "--requirements-max-depth",
+            type="int",
+            default=1,
+            help=(
+                "Max depth to resolve recursive requirements. Defaults to 1 "
+                "(one level of recursion allowed)."
+            ),
+            **kw
+        )
 
     @classmethod
     def parse_options(cls, options):
@@ -288,17 +301,41 @@ def parse_options(cls, options):
                 for x in re.split(r"],?", options.known_modules)[:-1]
             ]
         }
+        cls.requirements_max_depth = options.requirements_max_depth
 
     @classmethod
     @memoize
     def get_requirements(cls):
         """Get package requirements."""
-        try:
-            with open("requirements.txt") as f:
-                return tuple(parse_requirements(f.readlines()))
-        except IOError as e:
-            LOG.debug("Couldn't open requirements file: %s", e)
+        if not os.path.exists("requirements.txt"):
+            LOG.debug("No requirements.txt file")
             return ()
+        return tuple(parse_requirements(cls.resolve_requirement(
+            "-r requirements.txt", cls.requirements_max_depth + 1)))
+
+    @classmethod
+    def resolve_requirement(cls, requirement, max_depth=0):
+        """Resolves flags like -r in an individual requirement line."""
+        requirement = requirement.strip()
+
+        if requirement.startswith("#") or not requirement:
+            return []
+
+        if requirement.startswith("-r "):
+            # Error out if we need to recurse deeper than allowed.
+            if max_depth <= 0:
+                msg = "Cannot resolve {}: beyond max depth"
+                raise RuntimeError(msg.format(requirement.strip()))
+
+            resolved = []
+            # Error out if requirements file cannot be opened.
+            with open(requirement[3:].lstrip()) as f:
+                for line in f.readlines():
+                    resolved.extend(cls.resolve_requirement(
+                        line, max_depth - 1))
+            return resolved
+
+        return [requirement]
 
     @classmethod
     @memoize

test/test_checker.py

@@ -105,6 +105,7 @@ def test_relative(self):
     def test_custom_mapping_parser(self):
         class Flake8Options:
             known_modules = ":[pydrmcodec],mylib:[mylib.drm,mylib.ex]"
+            requirements_max_depth = 0
         Flake8Checker.parse_options(Flake8Options)
         self.assertEqual(
             Flake8Checker.known_modules,
@@ -114,6 +115,7 @@ class Flake8Options:
     def test_custom_mapping(self):
         class Flake8Options:
             known_modules = "flake8-requires:[flake8req]"
+            requirements_max_depth = 0
         Flake8Checker.parse_options(Flake8Options)
         errors = check("from flake8req import mymodule")
         self.assertEqual(len(errors), 0)

test/test_requirements.py

@@ -0,0 +1,143 @@
+from unittest import TestCase
+
+from pkg_resources import parse_requirements
+
+from flake8_requirements.checker import Flake8Checker
+
+try:
+    from unittest import mock
+    builtins_open = 'builtins.open'
+except ImportError:
+    import mock
+    builtins_open = '__builtin__.open'
+
+
+class RequirementsTestCase(TestCase):
+
+    def test_resolve_requirement_with_blank(self):
+        self.assertEqual(Flake8Checker.resolve_requirement(""), [])
+
+    def test_resolve_requirement_with_comment(self):
+        self.assertEqual(
+            Flake8Checker.resolve_requirement("#-r requirements.txt"),
+            [],
+        )
+
+    def test_resolve_requirement_with_simple(self):
+        self.assertEqual(
+            Flake8Checker.resolve_requirement("foo >= 1.0.0"),
+            ["foo >= 1.0.0"],
+        )
+
+    def test_resolve_requirement_with_file_beyond_max_depth(self):
+        with self.assertRaises(RuntimeError):
+            Flake8Checker.resolve_requirement("-r requirements.txt")
+
+    def test_resolve_requirement_with_file_empty(self):
+        with mock.patch(builtins_open, mock.mock_open()) as m:
+            self.assertEqual(
+                Flake8Checker.resolve_requirement("-r requirements.txt", 1),
+                [],
+            )
+            m.assert_called_once_with("requirements.txt")
+
+    def test_resolve_requirement_with_file_content(self):
+        content = "foo >= 1.0.0\nbar <= 1.0.0\n"
+        with mock.patch(builtins_open, mock.mock_open(read_data=content)):
+            self.assertEqual(
+                Flake8Checker.resolve_requirement("-r requirements.txt", 1),
+                ["foo >= 1.0.0", "bar <= 1.0.0"],
+            )
+
+    def test_resolve_requirement_with_file_recursion_beyond_max_depth(self):
+        content = "-r requirements.txt\n"
+        with mock.patch(builtins_open, mock.mock_open(read_data=content)):
+            with self.assertRaises(RuntimeError):
+                Flake8Checker.resolve_requirement("-r requirements.txt", 1),
+
+    def test_resolve_requirement_with_file_recursion(self):
+        content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"
+        inner_content = "# inner\nbaz\n\nqux\n"
+
+        with mock.patch(builtins_open, mock.mock_open()) as m:
+            m.side_effect = (
+                mock.mock_open(read_data=content).return_value,
+                mock.mock_open(read_data=inner_content).return_value,
+            )
+
+            self.assertEqual(
+                Flake8Checker.resolve_requirement("-r requirements.txt", 2),
+                ["foo >= 1.0.0", "baz", "qux", "bar <= 1.0.0"],
+            )
+
+    def test_init_with_no_requirements(self):
+        with mock.patch("os.path.exists", return_value=False) as exists:
+            with mock.patch(builtins_open, mock.mock_open()) as m:
+                checker = Flake8Checker(None, None)
+                self.assertEqual(checker.requirements, ())
+                m.assert_called_once_with("setup.py")
+            exists.assert_called_once_with("requirements.txt")
+
+    def test_init_with_simple_requirements(self):
+        requirements_content = "foo >= 1.0.0\nbar <= 1.0.0\n"
+        setup_content = ""
+
+        with mock.patch("os.path.exists", return_value=True):
+            with mock.patch(builtins_open, mock.mock_open()) as m:
+                m.side_effect = (
+                    mock.mock_open(read_data=requirements_content).return_value,
+                    mock.mock_open(read_data=setup_content).return_value,
+                )
+
+                checker = Flake8Checker(None, None)
+                self.assertEqual(
+                    sorted(checker.requirements, key=lambda x: x.project_name),
+                    sorted(parse_requirements([
+                        "foo >= 1.0.0",
+                        "bar <= 1.0.0",
+                    ]), key=lambda x: x.project_name),
+                )
+
+    def test_init_with_recursive_requirements_beyond_max_depth(self):
+        requirements_content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"
+        inner_content = "# inner\nbaz\n\nqux\n"
+        setup_content = ""
+
+        with mock.patch("os.path.exists", return_value=True):
+            with mock.patch(builtins_open, mock.mock_open()) as m:
+                m.side_effect = (
+                    mock.mock_open(read_data=requirements_content).return_value,
+                    mock.mock_open(read_data=inner_content).return_value,
+                    mock.mock_open(read_data=setup_content).return_value,
+                )
+
+                with self.assertRaises(RuntimeError):
+                    try:
+                        Flake8Checker.requirements_max_depth = 0
+                        Flake8Checker(None, None)
+                    finally:
+                        Flake8Checker.requirements_max_depth = 1
+
+    def test_init_with_recursive_requirements(self):
+        requirements_content = "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"
+        inner_content = "# inner\nbaz\n\nqux\n"
+        setup_content = ""
+
+        with mock.patch("os.path.exists", return_value=True):
+            with mock.patch(builtins_open, mock.mock_open()) as m:
+                m.side_effect = (
+                    mock.mock_open(read_data=requirements_content).return_value,
+                    mock.mock_open(read_data=inner_content).return_value,
+                    mock.mock_open(read_data=setup_content).return_value,
+                )
+
+                checker = Flake8Checker(None, None)
+                self.assertEqual(
+                    sorted(checker.requirements, key=lambda x: x.project_name),
+                    sorted(parse_requirements([
+                        "foo >= 1.0.0",
+                        "baz",
+                        "qux",
+                        "bar <= 1.0.0",
+                    ]), key=lambda x: x.project_name),
+                )