diff --git a/README.rst b/README.rst index e2434a1..fbad4ee 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,9 @@ Important notice In order to collect project's dependencies, this checker evaluates Python code from the ``setup.py`` file stored in the project's root directory. Code evaluation is done with the `eval() `_ function. As a fall-back -method, this checker also tries to load dependencies from the ``requirements.txt`` file. +method, this checker also tries to load dependencies from the ``pyproject.toml`` file from +the `poetry `_ tool section, or from the ``requirements.txt`` +text file. At this point it is very important to be aware of the consequences of the above approach. One might inject malicious code into the ``setup.py`` file, which will be executed by this checker. diff --git a/setup.py b/setup.py index 9a20e36..165d49f 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ def get_abs_path(pathname): install_requires=[ "flake8 >= 2.0.0", "setuptools", + "toml", ], setup_requires=["pytest-runner"], tests_require=["mock", "pytest"], diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index fbdc120..9420994 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -7,6 +7,7 @@ from logging import getLogger import flake8 +import toml from pkg_resources import parse_requirements from .modules import KNOWN_3RD_PARTIES @@ -340,61 +341,99 @@ def resolve_requirement(cls, requirement, max_depth=0): @classmethod @memoize - def get_requirements(cls): - """Get package requirements.""" - if not os.path.exists("requirements.txt"): - LOG.debug("No requirements.txt file") + def get_pyproject_toml(cls): + """Try to load PEP 518 configuration file.""" + try: + with open("pyproject.toml") as f: + return toml.loads(f.read()) + except IOError as e: + LOG.warning("Couldn't load project setup: %s", e) + return {} + + @classmethod + def get_pyproject_toml_poetry(cls): + """Try to get poetry configuration.""" + cfg_pep518 = cls.get_pyproject_toml() + return cfg_pep518.get('tool', {}).get('poetry', {}) + + @classmethod + @memoize + def get_requirements_txt(cls): + """Try to load requirements from text file.""" + try: + if not os.path.exists("requirements.txt"): + return () + return tuple(parse_requirements(cls.resolve_requirement( + "-r requirements.txt", cls.requirements_max_depth + 1))) + except IOError as e: + LOG.debug("Couldn't load requirements: %s", e) return () - return tuple(parse_requirements(cls.resolve_requirement( - "-r requirements.txt", cls.requirements_max_depth + 1))) + + @classmethod + @memoize + def get_setup_py(cls): + """Try to load standard setup file.""" + try: + with open("setup.py") as f: + return SetupVisitor(ast.parse(f.read())) + except IOError as e: + LOG.debug("Couldn't load project setup: %s", e) + return SetupVisitor(ast.parse("")) @classmethod @memoize def get_mods_1st_party(cls): - setup = cls.get_setup() mods_1st_party = set() # Get 1st party modules (used for absolute imports). - modules = [project2module(setup.keywords.get('name', ""))] + modules = [project2module( + cls.get_setup_py().keywords.get('name') or + cls.get_pyproject_toml_poetry().get('name') or + "")] if modules[0] in cls.known_modules: modules = cls.known_modules[modules[0]] mods_1st_party.update(modsplit(x) for x in modules) return mods_1st_party - @memoize - def get_mods_3rd_party(cls): - setup = cls.get_setup() - requirements = cls.get_requirements() - - if setup.redirected: - # Use requirements from setup if available. - requirements = setup.get_requirements( - setup=cls.processing_setup_py, + def get_mods_3rd_party_requirements(self): + """Get list of 3rd party requirements.""" + + # Use requirements from setup if available. + cfg_setup = self.get_setup_py() + if cfg_setup.redirected: + return cfg_setup.get_requirements( + setup=self.processing_setup_py, tests=True, ) + # Check project configuration for requirements. + cfg_poetry = self.get_pyproject_toml_poetry() + if cfg_poetry: + requirements = [] + requirements.extend(parse_requirements( + cfg_poetry.get('dependencies', ()), + )) + requirements.extend(parse_requirements( + cfg_poetry.get('dev-dependencies', ()), + )) + return requirements + + # Get requirements from text file. + return self.get_requirements_txt() + + @memoize + def get_mods_3rd_party(self): mods_3rd_party = set() # Get 3rd party module names based on requirements. - for requirement in requirements: + for requirement in self.get_mods_3rd_party_requirements(): modules = [project2module(requirement.project_name)] if modules[0] in KNOWN_3RD_PARTIES: modules = KNOWN_3RD_PARTIES[modules[0]] - if modules[0] in cls.known_modules: - modules = cls.known_modules[modules[0]] + if modules[0] in self.known_modules: + modules = self.known_modules[modules[0]] mods_3rd_party.update(modsplit(x) for x in modules) return mods_3rd_party - @classmethod - @memoize - def get_setup(cls): - """Get package setup.""" - try: - with open("setup.py") as f: - return SetupVisitor(ast.parse(f.read())) - except IOError as e: - LOG.warning("Couldn't open setup file: %s", e) - return SetupVisitor(ast.parse("")) - @property def processing_setup_py(self): """Determine whether we are processing setup.py file.""" diff --git a/test/test_checker.py b/test/test_checker.py index 1b4df0f..06fb12f 100644 --- a/test/test_checker.py +++ b/test/test_checker.py @@ -24,7 +24,7 @@ def __init__(self): class Flake8Checker(checker.Flake8Checker): @classmethod - def get_setup(cls): + def get_setup_py(cls): return SetupVisitorMock() @property diff --git a/test/test_poetry.py b/test/test_poetry.py new file mode 100644 index 0000000..551bf3e --- /dev/null +++ b/test/test_poetry.py @@ -0,0 +1,50 @@ +import unittest + +from flake8_requirements.checker import Flake8Checker +from flake8_requirements.checker import memoize + +try: + from unittest import mock + builtins_open = 'builtins.open' +except ImportError: + import mock + builtins_open = '__builtin__.open' + + +class PoetryTestCase(unittest.TestCase): + + def setUp(self): + memoize.mem = {} + + def test_get_pyproject_toml_poetry(self): + content = "[tool.poetry]\nname='x'\n[tool.poetry.tag]\nx=0\n" + with mock.patch(builtins_open, mock.mock_open(read_data=content)): + poetry = Flake8Checker.get_pyproject_toml_poetry() + self.assertDictEqual(poetry, {'name': "x", 'tag': {'x': 0}}) + + def test_1st_party(self): + content = "[tool.poetry]\nname='book'\n" + + with mock.patch(builtins_open, mock.mock_open()) as m: + m.side_effect = ( + IOError("No such file or directory: 'setup.py'"), + mock.mock_open(read_data=content).return_value, + ) + + checker = Flake8Checker(None, None) + mods = checker.get_mods_1st_party() + self.assertEqual(mods, set([("book",)])) + + def test_3rd_party(self): + content = "[tool.poetry.dependencies]\ntools='1.0'\n" + content += "[tool.poetry.dev-dependencies]\ndev-tools='1.0'\n" + + with mock.patch(builtins_open, mock.mock_open()) as m: + m.side_effect = ( + IOError("No such file or directory: 'setup.py'"), + mock.mock_open(read_data=content).return_value, + ) + + checker = Flake8Checker(None, None) + mods = checker.get_mods_3rd_party() + self.assertEqual(mods, set([("tools",), ("dev_tools",)])) diff --git a/test/test_requirements.py b/test/test_requirements.py index 17a5dc3..2a13e20 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -1,4 +1,4 @@ -from unittest import TestCase +import unittest from pkg_resources import parse_requirements @@ -13,7 +13,10 @@ builtins_open = '__builtin__.open' -class RequirementsTestCase(TestCase): +class RequirementsTestCase(unittest.TestCase): + + def setUp(self): + memoize.mem = {} def test_resolve_requirement_with_blank(self): self.assertEqual(Flake8Checker.resolve_requirement(""), []) @@ -73,9 +76,8 @@ def test_resolve_requirement_with_file_recursion(self): def test_init_with_no_requirements(self): with mock.patch("os.path.exists", return_value=False) as exists: - memoize.mem = {} checker = Flake8Checker(None, None) - requirements = checker.get_requirements() + requirements = checker.get_requirements_txt() self.assertEqual(requirements, ()) exists.assert_called_once_with("requirements.txt") @@ -88,9 +90,8 @@ def test_init_with_simple_requirements(self): mock.mock_open(read_data=content).return_value, ) - memoize.mem = {} checker = Flake8Checker(None, None) - requirements = checker.get_requirements() + requirements = checker.get_requirements_txt() self.assertEqual( sorted(requirements, key=lambda x: x.project_name), @@ -113,10 +114,9 @@ def test_init_with_recursive_requirements_beyond_max_depth(self): with self.assertRaises(RuntimeError): try: - memoize.mem = {} Flake8Checker.requirements_max_depth = 0 checker = Flake8Checker(None, None) - checker.get_requirements() + checker.get_requirements_txt() finally: Flake8Checker.requirements_max_depth = 1 @@ -131,9 +131,8 @@ def test_init_with_recursive_requirements(self): mock.mock_open(read_data=inner_content).return_value, ) - memoize.mem = {} checker = Flake8Checker(None, None) - requirements = checker.get_requirements() + requirements = checker.get_requirements_txt() self.assertEqual( sorted(requirements, key=lambda x: x.project_name),