From 978e2057656adf6f2b05dd554d79dc22a372c9a6 Mon Sep 17 00:00:00 2001 From: Matthew Gamble <4494785+mwgamble@users.noreply.github.com> Date: Thu, 12 Jan 2023 19:48:27 +1100 Subject: [PATCH 01/18] Switch from 'toml' package to tomli/tomllib Python 3.11 comes with built-in support for TOML. It's an almost direct import of the tomli package. As the 'toml' library is unmaintained, and tomli has the same interface as what's in the standard library, this switches the TOML support to a supported library with a minimum of fuss. Tomllib requires files to be opened in 'rb' mode, so the relevant tests had to be updated accordingly. --- setup.py | 2 +- src/flake8_requirements/checker.py | 15 ++++++++++----- test/test_pep621.py | 4 ++-- test/test_poetry.py | 12 ++++++------ 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 22ec537..8e7f39c 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def get_abs_path(pathname): install_requires=[ "flake8 >= 2.0.0", "setuptools >= 10.0.0", - "toml >= 0.7.0", + "tomli>=1.2.1; python_version < '3.11'", ], setup_requires=["pytest-runner"], tests_require=["mock", "pytest"], diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index d16aa93..61acaa6 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -9,16 +9,20 @@ from logging import getLogger import flake8 -import toml from pkg_resources import parse_requirements from pkg_resources import yield_lines +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + from .modules import KNOWN_3RD_PARTIES from .modules import STDLIB_PY2 from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "1.7.5" +__version__ = "1.7.6" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') @@ -545,10 +549,11 @@ def resolve_requirement(cls, requirement, max_depth=0, path=None): @memoize def get_pyproject_toml(cls): """Try to load PEP 518 configuration file.""" + pyproject_config_path = os.path.join(cls.root_dir, "pyproject.toml") try: - with open(os.path.join(cls.root_dir, "pyproject.toml")) as f: - return toml.loads(f.read()) - except (IOError, toml.TomlDecodeError) as e: + with open(pyproject_config_path, mode="rb") as f: + return tomllib.load(f) + except (IOError, tomllib.TOMLDecodeError) as e: LOG.debug("Couldn't load project setup: %s", e) return {} diff --git a/test/test_pep621.py b/test/test_pep621.py index cf0a52a..b74a46e 100644 --- a/test/test_pep621.py +++ b/test/test_pep621.py @@ -21,7 +21,7 @@ class Flake8Options: class Pep621TestCase(unittest.TestCase): - content = """ + content = b""" [project] name="test" dependencies=["tools==1.0"] @@ -58,7 +58,7 @@ def test_get_pyproject_toml_pep621(self): self.assertDictEqual(pep621, expected) def test_get_pyproject_toml_invalid(self): - content = self.content + "invalid" + content = self.content + b"invalid" with mock.patch(builtins_open, mock.mock_open(read_data=content)): self.assertDictEqual(Flake8Checker.get_pyproject_toml_pep621(), {}) diff --git a/test/test_poetry.py b/test/test_poetry.py index a387b7d..5637b12 100644 --- a/test/test_poetry.py +++ b/test/test_poetry.py @@ -18,13 +18,13 @@ def setUp(self): memoize.mem = {} def test_get_pyproject_toml_poetry(self): - content = "[tool.poetry]\nname='x'\n[tool.poetry.tag]\nx=0\n" + content = b"[tool.poetry]\nname='x'\n[tool.poetry.tag]\nx=0\n" with mock.patch(builtins_open, mock.mock_open(read_data=content)): poetry = Flake8Checker.get_pyproject_toml_poetry() self.assertDictEqual(poetry, {'name': "x", 'tag': {'x': 0}}) def test_1st_party(self): - content = "[tool.poetry]\nname='book'\n" + content = b"[tool.poetry]\nname='book'\n" with mock.patch(builtins_open, mock.mock_open()) as m: m.side_effect = ( @@ -38,8 +38,8 @@ def test_1st_party(self): self.assertEqual(mods, ModuleSet({"book": {}})) def test_3rd_party(self): - content = "[tool.poetry.dependencies]\ntools='1.0'\n" - content += "[tool.poetry.dev-dependencies]\ndev-tools='1.0'\n" + content = b"[tool.poetry.dependencies]\ntools='1.0'\n" + content += b"[tool.poetry.dev-dependencies]\ndev-tools='1.0'\n" with mock.patch(builtins_open, mock.mock_open()) as m: m.side_effect = ( @@ -53,8 +53,8 @@ def test_3rd_party(self): self.assertEqual(mods, ModuleSet({"tools": {}, "dev_tools": {}})) def test_3rd_party_groups(self): - content = "[tool.poetry.dependencies]\ntools='1.0'\n" - content += "[tool.poetry.group.dev.dependencies]\ndev-tools='1.0'\n" + content = b"[tool.poetry.dependencies]\ntools='1.0'\n" + content += b"[tool.poetry.group.dev.dependencies]\ndev-tools='1.0'\n" with mock.patch(builtins_open, mock.mock_open()) as m: m.side_effect = ( From 9f6c01cece186ac1954f51ddb161b48521415e87 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Fri, 3 Feb 2023 18:32:24 +0100 Subject: [PATCH 02/18] Update known 3rd parties: plotly, pycryptodome --- src/flake8_requirements/checker.py | 2 +- src/flake8_requirements/modules.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 61acaa6..60d2800 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -22,7 +22,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "1.7.6" +__version__ = "1.7.7" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') diff --git a/src/flake8_requirements/modules.py b/src/flake8_requirements/modules.py index c390c35..57ace43 100644 --- a/src/flake8_requirements/modules.py +++ b/src/flake8_requirements/modules.py @@ -657,11 +657,18 @@ "pillow": ["PIL"], "pillow-simd": ["PIL"], "pip-tools": ["piptools"], + "plotly": [ + "jupyterlab_plotly", + "plotly", + "_plotly_utils", + "_plotly_future_", + ], "progressbar2": ["progressbar"], "protobuf": ["google.protobuf"], "psycopg2-binary": ["psycopg2"], "py-lru-cache": ["lru"], "pycrypto": ["Crypto"], + "pycryptodome": ["Crypto"], "pygithub": ["github"], "pygobject": ["gi", "pygtkcompat"], "pyhamcrest": ["hamcrest"], From bac0b042b17227e43773a58a296bc6d4502d1302 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Mon, 15 May 2023 17:45:49 +0200 Subject: [PATCH 03/18] Fix auto-detected module name generation In most cases conventional "python-" prefix used in project names is not used in exported module names. However, there are some exceptions and we have to account for that. Fixes #74 --- src/flake8_requirements/checker.py | 40 ++++++++++++++++++------------ src/flake8_requirements/modules.py | 8 ------ test/test_checker.py | 5 ++++ 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 60d2800..4bb84e0 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -22,7 +22,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "1.7.7" +__version__ = "1.7.8" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') @@ -59,14 +59,14 @@ def modsplit(module): return tuple(module.split(".")) -def project2module(project): - """Convert project name into a module name.""" +def project2modules(project): + """Convert project name into auto-detected module names.""" # Name unification in accordance with PEP 426. - project = project.lower().replace("-", "_") - if project.startswith("python_"): + modules = [project.lower().replace("-", "_")] + if modules[0].startswith("python_"): # Remove conventional "python-" prefix. - project = project[7:] - return project + modules.append(modules[0][7:]) + return modules def joinlines(lines): @@ -302,8 +302,9 @@ class Flake8Checker(object): # Build-in mapping for known 3rd party modules. known_3rd_parties = { - project2module(k): v + k: v for k, v in KNOWN_3RD_PARTIES.items() + for k in project2modules(k) } # Host-based mapping for 3rd party modules. @@ -382,15 +383,17 @@ def parse_options(cls, options): if isinstance(options.known_modules, dict): # Support for nicer known-modules using flake8-pyproject. cls.known_modules = { - project2module(k): v for k, v in options.known_modules.items() + k: v + for k, v in options.known_modules.items() + for k in project2modules(k) } else: cls.known_modules = { - project2module(k): v.split(",") + k: v.split(",") for k, v in [ x.split(":[") - for x in re.split(r"],?", options.known_modules)[:-1] - ] + for x in re.split(r"],?", options.known_modules)[:-1]] + for k in project2modules(k) } cls.requirements_file = options.requirements_file cls.requirements_max_depth = options.requirements_max_depth @@ -426,7 +429,8 @@ def discover_host_3rd_party_modules(): ), "") with open(modules_path) as f: modules = list(yield_lines(f.readlines())) - mapping[project2module(name)] = modules + for name in project2modules(name): + mapping[name] = modules return mapping @staticmethod @@ -668,12 +672,15 @@ def get_setup_py_requirements(cls, is_setup_py): def get_mods_1st_party(cls): mods_1st_party = ModuleSet() # Get 1st party modules (used for absolute imports). - modules = [project2module( + modules = project2modules( cls.get_setup_py().keywords.get('name') or cls.get_setup_cfg().get('metadata', 'name') or cls.get_pyproject_toml_pep621().get('name') or cls.get_pyproject_toml_poetry().get('name') or - "")] + "") + # Use known module mappings to correct auto-detected name. Please note + # that we're using the first module name only, since all mappings shall + # contain all possible auto-detected module names. if modules[0] in cls.known_modules: modules = cls.known_modules[modules[0]] for module in modules: @@ -686,7 +693,8 @@ def get_mods_3rd_party(cls, is_setup_py): mods_3rd_party = ModuleSet() # Get 3rd party module names based on requirements. for requirement in cls.get_mods_3rd_party_requirements(is_setup_py): - modules = [project2module(requirement.project_name)] + modules = project2modules(requirement.project_name) + # Use known module mappings to correct auto-detected module name. if modules[0] in cls.known_modules: modules = cls.known_modules[modules[0]] elif modules[0] in cls.known_3rd_parties: diff --git a/src/flake8_requirements/modules.py b/src/flake8_requirements/modules.py index 57ace43..d0c0d0e 100644 --- a/src/flake8_requirements/modules.py +++ b/src/flake8_requirements/modules.py @@ -684,16 +684,8 @@ "pyside6": ["PySide6"], "pytest": ["pytest", "_pytest"], "pytest-runner": ["ptr"], - "python-dateutil": ["dateutil"], - "python-docx": ["docx"], - "python-dotenv": ["dotenv"], - "python-hcl2": ["hcl2"], - "python-jose": ["jose"], "python-levenshtein": ["Levenshtein"], "python-lsp-jsonrpc": ["pylsp_jsonrpc"], - "python-magic": ["magic"], - "python-pptx": ["pptx"], - "python-socketio": ["socketio"], "pyturbojpeg": ["turbojpeg"], "pyyaml": ["yaml"], "scikit-fda": ["skfda"], diff --git a/test/test_checker.py b/test/test_checker.py index b0483c0..fb7d10b 100644 --- a/test/test_checker.py +++ b/test/test_checker.py @@ -15,6 +15,7 @@ def __init__(self): "bar", "hyp-hen", "python-boom", + "python-snake", "pillow", "space.module", ], @@ -90,6 +91,10 @@ def test_3rd_party_python_prefix(self): errors = check("from boom import blast") self.assertEqual(len(errors), 0) + def test_3rd_party_python_prefix_no_strip(self): + errors = check("import python_snake as snake") + self.assertEqual(len(errors), 0) + def test_3rd_party_missing(self): errors = check("import os\nfrom cat import Cat") self.assertEqual(len(errors), 1) From 8d58340b127ebf438a2a54df9146340cc12b98d5 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Mon, 15 May 2023 22:25:00 +0200 Subject: [PATCH 04/18] Drop support for unmaintained python2 Closes #71 --- setup.cfg | 2 +- setup.py | 6 +- src/flake8_requirements/checker.py | 30 ++- src/flake8_requirements/modules.py | 282 ----------------------------- test/test_pep621.py | 21 +-- test/test_poetry.py | 23 +-- test/test_requirements.py | 39 ++-- test/test_setup.py | 11 +- 8 files changed, 51 insertions(+), 363 deletions(-) diff --git a/setup.cfg b/setup.cfg index e096a8c..2b5b9ac 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bdist_wheel] -universal = 1 +python-tag = py3 [doc8] max-line-length = 99 diff --git a/setup.py b/setup.py index 8e7f39c..4e2a679 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,3 @@ -from __future__ import with_statement - import re from os import path @@ -27,7 +25,7 @@ def get_abs_path(pathname): package_dir={'': "src"}, packages=["flake8_requirements"], install_requires=[ - "flake8 >= 2.0.0", + "flake8 >= 4.0.0", "setuptools >= 10.0.0", "tomli>=1.2.1; python_version < '3.11'", ], @@ -43,8 +41,8 @@ def get_abs_path(pathname): "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python", - "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only" "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance", ], diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 4bb84e0..3bb1429 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -8,7 +8,6 @@ from functools import wraps from logging import getLogger -import flake8 from pkg_resources import parse_requirements from pkg_resources import yield_lines @@ -18,11 +17,10 @@ import tomli as tomllib from .modules import KNOWN_3RD_PARTIES -from .modules import STDLIB_PY2 from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "1.7.8" +__version__ = "2.0.0" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') @@ -33,10 +31,7 @@ } STDLIB = set() -if sys.version_info[0] == 2: - STDLIB.update(STDLIB_PY2) -if sys.version_info[0] == 3: - STDLIB.update(STDLIB_PY3) +STDLIB.update(STDLIB_PY3) def memoize(f): @@ -334,48 +329,45 @@ def __init__(self, tree, filename, lines=None): @classmethod def add_options(cls, manager): """Register plug-in specific options.""" - kw = {} - if flake8.__version__ >= '3.0.0': - kw['parse_from_config'] = True manager.add_option( "--known-modules", action='store', default="", + parse_from_config=True, help=( "User defined mapping between a project name and a list of" " provided modules. For example: ``--known-modules=project:" "[Project],extra-project:[extras,utilities]``." - ), - **kw) + )) manager.add_option( "--requirements-file", action='store', + parse_from_config=True, help=( "Specify the name (location) of the requirements text file. " "Unless an absolute path is given, the file will be searched " "relative to the project's root directory. If this option is " "given, requirements from setup.py, setup.cfg or " "pyproject.toml will not be taken into account." - ), - **kw) + )) manager.add_option( "--requirements-max-depth", - type=int if flake8.__version__ >= '3.8.0' else 'int', + type=int, default=1, + parse_from_config=True, help=( "Max depth to resolve recursive requirements. Defaults to 1 " "(one level of recursion allowed)." - ), - **kw) + )) manager.add_option( "--scan-host-site-packages", action='store_true', + parse_from_config=True, help=( "Scan host's site-packages directory for 3rd party projects, " "which provide more than one module or the name of the module" " is different than the project name itself." - ), - **kw) + )) @classmethod def parse_options(cls, options): diff --git a/src/flake8_requirements/modules.py b/src/flake8_requirements/modules.py index d0c0d0e..4a8bb49 100644 --- a/src/flake8_requirements/modules.py +++ b/src/flake8_requirements/modules.py @@ -1,285 +1,3 @@ -# List of all modules (standard library) available in Python 2. -STDLIB_PY2 = ( - "AL", - "BaseHTTPServer", - "Bastion", - "CGIHTTPServer", - "Carbon", - "ColorPicker", - "ConfigParser", - "Cookie", - "DEVICE", - "DocXMLRPCServer", - "EasyDialogs", - "FL", - "FrameWork", - "GL", - "HTMLParser", - "MacOS", - "MimeWriter", - "MiniAEFrame", - "Queue", - "SUNAUDIODEV", - "ScrolledText", - "SimpleHTTPServer", - "SimpleXMLRPCServer", - "SocketServer", - "StringIO", - "Tix", - "Tkinter", - "UserDict", - "UserList", - "UserString", - "__builtin__", - "__future__", - "__main__", - "_winreg", - "abc", - "aepack", - "aetools", - "aetypes", - "aifc", - "al", - "anydbm", - "argparse", - "array", - "ast", - "asynchat", - "asyncore", - "atexit", - "audioop", - "autoGIL", - "base64", - "bdb", - "binascii", - "binhex", - "bisect", - "bsddb", - "bz2", - "cPickle", - "cProfile", - "cStringIO", - "calendar", - "cd", - "cgi", - "cgitb", - "chunk", - "cmath", - "cmd", - "code", - "codecs", - "codeop", - "collections", - "colorsys", - "commands", - "compileall", - "compiler", - "contextlib", - "cookielib", - "copy", - "copy_reg", - "crypt", - "csv", - "ctypes", - "curses", - "datetime", - "dbhash", - "dbm", - "decimal", - "difflib", - "dircache", - "dis", - "distutils", - "dl", - "doctest", - "dumbdbm", - "dummy_thread", - "dummy_threading", - "email", - "ensurepip", - "errno", - "fcntl", - "filecmp", - "fileinput", - "findertools", - "fl", - "flp", - "fm", - "fnmatch", - "formatter", - "fpectl", - "fpformat", - "fractions", - "ftplib", - "functools", - "future_builtins", - "gc", - "gdbm", - "gensuitemodule", - "getopt", - "getpass", - "gettext", - "gl", - "glob", - "grp", - "gzip", - "hashlib", - "heapq", - "hmac", - "hotshot", - "htmlentitydefs", - "htmllib", - "httplib", - "ic", - "imageop", - "imaplib", - "imgfile", - "imghdr", - "imp", - "importlib", - "imputil", - "inspect", - "io", - "itertools", - "jpeg", - "json", - "keyword", - "linecache", - "locale", - "logging", - "macostools", - "macpath", - "mailbox", - "mailcap", - "marshal", - "math", - "md5", - "mhlib", - "mimetools", - "mimetypes", - "mimify", - "mmap", - "modulefinder", - "msilib", - "msvcrt", - "multifile", - "multiprocessing", - "mutex", - "netrc", - "new", - "nis", - "nntplib", - "ntpath", - "numbers", - "operator", - "optparse", - "os", - "os2emxpath", - "ossaudiodev", - "parser", - "pdb", - "pickle", - "pickletools", - "pipes", - "pkgutil", - "platform", - "plistlib", - "popen2", - "poplib", - "posix", - "posixfile", - "posixpath", - "pprint", - "profile", - "pstats", - "pty", - "pwd", - "py_compile", - "pyclbr", - "pydoc", - "quopri", - "random", - "re", - "readline", - "repr", - "resource", - "rexec", - "rfc822", - "rlcompleter", - "robotparser", - "runpy", - "sched", - "select", - "sets", - "sgmllib", - "sha", - "shelve", - "shlex", - "shutil", - "signal", - "site", - "smtpd", - "smtplib", - "sndhdr", - "socket", - "spwd", - "sqlite3", - "ssl", - "stat", - "statvfs", - "string", - "stringprep", - "struct", - "subprocess", - "sunau", - "sunaudiodev", - "symbol", - "symtable", - "sys", - "sysconfig", - "syslog", - "tabnanny", - "tarfile", - "telnetlib", - "tempfile", - "termios", - "test", - "textwrap", - "thread", - "threading", - "time", - "timeit", - "token", - "tokenize", - "trace", - "traceback", - "ttk", - "tty", - "turtle", - "types", - "unicodedata", - "unittest", - "urllib", - "urllib2", - "urlparse", - "user", - "uu", - "uuid", - "warnings", - "wave", - "weakref", - "webbrowser", - "whichdb", - "winsound", - "wsgiref", - "xdrlib", - "xml", - "xmlrpclib", - "zipfile", - "zipimport", - "zlib", -) - # List of all modules (standard library) available in Python 3. STDLIB_PY3 = ( "__future__", diff --git a/test/test_pep621.py b/test/test_pep621.py index b74a46e..9148c8a 100644 --- a/test/test_pep621.py +++ b/test/test_pep621.py @@ -1,16 +1,11 @@ import unittest +from unittest import mock +from unittest.mock import mock_open from flake8_requirements.checker import Flake8Checker from flake8_requirements.checker import ModuleSet from flake8_requirements.checker import memoize -try: - from unittest import mock - builtins_open = 'builtins.open' -except ImportError: - import mock - builtins_open = '__builtin__.open' - class Flake8Options: known_modules = "" @@ -46,7 +41,7 @@ class Options(Flake8Options): ) def test_get_pyproject_toml_pep621(self): - with mock.patch(builtins_open, mock.mock_open(read_data=self.content)): + with mock.patch('builtins.open', mock_open(read_data=self.content)): pep621 = Flake8Checker.get_pyproject_toml_pep621() expected = { "name": "test", @@ -59,15 +54,15 @@ def test_get_pyproject_toml_pep621(self): def test_get_pyproject_toml_invalid(self): content = self.content + b"invalid" - with mock.patch(builtins_open, mock.mock_open(read_data=content)): + with mock.patch('builtins.open', mock_open(read_data=content)): self.assertDictEqual(Flake8Checker.get_pyproject_toml_pep621(), {}) def test_1st_party(self): - with mock.patch(builtins_open, mock.mock_open()) as m: + with mock.patch('builtins.open', mock_open()) as m: m.side_effect = ( IOError("No such file or directory: 'setup.py'"), IOError("No such file or directory: 'setup.cfg'"), - mock.mock_open(read_data=self.content).return_value, + mock_open(read_data=self.content).return_value, ) checker = Flake8Checker(None, None) @@ -75,11 +70,11 @@ def test_1st_party(self): self.assertEqual(mods, ModuleSet({"test": {}})) def test_3rd_party(self): - with mock.patch(builtins_open, mock.mock_open()) as m: + with mock.patch('builtins.open', mock_open()) as m: m.side_effect = ( IOError("No such file or directory: 'setup.py'"), IOError("No such file or directory: 'setup.cfg'"), - mock.mock_open(read_data=self.content).return_value, + mock_open(read_data=self.content).return_value, ) checker = Flake8Checker(None, None) diff --git a/test/test_poetry.py b/test/test_poetry.py index 5637b12..59b513b 100644 --- a/test/test_poetry.py +++ b/test/test_poetry.py @@ -1,16 +1,11 @@ import unittest +from unittest import mock +from unittest.mock import mock_open from flake8_requirements.checker import Flake8Checker from flake8_requirements.checker import ModuleSet from flake8_requirements.checker import memoize -try: - from unittest import mock - builtins_open = 'builtins.open' -except ImportError: - import mock - builtins_open = '__builtin__.open' - class PoetryTestCase(unittest.TestCase): @@ -19,18 +14,18 @@ def setUp(self): def test_get_pyproject_toml_poetry(self): content = b"[tool.poetry]\nname='x'\n[tool.poetry.tag]\nx=0\n" - with mock.patch(builtins_open, mock.mock_open(read_data=content)): + with mock.patch('builtins.open', mock_open(read_data=content)): poetry = Flake8Checker.get_pyproject_toml_poetry() self.assertDictEqual(poetry, {'name': "x", 'tag': {'x': 0}}) def test_1st_party(self): content = b"[tool.poetry]\nname='book'\n" - with mock.patch(builtins_open, mock.mock_open()) as m: + with mock.patch('builtins.open', mock_open()) as m: m.side_effect = ( IOError("No such file or directory: 'setup.py'"), IOError("No such file or directory: 'setup.cfg'"), - mock.mock_open(read_data=content).return_value, + mock_open(read_data=content).return_value, ) checker = Flake8Checker(None, None) @@ -41,11 +36,11 @@ def test_3rd_party(self): content = b"[tool.poetry.dependencies]\ntools='1.0'\n" content += b"[tool.poetry.dev-dependencies]\ndev-tools='1.0'\n" - with mock.patch(builtins_open, mock.mock_open()) as m: + with mock.patch('builtins.open', mock_open()) as m: m.side_effect = ( IOError("No such file or directory: 'setup.py'"), IOError("No such file or directory: 'setup.cfg'"), - mock.mock_open(read_data=content).return_value, + mock_open(read_data=content).return_value, ) checker = Flake8Checker(None, None) @@ -56,11 +51,11 @@ def test_3rd_party_groups(self): content = b"[tool.poetry.dependencies]\ntools='1.0'\n" content += b"[tool.poetry.group.dev.dependencies]\ndev-tools='1.0'\n" - with mock.patch(builtins_open, mock.mock_open()) as m: + with mock.patch('builtins.open', mock_open()) as m: m.side_effect = ( IOError("No such file or directory: 'setup.py'"), IOError("No such file or directory: 'setup.cfg'"), - mock.mock_open(read_data=content).return_value, + mock_open(read_data=content).return_value, ) checker = Flake8Checker(None, None) diff --git a/test/test_requirements.py b/test/test_requirements.py index 99a3094..b31801d 100644 --- a/test/test_requirements.py +++ b/test/test_requirements.py @@ -1,30 +1,25 @@ import os import unittest from collections import OrderedDict +from unittest import mock +from unittest.mock import mock_open from pkg_resources import parse_requirements from flake8_requirements.checker import Flake8Checker from flake8_requirements.checker import memoize -try: - from unittest import mock - builtins_open = 'builtins.open' -except ImportError: - import mock - builtins_open = '__builtin__.open' - def mock_open_with_name(read_data="", name="file.name"): """Mock open call with a specified `name` attribute.""" - m = mock.mock_open(read_data=read_data) + m = mock_open(read_data=read_data) m.return_value.name = name return m def mock_open_multiple(files=OrderedDict()): """Create a mock open object for multiple files.""" - m = mock.mock_open() + m = mock_open() m.side_effect = [ mock_open_with_name(read_data=content, name=name).return_value for name, content in files.items() @@ -60,7 +55,7 @@ def test_resolve_requirement_with_file_beyond_max_depth(self): Flake8Checker.resolve_requirement("-r requirements.txt") def test_resolve_requirement_with_file_empty(self): - with mock.patch(builtins_open, mock.mock_open()) as m: + with mock.patch('builtins.open', mock_open()) as m: self.assertEqual( Flake8Checker.resolve_requirement("-r requirements.txt", 1), [], @@ -68,7 +63,7 @@ def test_resolve_requirement_with_file_empty(self): m.assert_called_once_with("requirements.txt") def test_resolve_requirement_with_file_content(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "foo >= 1.0.0\nbar <= 1.0.0\n"), )))): self.assertEqual( @@ -77,7 +72,7 @@ def test_resolve_requirement_with_file_content(self): ) def test_resolve_requirement_with_file_content_line_continuation(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "foo[bar] \\\n>= 1.0.0\n"), )))): self.assertEqual( @@ -86,7 +81,7 @@ def test_resolve_requirement_with_file_content_line_continuation(self): ) def test_resolve_requirement_with_file_content_line_continuation_2(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "foo \\\n>= 1.0.0 \\\n# comment \\\nbar \\"), )))): self.assertEqual( @@ -95,14 +90,14 @@ def test_resolve_requirement_with_file_content_line_continuation_2(self): ) def test_resolve_requirement_with_file_recursion_beyond_max_depth(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "-r requirements.txt\n"), )))): with self.assertRaises(RuntimeError): Flake8Checker.resolve_requirement("-r requirements.txt", 1), def test_resolve_requirement_with_file_recursion(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "--requirement inner.txt\nbar <= 1.0.0\n"), ("inner.txt", "# inner\nbaz\n\nqux\n"), )))): @@ -112,7 +107,7 @@ def test_resolve_requirement_with_file_recursion(self): ) def test_resolve_requirement_with_relative_include(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "-r requirements/production.txt"), ("requirements/production.txt", "-r node/one.txt\nfoo"), ("requirements/node/one.txt", "-r common.txt\n-r /abs/path.txt"), @@ -132,13 +127,13 @@ def test_resolve_requirement_with_relative_include(self): ]) def test_init_with_no_requirements(self): - with mock.patch(builtins_open, mock.mock_open()) as m: + with mock.patch('builtins.open', mock_open()) as m: m.side_effect = IOError("No such file or directory"), checker = Flake8Checker(None, None) self.assertEqual(checker.get_requirements_txt(), ()) def test_init_with_user_requirements(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements/base.txt", "foo >= 1.0.0\n-r inner.txt\n"), ("requirements/inner.txt", "bar\n"), )))) as m: @@ -160,7 +155,7 @@ def test_init_with_user_requirements(self): Flake8Checker.requirements_file = None def test_init_with_simple_requirements(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "foo >= 1.0.0\nbar <= 1.0.0\n"), )))): checker = Flake8Checker(None, None) @@ -173,7 +168,7 @@ def test_init_with_simple_requirements(self): ) def test_init_with_recursive_requirements_beyond_max_depth(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"), ("inner.txt", "# inner\nbaz\n\nqux\n"), )))): @@ -186,7 +181,7 @@ def test_init_with_recursive_requirements_beyond_max_depth(self): Flake8Checker.requirements_max_depth = 1 def test_init_with_recursive_requirements(self): - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"), ("inner.txt", "# inner\nbaz\n\nqux\n"), )))): @@ -205,7 +200,7 @@ def test_init_misc(self): curdir = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(curdir, "test_requirements.txt")) as f: requirements_content = f.read() - with mock.patch(builtins_open, mock_open_multiple(files=OrderedDict(( + with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( ("requirements.txt", requirements_content), )))): checker = Flake8Checker(None, None) diff --git a/test/test_setup.py b/test/test_setup.py index 83dc89f..3a23e2d 100644 --- a/test/test_setup.py +++ b/test/test_setup.py @@ -1,19 +1,14 @@ import ast import os import unittest +from unittest import mock +from unittest.mock import mock_open from pkg_resources import parse_requirements from flake8_requirements.checker import Flake8Checker from flake8_requirements.checker import SetupVisitor -try: - from unittest import mock - builtins_open = 'builtins.open' -except ImportError: - import mock - builtins_open = '__builtin__.open' - class SetupTestCase(unittest.TestCase): @@ -100,7 +95,7 @@ def test_get_setup_cfg_requirements(self): curdir = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(curdir, "test_setup.cfg")) as f: content = f.read() - with mock.patch(builtins_open, mock.mock_open(read_data=content)): + with mock.patch('builtins.open', mock_open(read_data=content)): checker = Flake8Checker(None, None) self.assertEqual( checker.get_setup_cfg_requirements(False), From ea99799b69b9d067116258a64b3627673935e81b Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Fri, 4 Aug 2023 09:17:31 +0200 Subject: [PATCH 05/18] Mark I901 warning as not implemented yet (#77) Fixes #76 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 26a53e8..797ec73 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ package import requirements. It reports missing and/or not used project direct d This plug-in adds new flake8 warnings: - ``I900``: Package is not listed as a requirement. -- ``I901``: Package is required but not used. +- ``I901``: Package is required but not used. (not implemented yet) Important notice ---------------- From d818bfb5ca7714e0e6562b89c8788cf2a0927d5a Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Wed, 8 Nov 2023 07:33:50 +0100 Subject: [PATCH 06/18] Install setuptools before running tests --- .github/workflows/check.yaml | 4 ++++ .github/workflows/codecov.yaml | 3 ++- setup.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 07b72df..24dc9f8 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -21,6 +21,10 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools - name: Run Tests run: python setup.py pytest diff --git a/.github/workflows/codecov.yaml b/.github/workflows/codecov.yaml index aaaabbe..3fdfb5a 100644 --- a/.github/workflows/codecov.yaml +++ b/.github/workflows/codecov.yaml @@ -14,7 +14,8 @@ jobs: python-version: '3.x' - name: Generate Coverage Report run: | - pip install coverage + python -m pip install --upgrade pip + pip install coverage setuptools coverage run --include=src/* setup.py pytest - name: Upload Coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/setup.py b/setup.py index 4e2a679..0ab5f33 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ def get_abs_path(pathname): "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only" + "Programming Language :: Python :: 3 :: Only", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance", ], From 4b47c13f7aba47fa762d9051d017ae2665f0275b Mon Sep 17 00:00:00 2001 From: wankata Date: Wed, 8 Nov 2023 17:08:42 +0200 Subject: [PATCH 07/18] Add encodings to modules.STDLIB_PY3 (#78) --- src/flake8_requirements/checker.py | 2 +- src/flake8_requirements/modules.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 3bb1429..8ee8863 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -20,7 +20,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "2.0.0" +__version__ = "2.0.1" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') diff --git a/src/flake8_requirements/modules.py b/src/flake8_requirements/modules.py index 4a8bb49..d12b6ab 100644 --- a/src/flake8_requirements/modules.py +++ b/src/flake8_requirements/modules.py @@ -53,6 +53,7 @@ "doctest", "dummy_threading", "email", + "encodings", "ensurepip", "enum", "errno", From ab13226e04b633089d5465bb0bf762b62af84b77 Mon Sep 17 00:00:00 2001 From: James Braza Date: Mon, 5 Feb 2024 13:06:08 -0800 Subject: [PATCH 08/18] Update documentation for --requirements-file --- src/flake8_requirements/checker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 8ee8863..77b83d7 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -20,7 +20,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "2.0.1" +__version__ = "2.0.2" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') @@ -347,8 +347,9 @@ def add_options(cls, manager): "Specify the name (location) of the requirements text file. " "Unless an absolute path is given, the file will be searched " "relative to the project's root directory. If this option is " - "given, requirements from setup.py, setup.cfg or " - "pyproject.toml will not be taken into account." + "not specified, the plugin look up for requirements in " + "(1) setup.py, (2) setup.cfg, (3) pyproject.toml, and (4) " + "requirements.txt. If specified, look up will not take place." )) manager.add_option( "--requirements-max-depth", From 0061d9acce6864ab10aea5d30c1d0d0df3327ac0 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Tue, 6 Feb 2024 10:57:38 +0100 Subject: [PATCH 09/18] Deploy to PyPI with trusted publishing --- .github/workflows/publish.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index cc3192b..3bf3694 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -7,6 +7,11 @@ jobs: deploy: if: github.repository_owner == 'arkq' runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/flake8-requirements + permissions: + id-token: write steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -16,10 +21,7 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py bdist_wheel - twine upload dist/* + - name: Build + run: python setup.py bdist_wheel + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 From f36956ed7768e37a1948f307c00de12365be7c5a Mon Sep 17 00:00:00 2001 From: James Braza Date: Tue, 20 Feb 2024 13:38:51 -0500 Subject: [PATCH 10/18] Added `flake8-requirements` self-checking to CI (#87) --- .github/workflows/check.yaml | 2 +- setup.cfg | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 24dc9f8..7f49c57 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -64,5 +64,5 @@ jobs: python-version: '3.x' - name: Run flake8 Linter run: | - pip install flake8 + pip install -e . flake8 flake8 --count --show-source --statistics src test diff --git a/setup.cfg b/setup.cfg index 2b5b9ac..2d76c1e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,5 +4,8 @@ python-tag = py3 [doc8] max-line-length = 99 +[flake8] +extend-exclude = venv + [isort] force_single_line = true From 894d7f233fabfcc2697524e49115d87b68d6289e Mon Sep 17 00:00:00 2001 From: James Braza Date: Tue, 20 Feb 2024 13:40:54 -0500 Subject: [PATCH 11/18] Created `pyproject` extra for `Flake8-pyproject` (#83) --- README.rst | 5 +++-- setup.py | 1 + src/flake8_requirements/checker.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 797ec73..61e21a1 100644 --- a/README.rst +++ b/README.rst @@ -64,8 +64,9 @@ Real life example:: max-line-length = 100 known-modules = my-lib:[mylib.drm,mylib.encryption] -If you use `flake8-pyproject `_, you can also configure -the known modules using a nicer syntax:: +If you use `Flake8-pyproject `_ +(can include for installation using ``flake8-requirements[pyproject]``), +you can also configure the known modules using a nicer syntax in ``pyproject.toml``:: $ cat pyproject.toml ... diff --git a/setup.py b/setup.py index 0ab5f33..5c33bab 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ def get_abs_path(pathname): "setuptools >= 10.0.0", "tomli>=1.2.1; python_version < '3.11'", ], + extras_require={"pyproject": ["Flake8-pyproject"]}, setup_requires=["pytest-runner"], tests_require=["mock", "pytest"], entry_points={ diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 77b83d7..4183204 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -20,7 +20,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "2.0.2" +__version__ = "2.1.0" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') From f9291b01208aa24c06b8ebc325c1ba3abfcba9ae Mon Sep 17 00:00:00 2001 From: Ivan Kirilov Ivanov Date: Mon, 25 Mar 2024 10:59:45 +0200 Subject: [PATCH 12/18] Update known 3rd parties (#89) --- src/flake8_requirements/modules.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/flake8_requirements/modules.py b/src/flake8_requirements/modules.py index d12b6ab..f0e7ad7 100644 --- a/src/flake8_requirements/modules.py +++ b/src/flake8_requirements/modules.py @@ -235,6 +235,7 @@ "allure-pytest": ["allure"], "ansicolors": ["colors"], "apache-airflow": ["airflow"], + "appengine-python-standard": ["google.appengine"], "attrs": ["attr", "attrs"], "awesome-slugify": ["slugify"], "azure-common": ["azure.common"], @@ -293,7 +294,9 @@ "ffmpeg-python": ["ffmpeg"], "fluent-logger": ["fluent"], "gitpython": ["git"], + "google-api-core": ["google.api_core"], "google-api-python-client": ["apiclient", "googleapiclient"], + "google-auth": ["google.auth", "google.oauth2"], "google-cloud-aiplatform": ["google.cloud.aiplatform"], "google-cloud-bigquery": ["google.cloud.bigquery"], "google-cloud-bigtable": ["google.cloud.bigtable"], @@ -309,7 +312,12 @@ "google.cloud.logging_v2", "google.cloud.logging", ], - "google-cloud-pubsub": ["google.cloud.pubsub_v1", "google.cloud.pubsub"], + "google-cloud-pubsub": [ + "google.cloud.pubsub_v1", + "google.cloud.pubsub", + "google.pubsub_v1", + "google.pubsub", + ], "google-cloud-secret-manager": ["google.cloud.secretmanager"], "google-cloud-storage": ["google.cloud.storage"], "grpcio": ["grpc"], @@ -373,6 +381,7 @@ "opentelemetry-sdk": ["opentelemetry.sdk"], "opentelemetry-test-utils": ["opentelemetry.test"], "paho-mqtt": ["paho"], + "phonenumberslite": ["phonenumbers"], "pillow": ["PIL"], "pillow-simd": ["PIL"], "pip-tools": ["piptools"], From f7f0c97b942eff52ecb92754a6b84433d76fe435 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Wed, 27 Mar 2024 07:48:07 +0100 Subject: [PATCH 13/18] Bump version to 2.1.1 --- src/flake8_requirements/checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 4183204..c3c595f 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -20,7 +20,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "2.1.0" +__version__ = "2.1.1" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') From 752581c0c9bef18136d04f65547fe8a3a58d97b1 Mon Sep 17 00:00:00 2001 From: Mihail Andreev Date: Wed, 8 May 2024 14:55:57 +0300 Subject: [PATCH 14/18] Add support for dynamic dependencies (#91) The dynamic dependencies are part of pep621. For now only the setuptools supports dynamic dependencies. Because of that the implementation is somehow dependent on setuptools. When other tool adds support for dynamic dependencies only the part for gathering of the dependencies has to be changed. --- src/flake8_requirements/checker.py | 31 ++++++++++- test/test_pep621.py | 87 ++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index c3c595f..ad8a733 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -20,7 +20,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "2.1.1" +__version__ = "2.2.0" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') @@ -560,6 +560,33 @@ def get_pyproject_toml_pep621(cls): cfg_pep518 = cls.get_pyproject_toml() return cfg_pep518.get('project', {}) + @classmethod + def get_setuptools_dynamic_requirements(cls): + """Retrieve dynamic requirements defined in setuptools config.""" + cfg = cls.get_pyproject_toml() + dynamic_keys = cfg.get('project', {}).get('dynamic', []) + dynamic_config = ( + cfg.get('tool', {}).get('setuptools', {}).get('dynamic', {}) + ) + requirements = [] + files_to_parse = [] + if 'dependencies' in dynamic_keys: + files_to_parse.extend( + dynamic_config.get('dependencies', {}).get('file', []) + ) + if 'optional-dependencies' in dynamic_keys: + for element in dynamic_config.get( + 'optional-dependencies', {} + ).values(): + files_to_parse.extend(element.get('file', [])) + for file_path in files_to_parse: + try: + with open(file_path, 'r') as file: + requirements.extend(parse_requirements(file)) + except IOError as e: + LOG.debug("Couldn't open requirements file: %s", e) + return requirements + @classmethod def get_pyproject_toml_pep621_requirements(cls): """Try to get PEP 621 metadata requirements.""" @@ -569,6 +596,8 @@ def get_pyproject_toml_pep621_requirements(cls): pep621.get("dependencies", ()))) for r in pep621.get("optional-dependencies", {}).values(): requirements.extend(parse_requirements(r)) + if len(requirements) == 0: + requirements = cls.get_setuptools_dynamic_requirements() return requirements @classmethod diff --git a/test/test_pep621.py b/test/test_pep621.py index 9148c8a..22e9bc3 100644 --- a/test/test_pep621.py +++ b/test/test_pep621.py @@ -1,6 +1,9 @@ import unittest from unittest import mock from unittest.mock import mock_open +from unittest.mock import patch + +from pkg_resources import parse_requirements from flake8_requirements.checker import Flake8Checker from flake8_requirements.checker import ModuleSet @@ -80,3 +83,87 @@ def test_3rd_party(self): checker = Flake8Checker(None, None) mods = checker.get_mods_3rd_party(False) self.assertEqual(mods, ModuleSet({"tools": {}, "dev_tools": {}})) + + def test_dynamic_requirements(self): + requirements_content = "package1\npackage2>=2.0" + data = { + "project": {"dynamic": ["dependencies"]}, + "tool": { + "setuptools": { + "dynamic": {"dependencies": {"file": ["requirements.txt"]}} + } + }, + } + with patch( + 'flake8_requirements.checker.Flake8Checker.get_pyproject_toml', + return_value=data, + ): + with patch( + 'builtins.open', mock_open(read_data=requirements_content) + ): + result = Flake8Checker.get_setuptools_dynamic_requirements() + expected_results = ['package1', 'package2>=2.0'] + parsed_results = [str(req) for req in result] + self.assertEqual(parsed_results, expected_results) + + def test_dynamic_optional_dependencies(self): + data = { + "project": {"dynamic": ["dependencies", "optional-dependencies"]}, + "tool": { + "setuptools": { + "dynamic": { + "dependencies": {"file": ["requirements.txt"]}, + "optional-dependencies": { + "test": {"file": ["optional-requirements.txt"]} + }, + } + } + }, + } + requirements_content = """ + package1 + package2>=2.0 + """ + optional_requirements_content = "package3[extra] >= 3.0" + with mock.patch( + 'flake8_requirements.checker.Flake8Checker.get_pyproject_toml', + return_value=data, + ): + with mock.patch('builtins.open', mock.mock_open()) as mocked_file: + mocked_file.side_effect = [ + mock.mock_open( + read_data=requirements_content + ).return_value, + mock.mock_open( + read_data=optional_requirements_content + ).return_value, + ] + result = Flake8Checker.get_setuptools_dynamic_requirements() + expected = list(parse_requirements(requirements_content)) + expected += list( + parse_requirements(optional_requirements_content) + ) + + self.assertEqual(len(result), len(expected)) + for i in range(len(result)): + self.assertEqual(result[i], expected[i]) + + def test_missing_requirements_file(self): + data = { + "project": {"dynamic": ["dependencies"]}, + "tool": { + "setuptools": { + "dynamic": { + "dependencies": { + "file": ["nonexistent-requirements.txt"] + } + } + } + }, + } + with mock.patch( + 'flake8_requirements.checker.Flake8Checker.get_pyproject_toml', + return_value=data, + ): + result = Flake8Checker.get_setuptools_dynamic_requirements() + self.assertEqual(result, []) From eb38c11a323b780987b895d1200ffd9d64c48b8b Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Mon, 24 Jun 2024 12:08:56 +0000 Subject: [PATCH 15/18] Add atlassian-python-api to 3rd party modules Fixes #92 --- src/flake8_requirements/checker.py | 2 +- src/flake8_requirements/modules.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index ad8a733..27587c4 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -20,7 +20,7 @@ from .modules import STDLIB_PY3 # NOTE: Changing this number will alter package version as well. -__version__ = "2.2.0" +__version__ = "2.2.1" __license__ = "MIT" LOG = getLogger('flake8.plugin.requirements') diff --git a/src/flake8_requirements/modules.py b/src/flake8_requirements/modules.py index f0e7ad7..f63eebb 100644 --- a/src/flake8_requirements/modules.py +++ b/src/flake8_requirements/modules.py @@ -236,6 +236,7 @@ "ansicolors": ["colors"], "apache-airflow": ["airflow"], "appengine-python-standard": ["google.appengine"], + "atlassian-python-api": ["atlassian"], "attrs": ["attr", "attrs"], "awesome-slugify": ["slugify"], "azure-common": ["azure.common"], From bc752041ea6cd6a4563d256d98d69e9b4f689806 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Mon, 13 Jan 2025 19:01:46 +0100 Subject: [PATCH 16/18] Setup dependency scans on monthly intervals --- .github/dependabot.yaml | 8 ++++++++ .github/workflows/check.yaml | 18 +++++++++--------- .github/workflows/codecov.yaml | 8 +++++--- .github/workflows/publish.yaml | 4 ++-- 4 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 .github/dependabot.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..e4f5a6d --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,8 @@ +version: 2 +updates: + + # Check for updates to GitHub Actions every month + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 7f49c57..36914bc 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -17,8 +17,8 @@ jobs: fail-fast: false runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - name: Install dependencies @@ -33,23 +33,23 @@ jobs: fail-fast: false runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: python queries: security-and-quality - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 doc8-lint: strategy: fail-fast: false runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run reStructuredText Linter - uses: deep-entertainment/doc8-action@v4 + uses: deep-entertainment/doc8-action@v5 with: scanPaths: ${{ github.workspace }} @@ -58,8 +58,8 @@ jobs: fail-fast: false runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - name: Run flake8 Linter diff --git a/.github/workflows/codecov.yaml b/.github/workflows/codecov.yaml index 3fdfb5a..b165be7 100644 --- a/.github/workflows/codecov.yaml +++ b/.github/workflows/codecov.yaml @@ -8,8 +8,8 @@ jobs: if: github.repository_owner == 'arkq' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - name: Generate Coverage Report @@ -18,4 +18,6 @@ jobs: pip install coverage setuptools coverage run --include=src/* setup.py pytest - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 3bf3694..59d56bd 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,8 +13,8 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.x' - name: Install dependencies From 792137a522f5e3511f8b642f3e9da76ff7ec1e0f Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Mon, 13 Jan 2025 20:20:21 +0100 Subject: [PATCH 17/18] Update build system to use pyproject.toml --- .github/workflows/check.yaml | 6 ++-- .github/workflows/codecov.yaml | 7 +++-- .github/workflows/publish.yaml | 6 ++-- pyproject.toml | 42 +++++++++++++++++++++++++ setup.cfg | 11 ------- setup.py | 50 ------------------------------ src/flake8_requirements/checker.py | 2 +- test/test_checker.py | 6 ++-- tox.ini | 35 +++++++++++++++++++++ 9 files changed, 89 insertions(+), 76 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py create mode 100644 tox.ini diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 36914bc..024f40f 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -22,11 +22,9 @@ jobs: with: python-version: '3.x' - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools + run: pip install tox - name: Run Tests - run: python setup.py pytest + run: tox -e py3 code-ql: strategy: diff --git a/.github/workflows/codecov.yaml b/.github/workflows/codecov.yaml index b165be7..4db7d2a 100644 --- a/.github/workflows/codecov.yaml +++ b/.github/workflows/codecov.yaml @@ -14,10 +14,11 @@ jobs: python-version: '3.x' - name: Generate Coverage Report run: | - python -m pip install --upgrade pip - pip install coverage setuptools - coverage run --include=src/* setup.py pytest + pip install tox + tox - name: Upload Coverage to Codecov uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} + files: .tox/coverage.xml + disable_search: true diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 59d56bd..8ce1ea6 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -18,10 +18,8 @@ jobs: with: python-version: '3.x' - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine + run: pip install build - name: Build - run: python setup.py bdist_wheel + run: python -m build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a80295e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel", "build"] +build-backend = "setuptools.build_meta" + +[project] +name = "flake8-requirements" +# NOTE: Keep in sync with src/flake8_requirements/checker.py file. +version = "2.2.1" +description = "Package requirements checker, plugin for flake8" +readme = "README.rst" +authors = [ { name = "Arkadiusz Bokowy", email = "arkadiusz.bokowy@gmail.com" } ] +requires-python = ">=3.6" +classifiers = [ + "Framework :: Flake8", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance", +] +dependencies = [ + "flake8 >= 4.0.0", + "setuptools >= 10.0.0", + "tomli>=1.2.1; python_version < '3.11'", +] + +[project.optional-dependencies] +pyproject = ["Flake8-pyproject"] + +[project.urls] +Homepage = "https://github.com/arkq/flake8-requirements" + +[project.entry-points."flake8.extension"] +I90 = "flake8_requirements:Flake8Checker" + +[tool.doc8] +max-line-length = 99 + +[tool.isort] +force_single_line = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2d76c1e..0000000 --- a/setup.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[bdist_wheel] -python-tag = py3 - -[doc8] -max-line-length = 99 - -[flake8] -extend-exclude = venv - -[isort] -force_single_line = true diff --git a/setup.py b/setup.py deleted file mode 100644 index 5c33bab..0000000 --- a/setup.py +++ /dev/null @@ -1,50 +0,0 @@ -import re -from os import path - -from setuptools import setup - - -def get_abs_path(pathname): - return path.join(path.dirname(__file__), pathname) - - -with open(get_abs_path("src/flake8_requirements/checker.py")) as f: - version = re.match(r'.*__version__ = "(.*?)"', f.read(), re.S).group(1) -with open(get_abs_path("README.rst")) as f: - long_description = f.read() - -setup( - name="flake8-requirements", - version=version, - author="Arkadiusz Bokowy", - author_email="arkadiusz.bokowy@gmail.com", - url="https://github.com/Arkq/flake8-requirements", - description="Package requirements checker, plugin for flake8", - long_description=long_description, - license="MIT", - package_dir={'': "src"}, - packages=["flake8_requirements"], - install_requires=[ - "flake8 >= 4.0.0", - "setuptools >= 10.0.0", - "tomli>=1.2.1; python_version < '3.11'", - ], - extras_require={"pyproject": ["Flake8-pyproject"]}, - setup_requires=["pytest-runner"], - tests_require=["mock", "pytest"], - entry_points={ - 'flake8.extension': [ - 'I90 = flake8_requirements:Flake8Checker', - ], - }, - classifiers=[ - "Framework :: Flake8", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: Software Development :: Quality Assurance", - ], -) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 27587c4..92ee581 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -19,7 +19,7 @@ from .modules import KNOWN_3RD_PARTIES from .modules import STDLIB_PY3 -# NOTE: Changing this number will alter package version as well. +# NOTE: Keep in sync with pyproject.toml file. __version__ = "2.2.1" __license__ = "MIT" diff --git a/test/test_checker.py b/test/test_checker.py index fb7d10b..cf83324 100644 --- a/test/test_checker.py +++ b/test/test_checker.py @@ -28,9 +28,9 @@ class Flake8Checker(checker.Flake8Checker): def get_setup_py(cls): return SetupVisitorMock() - @property - def processing_setup_py(self): - return self.filename == "setup.py" + @staticmethod + def is_project_setup_py(project_root_dir, filename): + return filename == "setup.py" class Flake8OptionManagerMock(dict): diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..00304f2 --- /dev/null +++ b/tox.ini @@ -0,0 +1,35 @@ +[tox] +envlist = + coverage + py3 +isolated_build = true + +[testenv] +description = Run the tests with pytest under {basepython}. +setenv = + COVERAGE_FILE = {toxworkdir}/.coverage.{envname} +commands = + pytest \ + --cov="{envsitepackagesdir}/flake8_requirements" \ + --cov-config="{toxinidir}/tox.ini" \ + test +deps = + pytest + pytest-cov + +[testenv:coverage] +description = Combine coverage data and create final XML report. +setenv = + COVERAGE_FILE = {toxworkdir}/.coverage +commands = + coverage combine + coverage report + coverage xml -o "{toxworkdir}/coverage.xml" +skip_install = true +deps = coverage +depends = py3 + +[coverage:paths] +source = src/flake8_requirements + */.tox/*/lib/python*/site-packages/flake8_requirements + */src/flake8_requirements From f607e61e8214b0216816ceb515dfb867367d5022 Mon Sep 17 00:00:00 2001 From: James Braza Date: Tue, 14 Jan 2025 12:14:04 -0800 Subject: [PATCH 18/18] Fixing debug messages about failed loads (#95) --- src/flake8_requirements/checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py index 92ee581..1c3ad93 100644 --- a/src/flake8_requirements/checker.py +++ b/src/flake8_requirements/checker.py @@ -551,7 +551,7 @@ def get_pyproject_toml(cls): with open(pyproject_config_path, mode="rb") as f: return tomllib.load(f) except (IOError, tomllib.TOMLDecodeError) as e: - LOG.debug("Couldn't load project setup: %s", e) + LOG.debug("Couldn't load pyproject: %s", e) return {} @classmethod @@ -675,7 +675,7 @@ def get_setup_py(cls): with open(os.path.join(cls.root_dir, "setup.py")) as f: return SetupVisitor(ast.parse(f.read()), cls.root_dir) except IOError as e: - LOG.debug("Couldn't load project setup: %s", e) + LOG.debug("Couldn't load setup: %s", e) return SetupVisitor(ast.parse(""), cls.root_dir) @classmethod