From 39c66b1fae35386df5653c0788c9768c565f0352 Mon Sep 17 00:00:00 2001 From: Arkadiusz Bokowy Date: Sat, 25 Nov 2017 13:59:28 +0100 Subject: [PATCH] Requirements checker initial commit --- .gitignore | 37 ++ LICENSE.txt | 21 ++ README.rst | 22 ++ setup.cfg | 2 + setup.py | 50 +++ src/flake8_requirements/__init__.py | 5 + src/flake8_requirements/checker.py | 200 +++++++++++ src/flake8_requirements/modules.py | 503 ++++++++++++++++++++++++++++ test/test_checker.py | 62 ++++ test/test_setup.py | 66 ++++ 10 files changed, 968 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE.txt create mode 100644 README.rst create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/flake8_requirements/__init__.py create mode 100644 src/flake8_requirements/checker.py create mode 100644 src/flake8_requirements/modules.py create mode 100644 test/test_checker.py create mode 100644 test/test_setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f36cadb --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..43d79e9 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2017 Arkadiusz Bokowy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..8cdc333 --- /dev/null +++ b/README.rst @@ -0,0 +1,22 @@ +Package requirements checker +============================ + +This module provides a plug-in for [flake8](http://flake8.pycqa.org), which checks/validates +package import requirements. It reports missing and/or not used project direct dependencies. + +Installation +------------ + +You can install, upgrade, or uninstall ``flake8-requirements`` with these commands:: + + $ pip install flake8-requirements + $ pip install --upgrade flake8-requirements + $ pip uninstall flake8-requirements + +Warnings +-------- + +This package adds new flake8 warnings as follows: + +- ``I900``: Package is not listed as a requirement. +- ``I901``: Package is require but not used. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2a9acf1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d526004 --- /dev/null +++ b/setup.py @@ -0,0 +1,50 @@ +from __future__ import with_statement + +import re +from os import path + +from setuptools import setup + + +def get_abs_path(pathname): + return path.join(path.dirname(__file__), pathname) + + +with open(get_abs_path("src/flake8_requirements/checker.py")) as f: + version = re.match(r'.*__version__ = "(.*?)"', f.read(), re.S).group(1) +with open(get_abs_path("README.rst")) as f: + long_description = f.read() + +setup( + name="flake8-requirements", + version=version, + author="Arkadiusz Bokowy", + author_email="arkadiusz.bokowy@gmail.com", + url="https://github.com/Arkq/flake8-requirements", + description="Package requirements checker, plugin for flake8", + long_description=long_description, + license="MIT", + package_dir={'': "src"}, + packages=["flake8_requirements"], + install_requires=[ + "flake8 > 2.0.0", + "setuptools", + ], + setup_requires=["pytest-runner"], + tests_require=["pytest"], + entry_points={ + 'flake8.extension': [ + 'I90 = flake8_requirements:Flake8Checker', + ], + }, + classifiers=[ + "Framework :: Flake8", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance", + ], +) diff --git a/src/flake8_requirements/__init__.py b/src/flake8_requirements/__init__.py new file mode 100644 index 0000000..2bbfb87 --- /dev/null +++ b/src/flake8_requirements/__init__.py @@ -0,0 +1,5 @@ +from .checker import Flake8Checker + +__all__ = ( + 'Flake8Checker', +) diff --git a/src/flake8_requirements/checker.py b/src/flake8_requirements/checker.py new file mode 100644 index 0000000..2563569 --- /dev/null +++ b/src/flake8_requirements/checker.py @@ -0,0 +1,200 @@ +import ast +import sys +from itertools import chain + +from pkg_resources import parse_requirements + +from .modules import KNOWN_3RD_PARTIES +from .modules import STDLIB_PY2 +from .modules import STDLIB_PY3 + +# NOTE: Changing this number will alter package version as well. +__version__ = "1.0.0" +__license__ = "MIT" + +ERRORS = { + 'I900': "I900 '{pkg}' not listed as a requirement", + 'I901': "I901 '{pkg}' required but not used", +} + +STDLIB = set() +if sys.version_info[0] == 2: + STDLIB.update(STDLIB_PY2) +if sys.version_info[0] == 3: + STDLIB.update(STDLIB_PY3) + + +class ImportVisitor(ast.NodeVisitor): + """Import statement visitor.""" + + def __init__(self, tree): + """Initialize import statement visitor.""" + self.imports = [] + self.visit(tree) + + def visit_Import(self, node): + self.imports.append((node, node.names[0].name)) + + def visit_ImportFrom(self, node): + self.imports.append((node, node.module)) + + +class SetupVisitor(ast.NodeVisitor): + """Package setup visitor. + + Warning: + This visitor class executes given Abstract Syntax Tree! + + """ + + # Set of keywords used by the setup() function. + attributes = { + 'required': { + 'name', + 'version', + }, + 'one-of': { + 'ext_modules', + 'packages', + 'py_modules', + }, + 'optional': { + 'author', + 'author_email', + 'classifiers', + 'cmdclass', + 'configuration', + 'convert_2to3_doctests', + 'dependency_links', + 'description', + 'download_url', + 'eager_resources', + 'entry_points', + 'exclude_package_data', + 'extras_require', + 'features', + 'include_package_data', + 'install_requires', + 'keywords', + 'license', + 'long_description', + 'maintainer', + 'maintainer_email', + 'message_extractors', + 'namespace_packages', + 'package_data', + 'package_dir', + 'platforms', + 'python_requires', + 'scripts', + 'setup_requires', + 'test_loader', + 'test_suite', + 'tests_require', + 'url', + 'use_2to3', + 'use_2to3_fixers', + 'zip_safe', + }, + } + + def __init__(self, tree): + """Initialize package setup visitor.""" + self.redirected = False + self.keywords = {} + + # Find setup() call and redirect it. + self.visit(tree) + + if not self.redirected: + return + + def setup(**kw): + self.keywords = kw + + eval( + compile(ast.fix_missing_locations(tree), "", mode='exec'), + {'__file__': "setup.py", '__f8r_setup': setup}, + ) + + def get_requirements(self, install=True, extras=True): + """Get package requirements.""" + requires = [] + if install: + requires.extend(parse_requirements( + self.keywords.get('install_requires', ()), + )) + if extras: + for r in self.keywords.get('extras_require', {}).values(): + requires.extend(parse_requirements(r)) + return requires + + def visit_Call(self, node): + """Call visitor - used for finding setup() call.""" + self.generic_visit(node) + + # Setuptools setup() is a keywords only function. + if not (not node.args and (node.keywords or node.kwargs)): + return + + keywords = {x.arg for x in node.keywords} + if node.kwargs: + keywords.update(x.s for x in node.kwargs.keys) + + if not keywords.issuperset(self.attributes['required']): + return + if not keywords.intersection(self.attributes['one-of']): + return + if not keywords.issubset(chain(*self.attributes.values())): + return + + # Redirect call to our setup() tap function. + node.func = ast.Name(id='__f8r_setup', ctx=node.func.ctx) + self.redirected = True + + +class Flake8Checker(object): + """Package requirements checker.""" + + name = "flake8-requires" + version = __version__ + + def __init__(self, tree, filename, lines=None): + """Initialize requirements checker.""" + self.setup = self.get_setup() + self.tree = tree + + def get_setup(self): + """Get package setup.""" + with open("setup.py") as f: + return SetupVisitor(ast.parse(f.read())) + + def run(self): + """Run checker.""" + + def modcmp(mod1=(), mod2=()): + """Compare import modules.""" + return all(a == b for a, b in zip(mod1, mod2)) + + requirements = set() + + # Get module names based on requirements. + for requirement in self.setup.get_requirements(): + project = requirement.project_name.lower() + modules = [project.replace("-", "_")] + if project in KNOWN_3RD_PARTIES: + modules = KNOWN_3RD_PARTIES[project] + requirements.update(tuple(x.split(".")) for x in modules) + + for node, module in ImportVisitor(self.tree).imports: + _module = module.split(".") + if any([_module[0] == x for x in STDLIB]): + continue + if any([modcmp(_module, x) for x in requirements]): + continue + yield ( + node.lineno, + node.col_offset, + ERRORS['I900'].format(pkg=module), + Flake8Checker, + ) diff --git a/src/flake8_requirements/modules.py b/src/flake8_requirements/modules.py new file mode 100644 index 0000000..6fd066c --- /dev/null +++ b/src/flake8_requirements/modules.py @@ -0,0 +1,503 @@ +# List of all modules (standard library) available in Python 2. +STDLIB_PY2 = ( + "AL", + "BaseHTTPServer", + "Bastion", + "CGIHTTPServer", + "Carbon", + "ColorPicker", + "ConfigParser", + "Cookie", + "DEVICE", + "DocXMLRPCServer", + "EasyDialogs", + "FL", + "FrameWork", + "GL", + "HTMLParser", + "MacOS", + "MimeWriter", + "MiniAEFrame", + "Queue", + "SUNAUDIODEV", + "ScrolledText", + "SimpleHTTPServer", + "SimpleXMLRPCServer", + "SocketServer", + "StringIO", + "Tix", + "Tkinter", + "UserDict", + "UserList", + "UserString", + "__builtin__", + "__future__", + "__main__", + "_winreg", + "abc", + "aepack", + "aetools", + "aetypes", + "aifc", + "al", + "anydbm", + "argparse", + "array", + "ast", + "asynchat", + "asyncore", + "atexit", + "audioop", + "autoGIL", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "bsddb", + "bz2", + "cPickle", + "cProfile", + "cStringIO", + "calendar", + "cd", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "commands", + "compileall", + "compiler", + "contextlib", + "cookielib", + "copy", + "copy_reg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbhash", + "dbm", + "decimal", + "difflib", + "dircache", + "dis", + "distutils", + "dl", + "doctest", + "dumbdbm", + "dummy_thread", + "dummy_threading", + "email", + "ensurepip", + "errno", + "fcntl", + "filecmp", + "fileinput", + "findertools", + "fl", + "flp", + "fm", + "fnmatch", + "formatter", + "fpectl", + "fpformat", + "fractions", + "ftplib", + "functools", + "future_builtins", + "gc", + "gdbm", + "gensuitemodule", + "getopt", + "getpass", + "gettext", + "gl", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "hotshot", + "htmlentitydefs", + "htmllib", + "httplib", + "ic", + "imageop", + "imaplib", + "imgfile", + "imghdr", + "imp", + "importlib", + "imputil", + "inspect", + "io", + "itertools", + "jpeg", + "json", + "keyword", + "linecache", + "locale", + "logging", + "macostools", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "md5", + "mhlib", + "mimetools", + "mimetypes", + "mimify", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multifile", + "multiprocessing", + "mutex", + "netrc", + "new", + "nis", + "nntplib", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "popen2", + "poplib", + "posix", + "posixfile", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "quopri", + "random", + "re", + "readline", + "repr", + "resource", + "rexec", + "rfc822", + "rlcompleter", + "robotparser", + "runpy", + "sched", + "select", + "sets", + "sgmllib", + "sha", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "spwd", + "sqlite3", + "ssl", + "stat", + "statvfs", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "sunaudiodev", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "thread", + "threading", + "time", + "timeit", + "token", + "tokenize", + "trace", + "traceback", + "ttk", + "tty", + "turtle", + "types", + "unicodedata", + "unittest", + "urllib", + "urllib2", + "urlparse", + "user", + "uu", + "uuid", + "warnings", + "wave", + "weakref", + "webbrowser", + "whichdb", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpclib", + "zipfile", + "zipimport", + "zlib", +) + +# List of all modules (standard library) available in Python 3. +STDLIB_PY3 = ( + "__future__", + "__main__", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collection", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fpectl", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "linecache", + "locale", + "logging", + "lzma", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "pprint", + "profile", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +) + +# Mapping for known 3rd party projects, which provide more than one module +# or the name of the module is different than the project name itself. +KNOWN_3RD_PARTIES = { + "awesome_slugify": ["slugify"], + "enum34": ["enum"], + "pillow": ["PIL"], + "py_lru_cache": ["lru"], + "pyicu": ["icu"], + "pyjwt": ["jwt"], + "pyyaml": ["yaml"], + "setuptools": ["pkg_resources", "setuptools"], + "xlwt_future": ["xlwt"], +} diff --git a/test/test_checker.py b/test/test_checker.py new file mode 100644 index 0000000..3523557 --- /dev/null +++ b/test/test_checker.py @@ -0,0 +1,62 @@ +import ast +import unittest + +from pkg_resources import parse_requirements + +from flake8_requirements import checker + + +class SetupVisitorMock: + def get_requirements(self): + return parse_requirements(( + "foo", + "bar", + "hyp-hen", + "setuptools", + )) + + +class Flake8Checker(checker.Flake8Checker): + def get_setup(self): + return SetupVisitorMock() + + +def check(code): + return list(Flake8Checker(ast.parse(code), "").run()) + + +class Flake8CheckerTestCase(unittest.TestCase): + + def test_stdlib(self): + errors = check("import os\nfrom unittest import TestCase") + self.assertEqual(len(errors), 0) + + def test_stdlib_case(self): + errors = check("from cProfile import Profile") + self.assertEqual(len(errors), 0) + errors = check("from cprofile import Profile") + self.assertEqual(len(errors), 1) + self.assertEqual( + errors[0][2], + "I900 'cprofile' not listed as a requirement", + ) + + def test_3rd_party(self): + errors = check("import foo\nfrom bar import Bar") + self.assertEqual(len(errors), 0) + + def test_3rd_party_missing(self): + errors = check("import os\nfrom cat import Cat") + self.assertEqual(len(errors), 1) + self.assertEqual( + errors[0][2], + "I900 'cat' not listed as a requirement", + ) + + def test_3rd_party_hyphen(self): + errors = check("from hyp_hen import Hyphen") + self.assertEqual(len(errors), 0) + + def test_3rd_party_multi_module(self): + errors = check("import pkg_resources") + self.assertEqual(len(errors), 0) diff --git a/test/test_setup.py b/test/test_setup.py new file mode 100644 index 0000000..f02f95c --- /dev/null +++ b/test/test_setup.py @@ -0,0 +1,66 @@ +import ast +import unittest +from itertools import chain + +from pkg_resources import parse_requirements + +from flake8_requirements.checker import SetupVisitor + + +class Flake8CheckerTestCase(unittest.TestCase): + + def test_detect_setup(self): + code = "setup(name='A', version='1', packages=[''])" + setup = SetupVisitor(ast.parse(code)) + self.assertEqual(setup.redirected, True) + self.assertDictEqual(setup.keywords, { + 'name': 'A', + 'version': '1', + 'packages': [''], + }) + + code = "setup({})".format(",".join( + "{}='{}'".format(x, x) + for x in chain(*SetupVisitor.attributes.values()) + )) + setup = SetupVisitor(ast.parse(code)) + self.assertEqual(setup.redirected, True) + self.assertDictEqual(setup.keywords, { + x: x for x in chain(*SetupVisitor.attributes.values()) + }) + + code = "setup(name='A', version='1')" + setup = SetupVisitor(ast.parse(code)) + self.assertEqual(setup.redirected, False) + + code = "setup(name='A', packages=[''])" + setup = SetupVisitor(ast.parse(code)) + self.assertEqual(setup.redirected, False) + + code = "setup('A', name='A', version='1', packages=[''])" + setup = SetupVisitor(ast.parse(code)) + self.assertEqual(setup.redirected, False) + + code = "setup(name='A', version='1', packages=[''], xxx=False)" + setup = SetupVisitor(ast.parse(code)) + self.assertEqual(setup.redirected, False) + + def test_get_requirements(self): + setup = SetupVisitor(ast.parse("setup(**{})".format(str({ + 'name': 'A', + 'version': '1', + 'packages': [''], + 'install_requires': ["ABC > 1.0.0", "bar.cat > 2, < 3"], + 'extras_require': { + 'extra': ["extra < 10"], + }, + })))) + self.assertEqual(setup.redirected, True) + self.assertEqual( + sorted(setup.get_requirements(), key=lambda x: x.project_name), + sorted(parse_requirements([ + "ABC > 1.0.0", + "bar.cat > 2, < 3", + "extra < 10", + ]), key=lambda x: x.project_name), + )