From 552937a7bb1872955bef37452a5b6cac75202ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:29:30 +0100 Subject: [PATCH] Make the whole thing importable in a Python code. --- bin/finder.py | 80 +------------ git_vuln_finder/__init__.py | 1 + git_vuln_finder/vulnerability.py | 81 ++++++++++++- poetry.lock | 199 ++++++++++++++++++++++++++++++- pyproject.toml | 1 + 5 files changed, 286 insertions(+), 76 deletions(-) diff --git a/bin/finder.py b/bin/finder.py index 0df0c86..f8c692c 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -10,13 +10,11 @@ # Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be -import git import json import sys import argparse -import typing -from git_vuln_finder import get_patterns, find_vuln, summary +from git_vuln_finder import find, find_vuln, summary def main(): @@ -49,84 +47,18 @@ def main(): ) args = parser.parse_args() - patterns = get_patterns() - vulnpatterns = patterns["en"]["medium"]["vuln"] - cryptopatterns = patterns["en"]["medium"]["crypto"] - cpatterns = patterns["en"]["medium"]["c"] - - if args.p == "vulnpatterns": - defaultpattern = vulnpatterns - elif args.p == "cryptopatterns": - defaultpattern = cryptopatterns - elif args.p == "cpatterns": - defaultpattern = cpatterns - elif args.p == "all": - defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] - else: + if args.p not in ["vulnpatterns", "cryptopatterns", "cpatterns", "all"]: parser.print_usage() parser.exit() if not args.r: parser.print_usage() parser.exit() - else: - repo = git.Repo(args.r) - # Initialization of the variables for the results - found = 0 - all_potential_vulnerabilities = {} - all_cve_found = set() - - repo_heads = repo.heads - repo_heads_names = [h.name for h in repo_heads] - print(repo_heads_names, file=sys.stderr) - origin = repo.remotes.origin.url - tagmap = {} - if args.t: - for t in repo.tags: - tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) - - for branch in repo_heads_names: - commits = list(repo.iter_commits(branch)) - defaultpattern - for commit in commits: - if isinstance(defaultpattern, typing.Pattern): - ret = find_vuln(commit, pattern=defaultpattern, verbose=args.v) - if ret: - rcommit = ret["commit"] - _, potential_vulnerabilities, cve_found = summary( - repo, - rcommit, - branch, - tagmap, - defaultpattern, - origin=origin, - vuln_match=ret["match"], - tags_matching=args.t, - commit_state=args.s, - ) - all_potential_vulnerabilities.update(potential_vulnerabilities) - all_cve_found.update(cve_found) - found += 1 - elif isinstance(defaultpattern, list): - for p in defaultpattern: - ret = find_vuln(commit, pattern=p, verbose=args.v) - if ret: - rcommit = ret["commit"] - _, potential_vulnerabilities, cve_found = summary( - repo, - rcommit, - branch, - tagmap, - p, - origin=origin, - vuln_match=ret["match"], - tags_matching=args.t, - commit_state=args.s, - ) - all_potential_vulnerabilities.update(potential_vulnerabilities) - all_cve_found.update(cve_found) - found += 1 + all_potential_vulnerabilities, all_cve_found, found = find( + args.r, tags_matching=args.t, commit_state=args.s, verbose=args.v, + defaultpattern=args.p + ) if not args.c: print(json.dumps(all_potential_vulnerabilities)) diff --git a/git_vuln_finder/__init__.py b/git_vuln_finder/__init__.py index 2bd2216..8dd73fc 100644 --- a/git_vuln_finder/__init__.py +++ b/git_vuln_finder/__init__.py @@ -1,5 +1,6 @@ from git_vuln_finder.pattern import build_pattern from git_vuln_finder.pattern import get_patterns +from git_vuln_finder.vulnerability import find from git_vuln_finder.vulnerability import find_vuln from git_vuln_finder.vulnerability import summary from git_vuln_finder.vulnerability import extract_cve diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index ad4eec6..9d80b56 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -9,11 +9,90 @@ # # Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be - +import git import re import sys +import typing from langdetect import detect as langdetect +from git_vuln_finder import get_patterns + + +def find(repo, tags_matching=False, commit_state="under-review", verbose=False, defaultpattern="all"): + # Initialization of the variables for the results + repo = git.Repo(repo) + found = 0 + all_potential_vulnerabilities = {} + all_cve_found = set() + + # Initialization of the patterns + patterns = get_patterns() + vulnpatterns = patterns["en"]["medium"]["vuln"] + cryptopatterns = patterns["en"]["medium"]["crypto"] + cpatterns = patterns["en"]["medium"]["c"] + + if defaultpattern == "vulnpatterns": + defaultpattern = vulnpatterns + elif defaultpattern == "cryptopatterns": + defaultpattern = cryptopatterns + elif defaultpattern == "cpatterns": + defaultpattern = cpatterns + elif defaultpattern == "all": + defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] + + repo_heads = repo.heads + repo_heads_names = [h.name for h in repo_heads] + print(repo_heads_names, file=sys.stderr) + origin = repo.remotes.origin.url + tagmap = {} + if tags_matching: + for t in repo.tags: + tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) + + for branch in repo_heads_names: + commits = list(repo.iter_commits(branch)) + defaultpattern + for commit in commits: + if isinstance(defaultpattern, typing.Pattern): + ret = find_vuln(commit, pattern=defaultpattern, verbose=verbose) + if ret: + rcommit = ret["commit"] + _, potential_vulnerabilities, cve_found = summary( + repo, + rcommit, + branch, + tagmap, + defaultpattern, + origin=origin, + vuln_match=ret["match"], + tags_matching=tags_matching, + commit_state=commit_state, + ) + all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) + found += 1 + elif isinstance(defaultpattern, list): + for p in defaultpattern: + ret = find_vuln(commit, pattern=p, verbose=verbose) + if ret: + rcommit = ret["commit"] + _, potential_vulnerabilities, cve_found = summary( + repo, + rcommit, + branch, + tagmap, + p, + origin=origin, + vuln_match=ret["match"], + tags_matching=tags_matching, + commit_state=commit_state, + ) + all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) + found += 1 + + return all_potential_vulnerabilities, all_cve_found, found + def find_vuln(commit, pattern, verbose=False): m = pattern.search(commit.message) diff --git a/poetry.lock b/poetry.lock index 02cd4ef..ff2dd23 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,35 @@ +[[package]] +category = "dev" +description = "Atomic file writes." +marker = "sys_platform == \"win32\"" +name = "atomicwrites" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.3.0" + +[[package]] +category = "dev" +description = "Classes Without Boilerplate" +name = "attrs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "19.3.0" + +[package.extras] +azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] +dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] +docs = ["sphinx", "zope.interface"] +tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] + +[[package]] +category = "dev" +description = "Cross-platform colored terminal text." +marker = "sys_platform == \"win32\"" +name = "colorama" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.4.3" + [[package]] category = "dev" description = "Code coverage measurement for Python" @@ -53,6 +85,22 @@ version = "3.0.5" [package.dependencies] gitdb2 = ">=2.0.0" +[[package]] +category = "dev" +description = "Read metadata from Python packages" +marker = "python_version < \"3.8\"" +name = "importlib-metadata" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +version = "1.3.0" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "importlib-resources"] + [[package]] category = "main" description = "Language detection library ported from Google's language-detection." @@ -72,6 +120,14 @@ optional = false python-versions = "*" version = "0.6.1" +[[package]] +category = "dev" +description = "More routines for operating on iterables, beyond itertools" +name = "more-itertools" +optional = false +python-versions = ">=3.5" +version = "8.0.2" + [[package]] category = "dev" description = "Optional static typing for Python" @@ -112,6 +168,42 @@ six = ">=1.7" coverage_plugin = ["coverage (>=4.4.1)"] doc = ["Sphinx (>=1.6.5)", "sphinx-rtd-theme", "mock"] +[[package]] +category = "dev" +description = "Core utilities for Python packages" +name = "packaging" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.0" + +[package.dependencies] +pyparsing = ">=2.0.2" +six = "*" + +[[package]] +category = "dev" +description = "plugin and hook calling mechanisms for python" +name = "pluggy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.13.1" + +[package.dependencies] +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +category = "dev" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "py" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.8.1" + [[package]] category = "dev" description = "Python style guide checker" @@ -128,6 +220,39 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.1.1" +[[package]] +category = "dev" +description = "Python parsing module" +name = "pyparsing" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.4.6" + +[[package]] +category = "dev" +description = "pytest: simple powerful testing with Python" +name = "pytest" +optional = false +python-versions = ">=3.5" +version = "5.3.2" + +[package.dependencies] +atomicwrites = ">=1.0" +attrs = ">=17.4.0" +colorama = "*" +more-itertools = ">=4.0.0" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.5.0" +wcwidth = "*" + +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + [[package]] category = "main" description = "Python 2 and 3 compatibility utilities" @@ -160,11 +285,47 @@ optional = false python-versions = "*" version = "3.7.4.1" +[[package]] +category = "dev" +description = "Measures number of Terminal column cells of wide-character codes" +name = "wcwidth" +optional = false +python-versions = "*" +version = "0.1.8" + +[[package]] +category = "dev" +description = "Backport of pathlib-compatible object wrapper for zip files" +marker = "python_version < \"3.8\"" +name = "zipp" +optional = false +python-versions = ">=2.7" +version = "0.6.0" + +[package.dependencies] +more-itertools = "*" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pathlib2", "contextlib2", "unittest2"] + [metadata] -content-hash = "4fd05852a9f3844298b1c0dbc4ab61ddbb77f4a42602c42982e19e531a7883d6" +content-hash = "9b6cd9aab07a97a57d8b9c67c7dfc9d859f1c743ef2a901004eea7bd976dd040" python-versions = "^3.6" [metadata.files] +atomicwrites = [ + {file = "atomicwrites-1.3.0-py2.py3-none-any.whl", hash = "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4"}, + {file = "atomicwrites-1.3.0.tar.gz", hash = "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"}, +] +attrs = [ + {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, + {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, +] +colorama = [ + {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, + {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, +] coverage = [ {file = "coverage-5.0.1-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:c90bda74e16bcd03861b09b1d37c0a4158feda5d5a036bb2d6e58de6ff65793e"}, {file = "coverage-5.0.1-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:bb3d29df5d07d5399d58a394d0ef50adf303ab4fbf66dfd25b9ef258effcb692"}, @@ -214,6 +375,10 @@ gitpython = [ {file = "GitPython-3.0.5-py3-none-any.whl", hash = "sha256:c155c6a2653593ccb300462f6ef533583a913e17857cfef8fc617c246b6dc245"}, {file = "GitPython-3.0.5.tar.gz", hash = "sha256:9c2398ffc3dcb3c40b27324b316f08a4f93ad646d5a6328cafbb871aa79f5e42"}, ] +importlib-metadata = [ + {file = "importlib_metadata-1.3.0-py2.py3-none-any.whl", hash = "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f"}, + {file = "importlib_metadata-1.3.0.tar.gz", hash = "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45"}, +] langdetect = [ {file = "langdetect-1.0.7.zip", hash = "sha256:91a170d5f0ade380db809b3ba67f08e95fe6c6c8641f96d67a51ff7e98a9bf30"}, ] @@ -221,6 +386,10 @@ mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] +more-itertools = [ + {file = "more-itertools-8.0.2.tar.gz", hash = "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d"}, + {file = "more_itertools-8.0.2-py3-none-any.whl", hash = "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564"}, +] mypy = [ {file = "mypy-0.750-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:de9ec8dba773b78c49e7bec9a35c9b6fc5235682ad1fc2105752ae7c22f4b931"}, {file = "mypy-0.750-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:3294821b5840d51a3cd7a2bb63b40fc3f901f6a3cfb3c6046570749c4c7ef279"}, @@ -245,6 +414,18 @@ nose2 = [ {file = "nose2-0.9.1-py2.py3-none-any.whl", hash = "sha256:31d8beb00aed3ccc6efb1742bb90227d883e471715188249f594310676e0ef0e"}, {file = "nose2-0.9.1.tar.gz", hash = "sha256:0ede156fd7974fa40893edeca0b709f402c0ccacd7b81b22e76f73c116d1b999"}, ] +packaging = [ + {file = "packaging-20.0-py2.py3-none-any.whl", hash = "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb"}, + {file = "packaging-20.0.tar.gz", hash = "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.8.1-py2.py3-none-any.whl", hash = "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"}, + {file = "py-1.8.1.tar.gz", hash = "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa"}, +] pycodestyle = [ {file = "pycodestyle-2.5.0-py2.py3-none-any.whl", hash = "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56"}, {file = "pycodestyle-2.5.0.tar.gz", hash = "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"}, @@ -253,6 +434,14 @@ pyflakes = [ {file = "pyflakes-2.1.1-py2.py3-none-any.whl", hash = "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0"}, {file = "pyflakes-2.1.1.tar.gz", hash = "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"}, ] +pyparsing = [ + {file = "pyparsing-2.4.6-py2.py3-none-any.whl", hash = "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"}, + {file = "pyparsing-2.4.6.tar.gz", hash = "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f"}, +] +pytest = [ + {file = "pytest-5.3.2-py3-none-any.whl", hash = "sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4"}, + {file = "pytest-5.3.2.tar.gz", hash = "sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa"}, +] six = [ {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, @@ -288,3 +477,11 @@ typing-extensions = [ {file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"}, {file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"}, ] +wcwidth = [ + {file = "wcwidth-0.1.8-py2.py3-none-any.whl", hash = "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603"}, + {file = "wcwidth-0.1.8.tar.gz", hash = "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"}, +] +zipp = [ + {file = "zipp-0.6.0-py2.py3-none-any.whl", hash = "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"}, + {file = "zipp-0.6.0.tar.gz", hash = "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e"}, +] diff --git a/pyproject.toml b/pyproject.toml index 95f779a..fcd1a24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ gitpython = "^3.0.5" mypy = "^0.750" flake8 = "^3.7.9" nose2 = "^0.9.1" +pytest = "^5.3.2" [build-system] requires = ["poetry>=0.12"]