From 73d5349b08fc81688b46b97582abf48878b58d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 15:50:50 +0100 Subject: [PATCH 01/19] in place merge number of CVE found via summary function --- bin/finder.py | 12 +++++++----- git_vuln_finder/vulnerability.py | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bin/finder.py b/bin/finder.py index db065db..0df0c86 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -75,7 +75,7 @@ def main(): # Initialization of the variables for the results found = 0 all_potential_vulnerabilities = {} - cve_found = set() + all_cve_found = set() repo_heads = repo.heads repo_heads_names = [h.name for h in repo_heads] @@ -94,7 +94,7 @@ def main(): ret = find_vuln(commit, pattern=defaultpattern, verbose=args.v) if ret: rcommit = ret["commit"] - _, potential_vulnerabilities = summary( + _, potential_vulnerabilities, cve_found = summary( repo, rcommit, branch, @@ -106,13 +106,14 @@ def main(): commit_state=args.s, ) all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) found += 1 elif isinstance(defaultpattern, list): for p in defaultpattern: ret = find_vuln(commit, pattern=p, verbose=args.v) if ret: rcommit = ret["commit"] - _, potential_vulnerabilities = summary( + _, potential_vulnerabilities, cve_found = summary( repo, rcommit, branch, @@ -124,15 +125,16 @@ def main(): commit_state=args.s, ) all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) found += 1 if not args.c: print(json.dumps(all_potential_vulnerabilities)) elif args.c: - print(json.dumps(list(cve_found))) + print(json.dumps(list(all_cve_found))) print( - "{} CVE referenced found in commit(s)".format(len(list(cve_found))), + "{} CVE referenced found in commit(s)".format(len(list(all_cve_found))), file=sys.stderr, ) print( diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index d1df6d0..ad4eec6 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -43,7 +43,7 @@ def summary( ): potential_vulnerabilities = {} rcommit = commit - cve = extract_cve(rcommit.message) + cve, cve_found = extract_cve(rcommit.message) if origin is not None: origin = origin if origin.find("github.com"): @@ -98,7 +98,7 @@ def summary( else: potential_vulnerabilities[rcommit.hexsha]["state"] = commit_state - return rcommit.hexsha, potential_vulnerabilities + return rcommit.hexsha, potential_vulnerabilities, cve_found def extract_cve(commit): @@ -108,6 +108,6 @@ def extract_cve(commit): if m: for v in m: cve_found.add(v) - return m + return m, cve_found else: - return None + return None, set() From 552937a7bb1872955bef37452a5b6cac75202ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:29:30 +0100 Subject: [PATCH 02/19] Make the whole thing importable in a Python code. --- bin/finder.py | 80 +------------ git_vuln_finder/__init__.py | 1 + git_vuln_finder/vulnerability.py | 81 ++++++++++++- poetry.lock | 199 ++++++++++++++++++++++++++++++- pyproject.toml | 1 + 5 files changed, 286 insertions(+), 76 deletions(-) diff --git a/bin/finder.py b/bin/finder.py index 0df0c86..f8c692c 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -10,13 +10,11 @@ # Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be -import git import json import sys import argparse -import typing -from git_vuln_finder import get_patterns, find_vuln, summary +from git_vuln_finder import find, find_vuln, summary def main(): @@ -49,84 +47,18 @@ def main(): ) args = parser.parse_args() - patterns = get_patterns() - vulnpatterns = patterns["en"]["medium"]["vuln"] - cryptopatterns = patterns["en"]["medium"]["crypto"] - cpatterns = patterns["en"]["medium"]["c"] - - if args.p == "vulnpatterns": - defaultpattern = vulnpatterns - elif args.p == "cryptopatterns": - defaultpattern = cryptopatterns - elif args.p == "cpatterns": - defaultpattern = cpatterns - elif args.p == "all": - defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] - else: + if args.p not in ["vulnpatterns", "cryptopatterns", "cpatterns", "all"]: parser.print_usage() parser.exit() if not args.r: parser.print_usage() parser.exit() - else: - repo = git.Repo(args.r) - # Initialization of the variables for the results - found = 0 - all_potential_vulnerabilities = {} - all_cve_found = set() - - repo_heads = repo.heads - repo_heads_names = [h.name for h in repo_heads] - print(repo_heads_names, file=sys.stderr) - origin = repo.remotes.origin.url - tagmap = {} - if args.t: - for t in repo.tags: - tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) - - for branch in repo_heads_names: - commits = list(repo.iter_commits(branch)) - defaultpattern - for commit in commits: - if isinstance(defaultpattern, typing.Pattern): - ret = find_vuln(commit, pattern=defaultpattern, verbose=args.v) - if ret: - rcommit = ret["commit"] - _, potential_vulnerabilities, cve_found = summary( - repo, - rcommit, - branch, - tagmap, - defaultpattern, - origin=origin, - vuln_match=ret["match"], - tags_matching=args.t, - commit_state=args.s, - ) - all_potential_vulnerabilities.update(potential_vulnerabilities) - all_cve_found.update(cve_found) - found += 1 - elif isinstance(defaultpattern, list): - for p in defaultpattern: - ret = find_vuln(commit, pattern=p, verbose=args.v) - if ret: - rcommit = ret["commit"] - _, potential_vulnerabilities, cve_found = summary( - repo, - rcommit, - branch, - tagmap, - p, - origin=origin, - vuln_match=ret["match"], - tags_matching=args.t, - commit_state=args.s, - ) - all_potential_vulnerabilities.update(potential_vulnerabilities) - all_cve_found.update(cve_found) - found += 1 + all_potential_vulnerabilities, all_cve_found, found = find( + args.r, tags_matching=args.t, commit_state=args.s, verbose=args.v, + defaultpattern=args.p + ) if not args.c: print(json.dumps(all_potential_vulnerabilities)) diff --git a/git_vuln_finder/__init__.py b/git_vuln_finder/__init__.py index 2bd2216..8dd73fc 100644 --- a/git_vuln_finder/__init__.py +++ b/git_vuln_finder/__init__.py @@ -1,5 +1,6 @@ from git_vuln_finder.pattern import build_pattern from git_vuln_finder.pattern import get_patterns +from git_vuln_finder.vulnerability import find from git_vuln_finder.vulnerability import find_vuln from git_vuln_finder.vulnerability import summary from git_vuln_finder.vulnerability import extract_cve diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index ad4eec6..9d80b56 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -9,11 +9,90 @@ # # Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be - +import git import re import sys +import typing from langdetect import detect as langdetect +from git_vuln_finder import get_patterns + + +def find(repo, tags_matching=False, commit_state="under-review", verbose=False, defaultpattern="all"): + # Initialization of the variables for the results + repo = git.Repo(repo) + found = 0 + all_potential_vulnerabilities = {} + all_cve_found = set() + + # Initialization of the patterns + patterns = get_patterns() + vulnpatterns = patterns["en"]["medium"]["vuln"] + cryptopatterns = patterns["en"]["medium"]["crypto"] + cpatterns = patterns["en"]["medium"]["c"] + + if defaultpattern == "vulnpatterns": + defaultpattern = vulnpatterns + elif defaultpattern == "cryptopatterns": + defaultpattern = cryptopatterns + elif defaultpattern == "cpatterns": + defaultpattern = cpatterns + elif defaultpattern == "all": + defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] + + repo_heads = repo.heads + repo_heads_names = [h.name for h in repo_heads] + print(repo_heads_names, file=sys.stderr) + origin = repo.remotes.origin.url + tagmap = {} + if tags_matching: + for t in repo.tags: + tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) + + for branch in repo_heads_names: + commits = list(repo.iter_commits(branch)) + defaultpattern + for commit in commits: + if isinstance(defaultpattern, typing.Pattern): + ret = find_vuln(commit, pattern=defaultpattern, verbose=verbose) + if ret: + rcommit = ret["commit"] + _, potential_vulnerabilities, cve_found = summary( + repo, + rcommit, + branch, + tagmap, + defaultpattern, + origin=origin, + vuln_match=ret["match"], + tags_matching=tags_matching, + commit_state=commit_state, + ) + all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) + found += 1 + elif isinstance(defaultpattern, list): + for p in defaultpattern: + ret = find_vuln(commit, pattern=p, verbose=verbose) + if ret: + rcommit = ret["commit"] + _, potential_vulnerabilities, cve_found = summary( + repo, + rcommit, + branch, + tagmap, + p, + origin=origin, + vuln_match=ret["match"], + tags_matching=tags_matching, + commit_state=commit_state, + ) + all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) + found += 1 + + return all_potential_vulnerabilities, all_cve_found, found + def find_vuln(commit, pattern, verbose=False): m = pattern.search(commit.message) diff --git a/poetry.lock b/poetry.lock index 02cd4ef..ff2dd23 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,35 @@ +[[package]] +category = "dev" +description = "Atomic file writes." +marker = "sys_platform == \"win32\"" +name = "atomicwrites" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.3.0" + +[[package]] +category = "dev" +description = "Classes Without Boilerplate" +name = "attrs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "19.3.0" + +[package.extras] +azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] +dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] +docs = ["sphinx", "zope.interface"] +tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] + +[[package]] +category = "dev" +description = "Cross-platform colored terminal text." +marker = "sys_platform == \"win32\"" +name = "colorama" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.4.3" + [[package]] category = "dev" description = "Code coverage measurement for Python" @@ -53,6 +85,22 @@ version = "3.0.5" [package.dependencies] gitdb2 = ">=2.0.0" +[[package]] +category = "dev" +description = "Read metadata from Python packages" +marker = "python_version < \"3.8\"" +name = "importlib-metadata" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +version = "1.3.0" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "importlib-resources"] + [[package]] category = "main" description = "Language detection library ported from Google's language-detection." @@ -72,6 +120,14 @@ optional = false python-versions = "*" version = "0.6.1" +[[package]] +category = "dev" +description = "More routines for operating on iterables, beyond itertools" +name = "more-itertools" +optional = false +python-versions = ">=3.5" +version = "8.0.2" + [[package]] category = "dev" description = "Optional static typing for Python" @@ -112,6 +168,42 @@ six = ">=1.7" coverage_plugin = ["coverage (>=4.4.1)"] doc = ["Sphinx (>=1.6.5)", "sphinx-rtd-theme", "mock"] +[[package]] +category = "dev" +description = "Core utilities for Python packages" +name = "packaging" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.0" + +[package.dependencies] +pyparsing = ">=2.0.2" +six = "*" + +[[package]] +category = "dev" +description = "plugin and hook calling mechanisms for python" +name = "pluggy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.13.1" + +[package.dependencies] +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +category = "dev" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "py" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.8.1" + [[package]] category = "dev" description = "Python style guide checker" @@ -128,6 +220,39 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.1.1" +[[package]] +category = "dev" +description = "Python parsing module" +name = "pyparsing" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.4.6" + +[[package]] +category = "dev" +description = "pytest: simple powerful testing with Python" +name = "pytest" +optional = false +python-versions = ">=3.5" +version = "5.3.2" + +[package.dependencies] +atomicwrites = ">=1.0" +attrs = ">=17.4.0" +colorama = "*" +more-itertools = ">=4.0.0" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.5.0" +wcwidth = "*" + +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + [[package]] category = "main" description = "Python 2 and 3 compatibility utilities" @@ -160,11 +285,47 @@ optional = false python-versions = "*" version = "3.7.4.1" +[[package]] +category = "dev" +description = "Measures number of Terminal column cells of wide-character codes" +name = "wcwidth" +optional = false +python-versions = "*" +version = "0.1.8" + +[[package]] +category = "dev" +description = "Backport of pathlib-compatible object wrapper for zip files" +marker = "python_version < \"3.8\"" +name = "zipp" +optional = false +python-versions = ">=2.7" +version = "0.6.0" + +[package.dependencies] +more-itertools = "*" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pathlib2", "contextlib2", "unittest2"] + [metadata] -content-hash = "4fd05852a9f3844298b1c0dbc4ab61ddbb77f4a42602c42982e19e531a7883d6" +content-hash = "9b6cd9aab07a97a57d8b9c67c7dfc9d859f1c743ef2a901004eea7bd976dd040" python-versions = "^3.6" [metadata.files] +atomicwrites = [ + {file = "atomicwrites-1.3.0-py2.py3-none-any.whl", hash = "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4"}, + {file = "atomicwrites-1.3.0.tar.gz", hash = "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"}, +] +attrs = [ + {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, + {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, +] +colorama = [ + {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, + {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, +] coverage = [ {file = "coverage-5.0.1-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:c90bda74e16bcd03861b09b1d37c0a4158feda5d5a036bb2d6e58de6ff65793e"}, {file = "coverage-5.0.1-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:bb3d29df5d07d5399d58a394d0ef50adf303ab4fbf66dfd25b9ef258effcb692"}, @@ -214,6 +375,10 @@ gitpython = [ {file = "GitPython-3.0.5-py3-none-any.whl", hash = "sha256:c155c6a2653593ccb300462f6ef533583a913e17857cfef8fc617c246b6dc245"}, {file = "GitPython-3.0.5.tar.gz", hash = "sha256:9c2398ffc3dcb3c40b27324b316f08a4f93ad646d5a6328cafbb871aa79f5e42"}, ] +importlib-metadata = [ + {file = "importlib_metadata-1.3.0-py2.py3-none-any.whl", hash = "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f"}, + {file = "importlib_metadata-1.3.0.tar.gz", hash = "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45"}, +] langdetect = [ {file = "langdetect-1.0.7.zip", hash = "sha256:91a170d5f0ade380db809b3ba67f08e95fe6c6c8641f96d67a51ff7e98a9bf30"}, ] @@ -221,6 +386,10 @@ mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] +more-itertools = [ + {file = "more-itertools-8.0.2.tar.gz", hash = "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d"}, + {file = "more_itertools-8.0.2-py3-none-any.whl", hash = "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564"}, +] mypy = [ {file = "mypy-0.750-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:de9ec8dba773b78c49e7bec9a35c9b6fc5235682ad1fc2105752ae7c22f4b931"}, {file = "mypy-0.750-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:3294821b5840d51a3cd7a2bb63b40fc3f901f6a3cfb3c6046570749c4c7ef279"}, @@ -245,6 +414,18 @@ nose2 = [ {file = "nose2-0.9.1-py2.py3-none-any.whl", hash = "sha256:31d8beb00aed3ccc6efb1742bb90227d883e471715188249f594310676e0ef0e"}, {file = "nose2-0.9.1.tar.gz", hash = "sha256:0ede156fd7974fa40893edeca0b709f402c0ccacd7b81b22e76f73c116d1b999"}, ] +packaging = [ + {file = "packaging-20.0-py2.py3-none-any.whl", hash = "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb"}, + {file = "packaging-20.0.tar.gz", hash = "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.8.1-py2.py3-none-any.whl", hash = "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"}, + {file = "py-1.8.1.tar.gz", hash = "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa"}, +] pycodestyle = [ {file = "pycodestyle-2.5.0-py2.py3-none-any.whl", hash = "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56"}, {file = "pycodestyle-2.5.0.tar.gz", hash = "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"}, @@ -253,6 +434,14 @@ pyflakes = [ {file = "pyflakes-2.1.1-py2.py3-none-any.whl", hash = "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0"}, {file = "pyflakes-2.1.1.tar.gz", hash = "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"}, ] +pyparsing = [ + {file = "pyparsing-2.4.6-py2.py3-none-any.whl", hash = "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"}, + {file = "pyparsing-2.4.6.tar.gz", hash = "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f"}, +] +pytest = [ + {file = "pytest-5.3.2-py3-none-any.whl", hash = "sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4"}, + {file = "pytest-5.3.2.tar.gz", hash = "sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa"}, +] six = [ {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, @@ -288,3 +477,11 @@ typing-extensions = [ {file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"}, {file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"}, ] +wcwidth = [ + {file = "wcwidth-0.1.8-py2.py3-none-any.whl", hash = "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603"}, + {file = "wcwidth-0.1.8.tar.gz", hash = "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"}, +] +zipp = [ + {file = "zipp-0.6.0-py2.py3-none-any.whl", hash = "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"}, + {file = "zipp-0.6.0.tar.gz", hash = "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e"}, +] diff --git a/pyproject.toml b/pyproject.toml index 95f779a..fcd1a24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ gitpython = "^3.0.5" mypy = "^0.750" flake8 = "^3.7.9" nose2 = "^0.9.1" +pytest = "^5.3.2" [build-system] requires = ["poetry>=0.12"] From ceb2c63458f24997dfb897b278014110b5ead9ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:29:45 +0100 Subject: [PATCH 03/19] added basic test --- tests/test_finder.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/test_finder.py diff --git a/tests/test_finder.py b/tests/test_finder.py new file mode 100644 index 0000000..f990708 --- /dev/null +++ b/tests/test_finder.py @@ -0,0 +1,9 @@ + + +from git_vuln_finder import find + + +def test_find_vuln(): + all_potential_vulnerabilities, all_cve_found, found = find("/home/cedric/git/curl") + + assert len(list(all_cve_found)) == 63 From 86b58987d330854f244289c36523dae5fac769f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:32:50 +0100 Subject: [PATCH 04/19] added GitHub workflow --- .github/workflows/pythonapp.yml | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/pythonapp.yml diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml new file mode 100644 index 0000000..1389c87 --- /dev/null +++ b/.github/workflows/pythonapp.yml @@ -0,0 +1,35 @@ +name: Python application + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + pip install poetry + poetry install + + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Test with pytest + run: | + poetry run pytest + env: + testing: actions From e12d2537935f93bed6f28f5a5d220daf052088c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:33:55 +0100 Subject: [PATCH 05/19] removed nose2 --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fcd1a24..0401c82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ gitpython = "^3.0.5" [tool.poetry.dev-dependencies] mypy = "^0.750" flake8 = "^3.7.9" -nose2 = "^0.9.1" pytest = "^5.3.2" [build-system] From 81786106d027e34136de2a0ffef15be6d4b0b970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:43:43 +0100 Subject: [PATCH 06/19] checkout a repo --- .github/workflows/pythonapp.yml | 5 +++++ tests/test_finder.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 1389c87..10bb338 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -28,6 +28,11 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Checkout curl for the tests + uses: curl/curl@7.67.0 + with: + path: test_repos/curl + - name: Test with pytest run: | poetry run pytest diff --git a/tests/test_finder.py b/tests/test_finder.py index f990708..4bd9e29 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -4,6 +4,6 @@ from git_vuln_finder import find def test_find_vuln(): - all_potential_vulnerabilities, all_cve_found, found = find("/home/cedric/git/curl") + all_potential_vulnerabilities, all_cve_found, found = find("test_repos/curl") assert len(list(all_cve_found)) == 63 From b4ead9123e4366cae389763ac7b3f47af817d72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:49:27 +0100 Subject: [PATCH 07/19] fixed tag of curl --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 10bb338..b69f517 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -29,7 +29,7 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Checkout curl for the tests - uses: curl/curl@7.67.0 + uses: curl/curl@curl-7_67_0 with: path: test_repos/curl From 29cbb7c98225bd73fb0c15e64d8136d393001012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:52:45 +0100 Subject: [PATCH 08/19] checkout curl with actions/checkout@v2 --- .github/workflows/pythonapp.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index b69f517..4386848 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -29,8 +29,10 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Checkout curl for the tests - uses: curl/curl@curl-7_67_0 + uses: actions/checkout@v2 with: + repository: curl/curl + ref: curl-7_67_0 path: test_repos/curl - name: Test with pytest From 86e9f1ff5b1fb1ea625954e3b1b01c4e4a0049db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:58:30 +0100 Subject: [PATCH 09/19] wip --- tests/test_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_finder.py b/tests/test_finder.py index 4bd9e29..77ef6a9 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -4,6 +4,6 @@ from git_vuln_finder import find def test_find_vuln(): - all_potential_vulnerabilities, all_cve_found, found = find("test_repos/curl") + all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl") assert len(list(all_cve_found)) == 63 From 4307c411739cfa26364af08c59d6cfd5fa58540b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 17:00:36 +0100 Subject: [PATCH 10/19] wip --- tests/test_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_finder.py b/tests/test_finder.py index 77ef6a9..07ee774 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -4,6 +4,6 @@ from git_vuln_finder import find def test_find_vuln(): - all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl") + all_potential_vulnerabilities, all_cve_found, found = find("/home/runner/work/git-vuln-finder/git-vuln-finder/test_repos/curl/") assert len(list(all_cve_found)) == 63 From a61719bf16b18f0951c43e116b8842c7b691139f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 17:04:10 +0100 Subject: [PATCH 11/19] Updated README> --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 5100ed8..59dd589 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,14 @@ $ poetry shell You can also use ``pip``. +~~~python +Python 3.8.0 (default, Dec 11 2019, 21:43:13) +[GCC 9.2.1 20191008] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> from git_vuln_finder import find +>>> all_potential_vulnerabilities, all_cve_found, found = find("~/git/curl") +~~~ + ## Use as a command line tool From a4a17ba265e3bf75e39ac8dc8a0007d0a4775de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:10:09 +0100 Subject: [PATCH 12/19] checkout version curl-7_67_0 of curl for the tests. --- .github/workflows/pythonapp.yml | 7 ------- tests/test_finder.py | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 4386848..1389c87 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -28,13 +28,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Checkout curl for the tests - uses: actions/checkout@v2 - with: - repository: curl/curl - ref: curl-7_67_0 - path: test_repos/curl - - name: Test with pytest run: | poetry run pytest diff --git a/tests/test_finder.py b/tests/test_finder.py index 07ee774..35cb2a2 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -3,7 +3,7 @@ from git_vuln_finder import find -def test_find_vuln(): - all_potential_vulnerabilities, all_cve_found, found = find("/home/runner/work/git-vuln-finder/git-vuln-finder/test_repos/curl/") +def test_find_vuln(clone_curl): + all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl/") assert len(list(all_cve_found)) == 63 From 1aec0290af72f91db6eedd648fba0bb637e3dea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:10:30 +0100 Subject: [PATCH 13/19] checkout version curl-7_67_0 of curl for the tests. --- tests/conftest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5267a2e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ + +import os +import pytest + +from git import Repo + + +@pytest.fixture(scope='session') +def clone_curl(): + """Clone the repository of curl for the tests.""" + git_url = 'https://github.com/curl/curl.git' + repo_dir = './test_repos/curl' + repo = Repo.clone_from(url=git_url, to_path=repo_dir) + repo.checkout('curl-7_67_0') + + def teardown(): + os.unlink(repo_dir) + + return repo_dir From 2e1d646f994bbf6d2cc2a24833c0b21a938583e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:11:43 +0100 Subject: [PATCH 14/19] checkout version curl-7_67_0 of curl for the tests. --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5267a2e..2b6f692 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ def clone_curl(): git_url = 'https://github.com/curl/curl.git' repo_dir = './test_repos/curl' repo = Repo.clone_from(url=git_url, to_path=repo_dir) - repo.checkout('curl-7_67_0') + repo.heads['curl-7_67_0'].checkout() def teardown(): os.unlink(repo_dir) From 43c8c432b896e1b98b059c90d53fef7239762288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:12:57 +0100 Subject: [PATCH 15/19] updated gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 119966c..88e45aa 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ dist/* .coverage .mypy_cache/ .cache/ +.test_repos/ # sphinx docs/_build From d2b88a3e6cb714774700f59eb10a627eb60ea519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:16:47 +0100 Subject: [PATCH 16/19] updated README. --- README.md | 4 ++-- tests/conftest.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 59dd589..f17f628 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ added automatically in the output. # Installation -## Use in your Python software +## Use it as a library ~~~bash $ poetry install git-vuln-finder @@ -33,7 +33,7 @@ Type "help", "copyright", "credits" or "license" for more information. ~~~ -## Use as a command line tool +## Use it as a command line tool ~~~bash $ pipx install git-vuln-finder diff --git a/tests/conftest.py b/tests/conftest.py index 2b6f692..e37d33b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ def clone_curl(): git_url = 'https://github.com/curl/curl.git' repo_dir = './test_repos/curl' repo = Repo.clone_from(url=git_url, to_path=repo_dir) - repo.heads['curl-7_67_0'].checkout() + #repo.heads['curl-7_67_0'].checkout() def teardown(): os.unlink(repo_dir) From 864def653a0abdccb094d4fda982bdecb7be6807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:24:30 +0100 Subject: [PATCH 17/19] updated the tests --- tests/test_finder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_finder.py b/tests/test_finder.py index 35cb2a2..4625ea2 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -6,4 +6,5 @@ from git_vuln_finder import find def test_find_vuln(clone_curl): all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl/") - assert len(list(all_cve_found)) == 63 + #assert len(list(all_cve_found)) == 64 + assert "CVE-2018-1000122" in all_cve_found From d43bddd65a20808ef3ff81c99cf8ef1dcdb19e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:28:24 +0100 Subject: [PATCH 18/19] added badge with temporary address --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f17f628..8b14f9d 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ ![git-vuln-finder logo](https://raw.githubusercontent.com/cve-search/git-vuln-finder/f22077452c37e110bff0564e1f7b34637dc726c3/doc/logos/git-vuln-finder-small.png) +[![Workflow](https://github.com/cedricbonhomme/git-vuln-finder/workflows/Python%20application/badge.svg?style=flat-square)](https://github.com/cedricbonhomme/git-vuln-finder/actions?query=workflow%3A%22Python+application%22) + Finding potential software vulnerabilities from git commit messages. The output format is a JSON with the associated commit which could contain a fix regarding a software vulnerability. The search is based on a set of regular From aab859b2632850d0ad29285e76f04752d40b9caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:30:27 +0100 Subject: [PATCH 19/19] Updated README. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8b14f9d..73b8afd 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ $ poetry install git-vuln-finder $ poetry shell ~~~ -You can also use ``pip``. +You can also use ``pip``. Then just import it: ~~~python Python 3.8.0 (default, Dec 11 2019, 21:43:13) @@ -46,7 +46,7 @@ $ git-vuln-finder --help separate virtualenvs to shield them from your system and each other. -# Usage +### Usage ~~~bash usage: git-vuln-finder [-h] [-v] [-r R] [-o O] [-s S] [-p P] [-c] [-t]