From dcb0fcc0a512ccc12902ce3831dd3f419093f0b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Fri, 3 Jan 2020 16:06:33 +0100 Subject: [PATCH 01/42] wip, but seems to be working as a cli --- README.md | 26 ++++- REQUIREMENTS | 2 - _config.yml | 1 - bin/finder.py | 169 ++++++------------------------- patterns/en/medium/c | 4 - patterns/en/medium/c.prefix | 1 - patterns/en/medium/c.suffix | 1 - patterns/en/medium/crypto | 55 ---------- patterns/en/medium/crypto.prefix | 1 - patterns/en/medium/crypto.suffix | 1 - patterns/en/medium/vuln | 30 ------ patterns/en/medium/vuln.prefix | 1 - patterns/en/medium/vuln.suffix | 1 - 13 files changed, 55 insertions(+), 238 deletions(-) delete mode 100644 REQUIREMENTS delete mode 100644 _config.yml delete mode 100644 patterns/en/medium/c delete mode 100644 patterns/en/medium/c.prefix delete mode 100644 patterns/en/medium/c.suffix delete mode 100644 patterns/en/medium/crypto delete mode 100644 patterns/en/medium/crypto.prefix delete mode 100644 patterns/en/medium/crypto.suffix delete mode 100644 patterns/en/medium/vuln delete mode 100644 patterns/en/medium/vuln.prefix delete mode 100644 patterns/en/medium/vuln.suffix diff --git a/README.md b/README.md index 5487ab7..e5cbd90 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,11 @@ ![git-vuln-finder logo](https://raw.githubusercontent.com/cve-search/git-vuln-finder/f22077452c37e110bff0564e1f7b34637dc726c3/doc/logos/git-vuln-finder-small.png) -Finding potential software vulnerabilities from git commit messages. The output format is a JSON with the associated commit which could contain a fix regarding a software vulnerability. The search is based on a set of regular expressions against the commit messages only. If CVE IDs are present, those are added automatically in the output. +Finding potential software vulnerabilities from git commit messages. +The output format is a JSON with the associated commit which could contain a +fix regarding a software vulnerability. The search is based on a set of regular +expressions against the commit messages only. If CVE IDs are present, those are +added automatically in the output. # Requirements @@ -10,6 +14,24 @@ Finding potential software vulnerabilities from git commit messages. The output - GitPython - langdetect + +# Installation + +## Use in your Python software + +~~~bash +$ poetry install git-vuln-finder +$ poetry shell +~~~ + +## Use as a command line tool + +~~~bash +$ pipx install git-vuln-finder +$ finder --help +~~~ + + # Usage ~~~bash @@ -166,5 +188,3 @@ collectively own this open source software. The contributors acknowledge the [De - [Notes](https://gist.github.com/adulau/dce5a6ca5c65017869bb01dfee576303#file-finding-vuln-git-commit-messages-md) - https://csce.ucmss.com/cr/books/2017/LFS/CSREA2017/ICA2077.pdf (mainly using CVE referenced in the commit message) - archive (http://archive.is/xep9o) - https://asankhaya.github.io/pdf/automated-identification-of-security-issues-from-commit-messages-and-bug-reports.pdf (2 main regexps) - - diff --git a/REQUIREMENTS b/REQUIREMENTS deleted file mode 100644 index be616c3..0000000 --- a/REQUIREMENTS +++ /dev/null @@ -1,2 +0,0 @@ -gitpython -langdetect diff --git a/_config.yml b/_config.yml deleted file mode 100644 index 2f7efbe..0000000 --- a/_config.yml +++ /dev/null @@ -1 +0,0 @@ -theme: jekyll-theme-minimal \ No newline at end of file diff --git a/bin/finder.py b/bin/finder.py index f7d79ab..9e26600 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # -*- coding: utf-8 -*- # # Finding potential software vulnerabilities from git commit messages @@ -10,16 +10,21 @@ # Copyright (c) 2019 Alexandre Dulaunoy - a@foo.be -import os -import re import git import json import sys import argparse import typing -from langdetect import detect as langdetect -PATTERNS_PATH="../patterns" +from git_vuln_finder import ( + build_pattern, + get_patterns, + find_vuln, + summary, + extract_cve +) + +PATTERNS_PATH="./git_vuln_finder/patterns" parser = argparse.ArgumentParser(description = "Finding potential software vulnerabilities from git commit messages.", epilog = "More info: https://github.com/cve-search/git-vuln-finder") parser.add_argument("-v", help="increase output verbosity", action="store_true") @@ -32,63 +37,6 @@ parser.add_argument("-t", help="Include tags matching a specific commit", action args = parser.parse_args() -def build_pattern(pattern_file): - fp = open(pattern_file, "r") - rex = "" - try: - prefix_fp = open(pattern_file + ".prefix", "r") - rex += prefix_fp.read() - prefix_fp.close() - except: - pass - - for line in fp.readlines(): - rex += line.rstrip() + "|" - rex = rex[:-1] # We remove the extra '| - fp.close() - - try: - suffix_fp = open(pattern_file + ".suffix", "r") - rex += suffix_fp.read() - suffix_fp.close() - except: - pass - - return rex - -def get_patterns(patterns_path=PATTERNS_PATH): - patterns = {} - for root, dirs, files in os.walk(patterns_path): - path = root.split(os.sep) - for f in files: - if f.endswith(".prefix") or f.endswith(".suffix"): - continue - npath = root[len(patterns_path):].split(os.sep) - try: - npath.remove('') - except ValueError: - pass - - lang = npath[0] - severity = npath[1] - pattern_category = f - - try: # FIXME: Is there a better way? - a = patterns[lang] - except KeyError: - patterns[lang] = {} - try: - a = patterns[lang][severity] - except KeyError: - patterns[lang][severity] = {} - try: - a = patterns[lang][severity][pattern_category] - except KeyError: - rex = build_pattern(root + os.sep + f) - patterns[lang][severity][pattern_category] = re.compile(rex) - - return patterns - patterns = get_patterns() vulnpatterns = patterns["en"]["medium"]["vuln"] cryptopatterns = patterns["en"]["medium"]["crypto"] @@ -114,79 +62,12 @@ else: found = 0 -potential_vulnerabilities = {} +all_potential_vulnerabilities = {} cve_found = set() -def find_vuln(commit, pattern=vulnpatterns): - m = pattern.search(commit.message) - if m: - if args.v: - print("Match found: {}".format(m.group(0)), file=sys.stderr) - print(commit.message, file=sys.stderr) - print("---", file=sys.stderr) - ret = {} - ret['commit'] = commit - ret['match'] = m.groups() - return ret - else: - return None +def main(): + pass -def summary(commit, branch, pattern, origin=None): - rcommit = commit - cve = extract_cve(rcommit.message) - if origin is not None: - origin = origin - if origin.find('github.com'): - origin_github_api = origin.split(':')[1] - (org_name, repo_name) = origin_github_api.split('/', 1) - if repo_name.find('.git$'): - repo_name = re.sub(r".git$","", repo_name) - origin_github_api = 'https://api.github.com/repos/{}/{}/commits/{}'.format(org_name, repo_name, rcommit.hexsha) - - else: - origin = 'git origin unknown' - # deduplication if similar commits on different branches - if rcommit.hexsha in potential_vulnerabilities: - potential_vulnerabilities[rcommit.hexsha]['branches'].append(branch) - else: - potential_vulnerabilities[rcommit.hexsha] = {} - potential_vulnerabilities[rcommit.hexsha]['message'] = rcommit.message - potential_vulnerabilities[rcommit.hexsha]['language'] = langdetect(rcommit.message) - potential_vulnerabilities[rcommit.hexsha]['commit-id'] = rcommit.hexsha - potential_vulnerabilities[rcommit.hexsha]['summary'] = rcommit.summary - potential_vulnerabilities[rcommit.hexsha]['stats'] = rcommit.stats.total - potential_vulnerabilities[rcommit.hexsha]['author'] = rcommit.author.name - potential_vulnerabilities[rcommit.hexsha]['author-email'] = rcommit.author.email - potential_vulnerabilities[rcommit.hexsha]['authored_date'] = rcommit.authored_date - potential_vulnerabilities[rcommit.hexsha]['committed_date'] = rcommit.committed_date - potential_vulnerabilities[rcommit.hexsha]['branches'] = [] - potential_vulnerabilities[rcommit.hexsha]['branches'].append(branch) - potential_vulnerabilities[rcommit.hexsha]['pattern-selected'] = pattern.pattern - potential_vulnerabilities[rcommit.hexsha]['pattern-matches'] = ret['match'] - potential_vulnerabilities[rcommit.hexsha]['origin'] = origin - if origin_github_api: - potential_vulnerabilities[commit.hexsha]['origin-github-api'] = origin_github_api - potential_vulnerabilities[rcommit.hexsha]['tags'] = [] - if args.t: - if repo.commit(rcommit).hexsha in tagmap: - potential_vulnerabilities[rcommit.hexsha]['tags'] = tagmap[repo.commit(rcommit).hexsha] - if cve: potential_vulnerabilities[rcommit.hexsha]['cve'] = cve - if cve: - potential_vulnerabilities[rcommit.hexsha]['state'] = "cve-assigned" - else: - potential_vulnerabilities[rcommit.hexsha]['state'] = args.s - - return rcommit.hexsha - -def extract_cve(commit): - cve_find = re.compile(r'CVE-[1-2]\d{1,4}-\d{1,7}', re.IGNORECASE) - m = cve_find.findall(commit) - if m: - for v in m: - cve_found.add(v) - return m - else: - return None repo_heads = repo.heads repo_heads_names = [h.name for h in repo_heads] @@ -202,20 +83,34 @@ for branch in repo_heads_names: defaultpattern for commit in commits: if isinstance(defaultpattern, typing.Pattern): - ret = find_vuln(commit, pattern=defaultpattern) + ret = find_vuln(commit, pattern=defaultpattern, versbose=args.v) if ret: rcommit = ret['commit'] - summary(rcommit, branch, defaultpattern, origin=origin) + _, potential_vulnerabilities = summary(rcommit, + branch, + defaultpattern, + origin=origin, + vuln_match=ret['match'], + tags_matching=args.t, + commit_state=args.s) + all_potential_vulnerabilities.update(potential_vulnerabilities) found += 1 elif isinstance(defaultpattern, list): for p in defaultpattern: - ret = find_vuln(commit, pattern=p) + ret = find_vuln(commit, pattern=p, versbose=args.v) if ret: rcommit = ret['commit'] - summary(rcommit, branch, p, origin=origin) + _, potential_vulnerabilities = summary(rcommit, + branch, + p, + origin=origin, + vuln_match=ret['match'], + tags_matching=args.t, + commit_state=args.s) + all_potential_vulnerabilities.update(potential_vulnerabilities) found += 1 if not args.c: - print(json.dumps(potential_vulnerabilities)) + print(json.dumps(all_potential_vulnerabilities)) elif args.c: print(json.dumps(list(cve_found))) diff --git a/patterns/en/medium/c b/patterns/en/medium/c deleted file mode 100644 index 42d4a36..0000000 --- a/patterns/en/medium/c +++ /dev/null @@ -1,4 +0,0 @@ -double[-| ]free -buffer overflow -double free -race[-| ]condition diff --git a/patterns/en/medium/c.prefix b/patterns/en/medium/c.prefix deleted file mode 100644 index 4a45df8..0000000 --- a/patterns/en/medium/c.prefix +++ /dev/null @@ -1 +0,0 @@ -(?i)( \ No newline at end of file diff --git a/patterns/en/medium/c.suffix b/patterns/en/medium/c.suffix deleted file mode 100644 index e8a0f87..0000000 --- a/patterns/en/medium/c.suffix +++ /dev/null @@ -1 +0,0 @@ -) \ No newline at end of file diff --git a/patterns/en/medium/crypto b/patterns/en/medium/crypto deleted file mode 100644 index b24e6c2..0000000 --- a/patterns/en/medium/crypto +++ /dev/null @@ -1,55 +0,0 @@ -assessment -lack of -bad -vulnerable -missing -unproper -unsuitable -breakable -broken -weak -incorrect -replace -assessment -pen([\s-]?)test -pentest -penetration([\s-]?)test -report -vulnerablity -replace -fix -issue -fixes -add -remove -check){s1,} - (crypto -cryptographic -cryptography -encipherement -encryption -ciphers -cipher -AES -DES -3DES -cipher -GPG -PGP -OpenSSL -SSH -wireguard -VPN -CBC -ECB -CTR -key[.|,|\s] -private([\s-]?)key -public([\s-]?)key size -length -strenght -generation -randomness -entropy -prng -rng diff --git a/patterns/en/medium/crypto.prefix b/patterns/en/medium/crypto.prefix deleted file mode 100644 index f5b5f62..0000000 --- a/patterns/en/medium/crypto.prefix +++ /dev/null @@ -1 +0,0 @@ -.*( \ No newline at end of file diff --git a/patterns/en/medium/crypto.suffix b/patterns/en/medium/crypto.suffix deleted file mode 100644 index 12953cd..0000000 --- a/patterns/en/medium/crypto.suffix +++ /dev/null @@ -1 +0,0 @@ -){1,} \ No newline at end of file diff --git a/patterns/en/medium/vuln b/patterns/en/medium/vuln deleted file mode 100644 index db3ca53..0000000 --- a/patterns/en/medium/vuln +++ /dev/null @@ -1,30 +0,0 @@ -denial of service -\bXXE\b -remote code execution -\bopen redirect -OSVDB -\bvuln -\bCVE\b -\bXSS\b -\bReDoS\b -\bNVD\b -malicious -x−frame−options -attack -cross site -exploit -malicious -directory traversal -\bRCE\b -\bdos\b -\bXSRF \b -\bXSS\b -clickjack -session.fixation -hijack -\badvisory -\binsecure -security -\bcross−origin\b -unauthori[z|s]ed -infinite loop \ No newline at end of file diff --git a/patterns/en/medium/vuln.prefix b/patterns/en/medium/vuln.prefix deleted file mode 100644 index 4a45df8..0000000 --- a/patterns/en/medium/vuln.prefix +++ /dev/null @@ -1 +0,0 @@ -(?i)( \ No newline at end of file diff --git a/patterns/en/medium/vuln.suffix b/patterns/en/medium/vuln.suffix deleted file mode 100644 index e8a0f87..0000000 --- a/patterns/en/medium/vuln.suffix +++ /dev/null @@ -1 +0,0 @@ -) \ No newline at end of file From 0b31bc36b2b5ea628604e57c3e0cf6e759e88f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Fri, 3 Jan 2020 16:07:40 +0100 Subject: [PATCH 02/42] added pyproject.toml file --- git_vuln_finder/__init__.py | 6 + .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 342 bytes .../__pycache__/finder.cpython-38.pyc | Bin 0 -> 3427 bytes git_vuln_finder/finder.py | 164 ++++++++++ git_vuln_finder/patterns/en/medium/c | 4 + git_vuln_finder/patterns/en/medium/c.prefix | 1 + git_vuln_finder/patterns/en/medium/c.suffix | 1 + git_vuln_finder/patterns/en/medium/crypto | 55 ++++ .../patterns/en/medium/crypto.prefix | 1 + .../patterns/en/medium/crypto.suffix | 1 + git_vuln_finder/patterns/en/medium/vuln | 30 ++ .../patterns/en/medium/vuln.prefix | 1 + .../patterns/en/medium/vuln.suffix | 1 + poetry.lock | 290 ++++++++++++++++++ pyproject.toml | 59 ++++ 15 files changed, 614 insertions(+) create mode 100644 git_vuln_finder/__init__.py create mode 100644 git_vuln_finder/__pycache__/__init__.cpython-38.pyc create mode 100644 git_vuln_finder/__pycache__/finder.cpython-38.pyc create mode 100644 git_vuln_finder/finder.py create mode 100644 git_vuln_finder/patterns/en/medium/c create mode 100644 git_vuln_finder/patterns/en/medium/c.prefix create mode 100644 git_vuln_finder/patterns/en/medium/c.suffix create mode 100644 git_vuln_finder/patterns/en/medium/crypto create mode 100644 git_vuln_finder/patterns/en/medium/crypto.prefix create mode 100644 git_vuln_finder/patterns/en/medium/crypto.suffix create mode 100644 git_vuln_finder/patterns/en/medium/vuln create mode 100644 git_vuln_finder/patterns/en/medium/vuln.prefix create mode 100644 git_vuln_finder/patterns/en/medium/vuln.suffix create mode 100644 poetry.lock create mode 100644 pyproject.toml diff --git a/git_vuln_finder/__init__.py b/git_vuln_finder/__init__.py new file mode 100644 index 0000000..646d685 --- /dev/null +++ b/git_vuln_finder/__init__.py @@ -0,0 +1,6 @@ + +from git_vuln_finder.finder import build_pattern +from git_vuln_finder.finder import get_patterns +from git_vuln_finder.finder import find_vuln +from git_vuln_finder.finder import summary +from git_vuln_finder.finder import extract_cve diff --git a/git_vuln_finder/__pycache__/__init__.cpython-38.pyc b/git_vuln_finder/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e53df7ed0268c75da6f6a5ff6cd0a14e0c253bea GIT binary patch literal 342 zcmYk0OAf*y5Qd>|AB{Wj(1q;*G{zI?!nk1}sRfm!R70uJBY7pauDpUPXWEz)GXH%4 z05k03ct~K)H0HHsjDmPAuVDY66XFsEe~*kx|Z9&==8^a*4C*2xLsV{R4jtRi^*| literal 0 HcmV?d00001 diff --git a/git_vuln_finder/__pycache__/finder.cpython-38.pyc b/git_vuln_finder/__pycache__/finder.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7837370a66c8681f2d6c1069e58db3b59ba91165 GIT binary patch literal 3427 zcmZ`+&2JmI6({Fg}iJ@+Ui)Qw>!4!osMIAx8tI>hTh2U zc$6@^6EKIlUy)9Y2M3gS%>RmZLbk?gEW~JqU1AZdV-&FlTZLSm(Z{6Ke2taKUW+P^ zamjmfdSh$5pUCdXWSDjjl9X|=J&vX1BF%eL3s;YaUcdM7=Knw=NdXI9Su@J4nKiv> zX!}~pvkR-R=X61ClbIe_3qr}~B#kI~N`6OD>kCrQxxFCw$+PvD19@leEC_S18J*)_ z7TOc;pR5f%11)E$R`IG;Txhw=RowrxiUqs9sx$;CVMKW%Aov88;+%@FNIv6Wk@E4t0Idd1Zqg6V! za%M38PQ)d6>}+Rykd63uk28_b>qX6Ds%>KQ$wBqRJ9n`Mqp3MkKcCo%(s^>M$9o;G9HIXPHzE zOGK_*9ky8Bh|mM&re@>}I+5uMjfgf2^ zkM`pddBz}s2Ed*}1mDaqY%wS-@rkq;&;&^1FofcQO7{@q_GgO{X=&MuG;`ii2z#d> z1+a9NK-*vFv4^qq7|2|{K#%F_r?Kn}HV?8%%62wi&dhNFMjh?Vsoie3m4|~E;Vj*e z7HtK}%Xut%1Lco+p2vNz+_6Yfsl0qO8|UuNRa`x0>$a5CdP%W4`2@0Bl{&rwstCV?GK-1`axUb)l+N5`F7uE>_AFw+g#!76xqZ-K>aYN_f-3wYPyD6u^>~9Y}n|$?h<6I+2@kCVq`F1HB)5ha7otb{WISs?99|E;1oz;r7 zy5y`G&f1c`Wavvvy17gG)gJZk@%4PJ3zihtVROm93>jxht%I_c)D=)6Q0Z@l-ZwEG zx%$=$*C63S;@|S}e7y)3^w5`Ajy%J;@`eWlJl}`bsNwrJMQzEuZg`gP*v67~W0$0j z!n;UouN3})F0J3%n}#S`gUS9D65x#Ph;>BI8Qq?`Ti}1!px?ATkaC>wY;VWoM6)aL zZNbM`zWs8h|H}N))U*ka%}gZyB;A~(M``vvRl&Z9)82sRVhanXKm*nUX*I2titU6c zKc9?7F~G{rWh{|a@k9WVJh1%-!>Lz}wl%a9 z#yFEal}S7_@WVyCpP;5CaFZ`eN%0Ufb}DCeOJ{ZAtWJ=Lephd}gR9m!GsAa~&6wGP ze*Opz2{fXGx{9D}0NHfgv>hv?*YU2IN}_9t{;lEHFdSWNJiNMX0ueo;M{8EYD(BaQ z`TWu7r%Sd$K5;#P#hQI(*b!p1eWfl@-}xFczdI=)%h7fOh= zH@$xEvj^?pf6)H$*;DrNgYW-v);9g^SAs$v4sH>%sXivK>3vh~5vI=6?oC&cZwY14^t@_y7O^ literal 0 HcmV?d00001 diff --git a/git_vuln_finder/finder.py b/git_vuln_finder/finder.py new file mode 100644 index 0000000..184adc2 --- /dev/null +++ b/git_vuln_finder/finder.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Finding potential software vulnerabilities from git commit messages +# +# Software is free software released under the "GNU Affero General Public License v3.0" +# +# This software is part of cve-search.org +# +# Copyright (c) 2019 Alexandre Dulaunoy - a@foo.be + + +import os +import re +import git +import json +import sys +import typing +from langdetect import detect as langdetect + + +PATTERNS_PATH="./git_vuln_finder/patterns" + + +def build_pattern(pattern_file): + fp = open(pattern_file, "r") + rex = "" + try: + prefix_fp = open(pattern_file + ".prefix", "r") + rex += prefix_fp.read() + prefix_fp.close() + except: + pass + + for line in fp.readlines(): + rex += line.rstrip() + "|" + rex = rex[:-1] # We remove the extra '| + fp.close() + + try: + suffix_fp = open(pattern_file + ".suffix", "r") + rex += suffix_fp.read() + suffix_fp.close() + except: + pass + + return rex + + +def get_patterns(patterns_path=PATTERNS_PATH): + patterns = {} + for root, dirs, files in os.walk(patterns_path): + path = root.split(os.sep) + for f in files: + if f.endswith(".prefix") or f.endswith(".suffix"): + continue + npath = root[len(patterns_path):].split(os.sep) + try: + npath.remove('') + except ValueError: + pass + + lang = npath[0] + severity = npath[1] + pattern_category = f + + try: # FIXME: Is there a better way? + a = patterns[lang] + except KeyError: + patterns[lang] = {} + try: + a = patterns[lang][severity] + except KeyError: + patterns[lang][severity] = {} + try: + a = patterns[lang][severity][pattern_category] + except KeyError: + rex = build_pattern(root + os.sep + f) + patterns[lang][severity][pattern_category] = re.compile(rex) + + return patterns + + +def find_vuln(commit, pattern, versbose=False): + m = pattern.search(commit.message) + if m: + if versbose: + print("Match found: {}".format(m.group(0)), file=sys.stderr) + print(commit.message, file=sys.stderr) + print("---", file=sys.stderr) + ret = {} + ret['commit'] = commit + ret['match'] = m.groups() + return ret + else: + return None + + +def summary(commit, + branch, + pattern, + origin=None, + vuln_match=None, + tags_matching=False, + commit_state="under-review" +): + potential_vulnerabilities = {} + rcommit = commit + cve = extract_cve(rcommit.message) + if origin is not None: + origin = origin + if origin.find('github.com'): + origin_github_api = origin.split(':')[1] + (org_name, repo_name) = origin_github_api.split('/', 1) + if repo_name.find('.git$'): + repo_name = re.sub(r".git$","", repo_name) + origin_github_api = 'https://api.github.com/repos/{}/{}/commits/{}'.format(org_name, repo_name, rcommit.hexsha) + + else: + origin = 'git origin unknown' + # deduplication if similar commits on different branches + if rcommit.hexsha in potential_vulnerabilities: + potential_vulnerabilities[rcommit.hexsha]['branches'].append(branch) + else: + potential_vulnerabilities[rcommit.hexsha] = {} + potential_vulnerabilities[rcommit.hexsha]['message'] = rcommit.message + potential_vulnerabilities[rcommit.hexsha]['language'] = langdetect(rcommit.message) + potential_vulnerabilities[rcommit.hexsha]['commit-id'] = rcommit.hexsha + potential_vulnerabilities[rcommit.hexsha]['summary'] = rcommit.summary + potential_vulnerabilities[rcommit.hexsha]['stats'] = rcommit.stats.total + potential_vulnerabilities[rcommit.hexsha]['author'] = rcommit.author.name + potential_vulnerabilities[rcommit.hexsha]['author-email'] = rcommit.author.email + potential_vulnerabilities[rcommit.hexsha]['authored_date'] = rcommit.authored_date + potential_vulnerabilities[rcommit.hexsha]['committed_date'] = rcommit.committed_date + potential_vulnerabilities[rcommit.hexsha]['branches'] = [] + potential_vulnerabilities[rcommit.hexsha]['branches'].append(branch) + potential_vulnerabilities[rcommit.hexsha]['pattern-selected'] = pattern.pattern + potential_vulnerabilities[rcommit.hexsha]['pattern-matches'] = vuln_match + potential_vulnerabilities[rcommit.hexsha]['origin'] = origin + if origin_github_api: + potential_vulnerabilities[commit.hexsha]['origin-github-api'] = origin_github_api + potential_vulnerabilities[rcommit.hexsha]['tags'] = [] + if tags_matching: + if repo.commit(rcommit).hexsha in tagmap: + potential_vulnerabilities[rcommit.hexsha]['tags'] = tagmap[repo.commit(rcommit).hexsha] + if cve: potential_vulnerabilities[rcommit.hexsha]['cve'] = cve + if cve: + potential_vulnerabilities[rcommit.hexsha]['state'] = "cve-assigned" + else: + potential_vulnerabilities[rcommit.hexsha]['state'] = commit_state + + return rcommit.hexsha, potential_vulnerabilities + + +def extract_cve(commit): + cve_found = set() + cve_find = re.compile(r'CVE-[1-2]\d{1,4}-\d{1,7}', re.IGNORECASE) + m = cve_find.findall(commit) + if m: + for v in m: + cve_found.add(v) + return m + else: + return None diff --git a/git_vuln_finder/patterns/en/medium/c b/git_vuln_finder/patterns/en/medium/c new file mode 100644 index 0000000..42d4a36 --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/c @@ -0,0 +1,4 @@ +double[-| ]free +buffer overflow +double free +race[-| ]condition diff --git a/git_vuln_finder/patterns/en/medium/c.prefix b/git_vuln_finder/patterns/en/medium/c.prefix new file mode 100644 index 0000000..4a45df8 --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/c.prefix @@ -0,0 +1 @@ +(?i)( \ No newline at end of file diff --git a/git_vuln_finder/patterns/en/medium/c.suffix b/git_vuln_finder/patterns/en/medium/c.suffix new file mode 100644 index 0000000..e8a0f87 --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/c.suffix @@ -0,0 +1 @@ +) \ No newline at end of file diff --git a/git_vuln_finder/patterns/en/medium/crypto b/git_vuln_finder/patterns/en/medium/crypto new file mode 100644 index 0000000..b24e6c2 --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/crypto @@ -0,0 +1,55 @@ +assessment +lack of +bad +vulnerable +missing +unproper +unsuitable +breakable +broken +weak +incorrect +replace +assessment +pen([\s-]?)test +pentest +penetration([\s-]?)test +report +vulnerablity +replace +fix +issue +fixes +add +remove +check){s1,} + (crypto +cryptographic +cryptography +encipherement +encryption +ciphers +cipher +AES +DES +3DES +cipher +GPG +PGP +OpenSSL +SSH +wireguard +VPN +CBC +ECB +CTR +key[.|,|\s] +private([\s-]?)key +public([\s-]?)key size +length +strenght +generation +randomness +entropy +prng +rng diff --git a/git_vuln_finder/patterns/en/medium/crypto.prefix b/git_vuln_finder/patterns/en/medium/crypto.prefix new file mode 100644 index 0000000..f5b5f62 --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/crypto.prefix @@ -0,0 +1 @@ +.*( \ No newline at end of file diff --git a/git_vuln_finder/patterns/en/medium/crypto.suffix b/git_vuln_finder/patterns/en/medium/crypto.suffix new file mode 100644 index 0000000..12953cd --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/crypto.suffix @@ -0,0 +1 @@ +){1,} \ No newline at end of file diff --git a/git_vuln_finder/patterns/en/medium/vuln b/git_vuln_finder/patterns/en/medium/vuln new file mode 100644 index 0000000..db3ca53 --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/vuln @@ -0,0 +1,30 @@ +denial of service +\bXXE\b +remote code execution +\bopen redirect +OSVDB +\bvuln +\bCVE\b +\bXSS\b +\bReDoS\b +\bNVD\b +malicious +x−frame−options +attack +cross site +exploit +malicious +directory traversal +\bRCE\b +\bdos\b +\bXSRF \b +\bXSS\b +clickjack +session.fixation +hijack +\badvisory +\binsecure +security +\bcross−origin\b +unauthori[z|s]ed +infinite loop \ No newline at end of file diff --git a/git_vuln_finder/patterns/en/medium/vuln.prefix b/git_vuln_finder/patterns/en/medium/vuln.prefix new file mode 100644 index 0000000..4a45df8 --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/vuln.prefix @@ -0,0 +1 @@ +(?i)( \ No newline at end of file diff --git a/git_vuln_finder/patterns/en/medium/vuln.suffix b/git_vuln_finder/patterns/en/medium/vuln.suffix new file mode 100644 index 0000000..e8a0f87 --- /dev/null +++ b/git_vuln_finder/patterns/en/medium/vuln.suffix @@ -0,0 +1 @@ +) \ No newline at end of file diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..02cd4ef --- /dev/null +++ b/poetry.lock @@ -0,0 +1,290 @@ +[[package]] +category = "dev" +description = "Code coverage measurement for Python" +name = "coverage" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" +version = "5.0.1" + +[package.extras] +toml = ["toml"] + +[[package]] +category = "dev" +description = "Discover and load entry points from installed packages." +name = "entrypoints" +optional = false +python-versions = ">=2.7" +version = "0.3" + +[[package]] +category = "dev" +description = "the modular source code checker: pep8, pyflakes and co" +name = "flake8" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "3.7.9" + +[package.dependencies] +entrypoints = ">=0.3.0,<0.4.0" +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.5.0,<2.6.0" +pyflakes = ">=2.1.0,<2.2.0" + +[[package]] +category = "main" +description = "Git Object Database" +name = "gitdb2" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.0.6" + +[package.dependencies] +smmap2 = ">=2.0.0" + +[[package]] +category = "main" +description = "Python Git Library" +name = "gitpython" +optional = false +python-versions = ">=3.0, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "3.0.5" + +[package.dependencies] +gitdb2 = ">=2.0.0" + +[[package]] +category = "main" +description = "Language detection library ported from Google's language-detection." +name = "langdetect" +optional = false +python-versions = "*" +version = "1.0.7" + +[package.dependencies] +six = "*" + +[[package]] +category = "dev" +description = "McCabe checker, plugin for flake8" +name = "mccabe" +optional = false +python-versions = "*" +version = "0.6.1" + +[[package]] +category = "dev" +description = "Optional static typing for Python" +name = "mypy" +optional = false +python-versions = ">=3.5" +version = "0.750" + +[package.dependencies] +mypy-extensions = ">=0.4.0,<0.5.0" +typed-ast = ">=1.4.0,<1.5.0" +typing-extensions = ">=3.7.4" + +[package.extras] +dmypy = ["psutil (>=4.0)"] + +[[package]] +category = "dev" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +name = "mypy-extensions" +optional = false +python-versions = "*" +version = "0.4.3" + +[[package]] +category = "dev" +description = "unittest2 with plugins, the succesor to nose" +name = "nose2" +optional = false +python-versions = "*" +version = "0.9.1" + +[package.dependencies] +coverage = ">=4.4.1" +six = ">=1.7" + +[package.extras] +coverage_plugin = ["coverage (>=4.4.1)"] +doc = ["Sphinx (>=1.6.5)", "sphinx-rtd-theme", "mock"] + +[[package]] +category = "dev" +description = "Python style guide checker" +name = "pycodestyle" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.5.0" + +[[package]] +category = "dev" +description = "passive checker of Python programs" +name = "pyflakes" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.1.1" + +[[package]] +category = "main" +description = "Python 2 and 3 compatibility utilities" +name = "six" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*" +version = "1.13.0" + +[[package]] +category = "main" +description = "A pure Python implementation of a sliding window memory map manager" +name = "smmap2" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.0.5" + +[[package]] +category = "dev" +description = "a fork of Python 2 and 3 ast modules with type comment support" +name = "typed-ast" +optional = false +python-versions = "*" +version = "1.4.0" + +[[package]] +category = "dev" +description = "Backported and Experimental Type Hints for Python 3.5+" +name = "typing-extensions" +optional = false +python-versions = "*" +version = "3.7.4.1" + +[metadata] +content-hash = "4fd05852a9f3844298b1c0dbc4ab61ddbb77f4a42602c42982e19e531a7883d6" +python-versions = "^3.6" + +[metadata.files] +coverage = [ + {file = "coverage-5.0.1-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:c90bda74e16bcd03861b09b1d37c0a4158feda5d5a036bb2d6e58de6ff65793e"}, + {file = "coverage-5.0.1-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:bb3d29df5d07d5399d58a394d0ef50adf303ab4fbf66dfd25b9ef258effcb692"}, + {file = "coverage-5.0.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1ca43dbd739c0fc30b0a3637a003a0d2c7edc1dd618359d58cc1e211742f8bd1"}, + {file = "coverage-5.0.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:591506e088901bdc25620c37aec885e82cc896528f28c57e113751e3471fc314"}, + {file = "coverage-5.0.1-cp27-cp27m-win32.whl", hash = "sha256:a50b0888d8a021a3342d36a6086501e30de7d840ab68fca44913e97d14487dc1"}, + {file = "coverage-5.0.1-cp27-cp27m-win_amd64.whl", hash = "sha256:c792d3707a86c01c02607ae74364854220fb3e82735f631cd0a345dea6b4cee5"}, + {file = "coverage-5.0.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f425f50a6dd807cb9043d15a4fcfba3b5874a54d9587ccbb748899f70dc18c47"}, + {file = "coverage-5.0.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:25b8f60b5c7da71e64c18888f3067d5b6f1334b9681876b2fb41eea26de881ae"}, + {file = "coverage-5.0.1-cp35-cp35m-macosx_10_12_x86_64.whl", hash = "sha256:7362a7f829feda10c7265b553455de596b83d1623b3d436b6d3c51c688c57bf6"}, + {file = "coverage-5.0.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:fcd4459fe35a400b8f416bc57906862693c9f88b66dc925e7f2a933e77f6b18b"}, + {file = "coverage-5.0.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:40fbfd6b044c9db13aeec1daf5887d322c710d811f944011757526ef6e323fd9"}, + {file = "coverage-5.0.1-cp35-cp35m-win32.whl", hash = "sha256:7f2675750c50151f806070ec11258edf4c328340916c53bac0adbc465abd6b1e"}, + {file = "coverage-5.0.1-cp35-cp35m-win_amd64.whl", hash = "sha256:24bcfa86fd9ce86b73a8368383c39d919c497a06eebb888b6f0c12f13e920b1a"}, + {file = "coverage-5.0.1-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:eeafb646f374988c22c8e6da5ab9fb81367ecfe81c70c292623373d2a021b1a1"}, + {file = "coverage-5.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:2ca2cd5264e84b2cafc73f0045437f70c6378c0d7dbcddc9ee3fe192c1e29e5d"}, + {file = "coverage-5.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2cc707fc9aad2592fc686d63ef72dc0031fc98b6fb921d2f5395d9ab84fbc3ef"}, + {file = "coverage-5.0.1-cp36-cp36m-win32.whl", hash = "sha256:04b961862334687549eb91cd5178a6fbe977ad365bddc7c60f2227f2f9880cf4"}, + {file = "coverage-5.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:232f0b52a5b978288f0bbc282a6c03fe48cd19a04202df44309919c142b3bb9c"}, + {file = "coverage-5.0.1-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:cfce79ce41cc1a1dc7fc85bb41eeeb32d34a4cf39a645c717c0550287e30ff06"}, + {file = "coverage-5.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:46c9c6a1d1190c0b75ec7c0f339088309952b82ae8d67a79ff1319eb4e749b96"}, + {file = "coverage-5.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1cbb88b34187bdb841f2599770b7e6ff8e259dc3bb64fc7893acf44998acf5f8"}, + {file = "coverage-5.0.1-cp37-cp37m-win32.whl", hash = "sha256:ff3936dd5feaefb4f91c8c1f50a06c588b5dc69fba4f7d9c79a6617ad80bb7df"}, + {file = "coverage-5.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:65bead1ac8c8930cf92a1ccaedcce19a57298547d5d1db5c9d4d068a0675c38b"}, + {file = "coverage-5.0.1-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:348630edea485f4228233c2f310a598abf8afa5f8c716c02a9698089687b6085"}, + {file = "coverage-5.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:960d7f42277391e8b1c0b0ae427a214e1b31a1278de6b73f8807b20c2e913bba"}, + {file = "coverage-5.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0101888bd1592a20ccadae081ba10e8b204d20235d18d05c6f7d5e904a38fc10"}, + {file = "coverage-5.0.1-cp38-cp38m-win32.whl", hash = "sha256:c0fff2733f7c2950f58a4fd09b5db257b00c6fec57bf3f68c5bae004d804b407"}, + {file = "coverage-5.0.1-cp38-cp38m-win_amd64.whl", hash = "sha256:5f622f19abda4e934938e24f1d67599249abc201844933a6f01aaa8663094489"}, + {file = "coverage-5.0.1-cp39-cp39m-win32.whl", hash = "sha256:2714160a63da18aed9340c70ed514973971ee7e665e6b336917ff4cca81a25b1"}, + {file = "coverage-5.0.1-cp39-cp39m-win_amd64.whl", hash = "sha256:b7dbc5e8c39ea3ad3db22715f1b5401cd698a621218680c6daf42c2f9d36e205"}, + {file = "coverage-5.0.1.tar.gz", hash = "sha256:5ac71bba1e07eab403b082c4428f868c1c9e26a21041436b4905c4c3d4e49b08"}, +] +entrypoints = [ + {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, + {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, +] +flake8 = [ + {file = "flake8-3.7.9-py2.py3-none-any.whl", hash = "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca"}, + {file = "flake8-3.7.9.tar.gz", hash = "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb"}, +] +gitdb2 = [ + {file = "gitdb2-2.0.6-py2.py3-none-any.whl", hash = "sha256:96bbb507d765a7f51eb802554a9cfe194a174582f772e0d89f4e87288c288b7b"}, + {file = "gitdb2-2.0.6.tar.gz", hash = "sha256:1b6df1433567a51a4a9c1a5a0de977aa351a405cc56d7d35f3388bad1f630350"}, +] +gitpython = [ + {file = "GitPython-3.0.5-py3-none-any.whl", hash = "sha256:c155c6a2653593ccb300462f6ef533583a913e17857cfef8fc617c246b6dc245"}, + {file = "GitPython-3.0.5.tar.gz", hash = "sha256:9c2398ffc3dcb3c40b27324b316f08a4f93ad646d5a6328cafbb871aa79f5e42"}, +] +langdetect = [ + {file = "langdetect-1.0.7.zip", hash = "sha256:91a170d5f0ade380db809b3ba67f08e95fe6c6c8641f96d67a51ff7e98a9bf30"}, +] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] +mypy = [ + {file = "mypy-0.750-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:de9ec8dba773b78c49e7bec9a35c9b6fc5235682ad1fc2105752ae7c22f4b931"}, + {file = "mypy-0.750-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:3294821b5840d51a3cd7a2bb63b40fc3f901f6a3cfb3c6046570749c4c7ef279"}, + {file = "mypy-0.750-cp35-cp35m-win_amd64.whl", hash = "sha256:6992133c95a2847d309b4b0c899d7054adc60481df6f6b52bb7dee3d5fd157f7"}, + {file = "mypy-0.750-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:41696a7d912ce16fdc7c141d87e8db5144d4be664a0c699a2b417d393994b0c2"}, + {file = "mypy-0.750-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:c87ac7233c629f305602f563db07f5221950fe34fe30af072ac838fa85395f78"}, + {file = "mypy-0.750-cp36-cp36m-win_amd64.whl", hash = "sha256:83fa87f556e60782c0fc3df1b37b7b4a840314ba1ac27f3e1a1e10cb37c89c17"}, + {file = "mypy-0.750-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:30e123b24931f02c5d99307406658ac8f9cd6746f0d45a3dcac2fe5fbdd60939"}, + {file = "mypy-0.750-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:02d9bdd3398b636723ecb6c5cfe9773025a9ab7f34612c1cde5c7f2292e2d768"}, + {file = "mypy-0.750-cp37-cp37m-win_amd64.whl", hash = "sha256:088f758a50af31cf8b42688118077292370c90c89232c783ba7979f39ea16646"}, + {file = "mypy-0.750-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4f42675fa278f3913340bb8c3371d191319704437758d7c4a8440346c293ecb2"}, + {file = "mypy-0.750-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:f385a0accf353ca1bca4bbf473b9d83ed18d923fdb809d3a70a385da23e25b6a"}, + {file = "mypy-0.750-cp38-cp38-win_amd64.whl", hash = "sha256:54d205ccce6ed930a8a2ccf48404896d456e8b87812e491cb907a355b1a9c640"}, + {file = "mypy-0.750-py3-none-any.whl", hash = "sha256:28e9fbc96d13397a7ddb7fad7b14f373f91b5cff538e0772e77c270468df083c"}, + {file = "mypy-0.750.tar.gz", hash = "sha256:6ecbd0e8e371333027abca0922b0c2c632a5b4739a0c61ffbd0733391e39144c"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +nose2 = [ + {file = "nose2-0.9.1-py2.py3-none-any.whl", hash = "sha256:31d8beb00aed3ccc6efb1742bb90227d883e471715188249f594310676e0ef0e"}, + {file = "nose2-0.9.1.tar.gz", hash = "sha256:0ede156fd7974fa40893edeca0b709f402c0ccacd7b81b22e76f73c116d1b999"}, +] +pycodestyle = [ + {file = "pycodestyle-2.5.0-py2.py3-none-any.whl", hash = "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56"}, + {file = "pycodestyle-2.5.0.tar.gz", hash = "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"}, +] +pyflakes = [ + {file = "pyflakes-2.1.1-py2.py3-none-any.whl", hash = "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0"}, + {file = "pyflakes-2.1.1.tar.gz", hash = "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"}, +] +six = [ + {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, + {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, +] +smmap2 = [ + {file = "smmap2-2.0.5-py2.py3-none-any.whl", hash = "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde"}, + {file = "smmap2-2.0.5.tar.gz", hash = "sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a"}, +] +typed-ast = [ + {file = "typed_ast-1.4.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e"}, + {file = "typed_ast-1.4.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b"}, + {file = "typed_ast-1.4.0-cp35-cp35m-win32.whl", hash = "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4"}, + {file = "typed_ast-1.4.0-cp35-cp35m-win_amd64.whl", hash = "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"}, + {file = "typed_ast-1.4.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631"}, + {file = "typed_ast-1.4.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233"}, + {file = "typed_ast-1.4.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1"}, + {file = "typed_ast-1.4.0-cp36-cp36m-win32.whl", hash = "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a"}, + {file = "typed_ast-1.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c"}, + {file = "typed_ast-1.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a"}, + {file = "typed_ast-1.4.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e"}, + {file = "typed_ast-1.4.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d"}, + {file = "typed_ast-1.4.0-cp37-cp37m-win32.whl", hash = "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36"}, + {file = "typed_ast-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0"}, + {file = "typed_ast-1.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66"}, + {file = "typed_ast-1.4.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2"}, + {file = "typed_ast-1.4.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47"}, + {file = "typed_ast-1.4.0-cp38-cp38-win32.whl", hash = "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161"}, + {file = "typed_ast-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e"}, + {file = "typed_ast-1.4.0.tar.gz", hash = "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34"}, +] +typing-extensions = [ + {file = "typing_extensions-3.7.4.1-py2-none-any.whl", hash = "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d"}, + {file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"}, + {file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d68b00e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,59 @@ +[tool.poetry] +name = "git-vuln-finder" +version = "1.0.0" +description = "Finding potential software vulnerabilities from git commit messages." +authors = [ + "Alexandre Dulaunoy " +] +license = "GPL-3.0-or-later" + +readme = "README.md" + +homepage = "https://github.com/cve-search/git-vuln-finder" +repository = "https://github.com/cve-search/git-vuln-finder" +documentation = "" + +keywords = [ + "git", + "cve", + "scanner", + "cve-search", + "cve-scanning", + "software-vulnerability", + "software-vulnerabilities" +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Security", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)" +] + +include = [ + "AUTHORS", + "COPYING", + "bin/*" +] + +[tool.poetry.scripts] +finder = "bin.finder:main" + +[tool.poetry.dependencies] +python = "^3.6" +langdetect = "^1.0.7" +gitpython = "^3.0.5" + +[tool.poetry.dev-dependencies] +mypy = "^0.750" +flake8 = "^3.7.9" +nose2 = "^0.9.1" + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" From dfa15c119dd2643e53f71afd69a81a3727661e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Fri, 3 Jan 2020 17:35:04 +0100 Subject: [PATCH 03/42] Renamed command line. --- README.md | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e5cbd90..64d2539 100644 --- a/README.md +++ b/README.md @@ -28,14 +28,14 @@ $ poetry shell ~~~bash $ pipx install git-vuln-finder -$ finder --help +$ git-vuln-finder --help ~~~ # Usage ~~~bash -usage: finder.py [-h] [-v] [-r R] [-o O] [-s S] [-p P] [-c] [-t] +usage: git-vuln-finder [-h] [-v] [-r R] [-o O] [-s S] [-p P] [-c] [-t] Finding potential software vulnerabilities from git commit messages. @@ -66,7 +66,7 @@ git-vuln-finder comes with 3 default patterns which can be selected to find the ## A sample partial output from Curl git repository ~~~bash -python3 finder.py -r /home/adulau/git/curl | jq . +$ git-vuln-finder -r /home/adulau/git/curl | jq . ... "6df916d751e72fc9a1febc07bb59c4ddd886c043": { "message": "loadlibrary: Only load system DLLs from the system directory\n\nInspiration provided by: Daniel Stenberg and Ray Satiro\n\nBug: https://curl.haxx.se/docs/adv_20160530.html\n\nRef: Windows DLL hijacking with curl, CVE-2016-4802\n", diff --git a/pyproject.toml b/pyproject.toml index d68b00e..95f779a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ include = [ ] [tool.poetry.scripts] -finder = "bin.finder:main" +git-vuln-finder = "bin.finder:main" [tool.poetry.dependencies] python = "^3.6" From fe4a656412368a571b5a93a6ec92b2381851720f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Fri, 3 Jan 2020 17:39:56 +0100 Subject: [PATCH 04/42] Updated README. --- README.md | 6 ++---- .../__pycache__/__init__.cpython-38.pyc | Bin 342 -> 342 bytes .../__pycache__/finder.cpython-38.pyc | Bin 3427 -> 3427 bytes 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 64d2539..990133e 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,7 @@ added automatically in the output. # Requirements -- Python 3.6 -- GitPython -- langdetect +- jq (sudo apt install jq) # Installation @@ -66,7 +64,7 @@ git-vuln-finder comes with 3 default patterns which can be selected to find the ## A sample partial output from Curl git repository ~~~bash -$ git-vuln-finder -r /home/adulau/git/curl | jq . +$ git-vuln-finder -r ~/git/curl | jq . ... "6df916d751e72fc9a1febc07bb59c4ddd886c043": { "message": "loadlibrary: Only load system DLLs from the system directory\n\nInspiration provided by: Daniel Stenberg and Ray Satiro\n\nBug: https://curl.haxx.se/docs/adv_20160530.html\n\nRef: Windows DLL hijacking with curl, CVE-2016-4802\n", diff --git a/git_vuln_finder/__pycache__/__init__.cpython-38.pyc b/git_vuln_finder/__pycache__/__init__.cpython-38.pyc index e53df7ed0268c75da6f6a5ff6cd0a14e0c253bea..6d105f5f47f121308cd25c57d7d652e6b67d6b25 100644 GIT binary patch delta 19 Zcmcb{bd8BCl$V!_0SG4MY~=D`1OP5u1UUcz delta 19 Zcmcb{bd8BCl$V!_0SK)9H*)zf0st$<1I7RV diff --git a/git_vuln_finder/__pycache__/finder.cpython-38.pyc b/git_vuln_finder/__pycache__/finder.cpython-38.pyc index 7837370a66c8681f2d6c1069e58db3b59ba91165..be0a36cf32dcef58ba7f807104e399bf2f83b580 100644 GIT binary patch delta 19 ZcmaDX^;n84l$V!_0SG4MY~+gM1pqUJ1e^c> delta 19 ZcmaDX^;n84l$V!_0SGF>H*!Vt0su0G1a1HT From 2fa7b4583e7435c51e3c40bc6e00c03b19c0bd72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Fri, 3 Jan 2020 17:51:55 +0100 Subject: [PATCH 05/42] improved cli. removed useless import --- bin/finder.py | 161 +++++++++--------- .../__pycache__/finder.cpython-38.pyc | Bin 3427 -> 3427 bytes git_vuln_finder/finder.py | 2 +- 3 files changed, 82 insertions(+), 81 deletions(-) diff --git a/bin/finder.py b/bin/finder.py index 9e26600..7ac005e 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -7,7 +7,7 @@ # # This software is part of cve-search.org # -# Copyright (c) 2019 Alexandre Dulaunoy - a@foo.be +# Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be import git @@ -17,102 +17,103 @@ import argparse import typing from git_vuln_finder import ( - build_pattern, get_patterns, find_vuln, - summary, - extract_cve + summary ) -PATTERNS_PATH="./git_vuln_finder/patterns" - -parser = argparse.ArgumentParser(description = "Finding potential software vulnerabilities from git commit messages.", epilog = "More info: https://github.com/cve-search/git-vuln-finder") -parser.add_argument("-v", help="increase output verbosity", action="store_true") -parser.add_argument("-r", type=str, help="git repository to analyse") -parser.add_argument("-o", type=str, help="Output format: [json]", default="json") -parser.add_argument("-s", type=str, help="State of the commit found", default="under-review") -parser.add_argument("-p", type=str, help="Matching pattern to use: [vulnpatterns, cryptopatterns, cpatterns] - the pattern 'all' is used to match all the patterns at once.", default="vulnpatterns") -parser.add_argument("-c", help="output only a list of the CVE pattern found in commit messages (disable by default)", action="store_true") -parser.add_argument("-t", help="Include tags matching a specific commit", action="store_true") -args = parser.parse_args() - - -patterns = get_patterns() -vulnpatterns = patterns["en"]["medium"]["vuln"] -cryptopatterns = patterns["en"]["medium"]["crypto"] -cpatterns = patterns["en"]["medium"]["c"] - -if args.p == "vulnpatterns": - defaultpattern = vulnpatterns -elif args.p == "cryptopatterns": - defaultpattern = cryptopatterns -elif args.p == "cpatterns": - defaultpattern = cpatterns -elif args.p == "all": - defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] -else: - parser.print_usage() - parser.exit() - -if not args.r: - parser.print_usage() - parser.exit() -else: - repo = git.Repo(args.r) - - -found = 0 -all_potential_vulnerabilities = {} -cve_found = set() def main(): - pass + """Point of entry for the script. + """ + # Parsing arguments + parser = argparse.ArgumentParser(description = "Finding potential software vulnerabilities from git commit messages.", epilog = "More info: https://github.com/cve-search/git-vuln-finder") + parser.add_argument("-v", help="increase output verbosity", action="store_true") + parser.add_argument("-r", type=str, help="git repository to analyse") + parser.add_argument("-o", type=str, help="Output format: [json]", default="json") + parser.add_argument("-s", type=str, help="State of the commit found", default="under-review") + parser.add_argument("-p", type=str, help="Matching pattern to use: [vulnpatterns, cryptopatterns, cpatterns] - the pattern 'all' is used to match all the patterns at once.", default="vulnpatterns") + parser.add_argument("-c", help="output only a list of the CVE pattern found in commit messages (disable by default)", action="store_true") + parser.add_argument("-t", help="Include tags matching a specific commit", action="store_true") + args = parser.parse_args() -repo_heads = repo.heads -repo_heads_names = [h.name for h in repo_heads] -print(repo_heads_names, file=sys.stderr) -origin = repo.remotes.origin.url -if args.t: - tagmap = {} - for t in repo.tags: - tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) + patterns = get_patterns() + vulnpatterns = patterns["en"]["medium"]["vuln"] + cryptopatterns = patterns["en"]["medium"]["crypto"] + cpatterns = patterns["en"]["medium"]["c"] -for branch in repo_heads_names: - commits = list(repo.iter_commits(branch)) - defaultpattern - for commit in commits: - if isinstance(defaultpattern, typing.Pattern): - ret = find_vuln(commit, pattern=defaultpattern, versbose=args.v) - if ret: - rcommit = ret['commit'] - _, potential_vulnerabilities = summary(rcommit, - branch, - defaultpattern, - origin=origin, - vuln_match=ret['match'], - tags_matching=args.t, - commit_state=args.s) - all_potential_vulnerabilities.update(potential_vulnerabilities) - found += 1 - elif isinstance(defaultpattern, list): - for p in defaultpattern: - ret = find_vuln(commit, pattern=p, versbose=args.v) + + if args.p == "vulnpatterns": + defaultpattern = vulnpatterns + elif args.p == "cryptopatterns": + defaultpattern = cryptopatterns + elif args.p == "cpatterns": + defaultpattern = cpatterns + elif args.p == "all": + defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] + else: + parser.print_usage() + parser.exit() + + if not args.r: + parser.print_usage() + parser.exit() + else: + repo = git.Repo(args.r) + + + # Initialization of the variables for the results + found = 0 + all_potential_vulnerabilities = {} + cve_found = set() + + + repo_heads = repo.heads + repo_heads_names = [h.name for h in repo_heads] + print(repo_heads_names, file=sys.stderr) + origin = repo.remotes.origin.url + if args.t: + tagmap = {} + for t in repo.tags: + tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) + + for branch in repo_heads_names: + commits = list(repo.iter_commits(branch)) + defaultpattern + for commit in commits: + if isinstance(defaultpattern, typing.Pattern): + ret = find_vuln(commit, pattern=defaultpattern, versbose=args.v) if ret: rcommit = ret['commit'] _, potential_vulnerabilities = summary(rcommit, branch, - p, + defaultpattern, origin=origin, vuln_match=ret['match'], tags_matching=args.t, commit_state=args.s) all_potential_vulnerabilities.update(potential_vulnerabilities) found += 1 -if not args.c: - print(json.dumps(all_potential_vulnerabilities)) -elif args.c: - print(json.dumps(list(cve_found))) + elif isinstance(defaultpattern, list): + for p in defaultpattern: + ret = find_vuln(commit, pattern=p, versbose=args.v) + if ret: + rcommit = ret['commit'] + _, potential_vulnerabilities = summary(rcommit, + branch, + p, + origin=origin, + vuln_match=ret['match'], + tags_matching=args.t, + commit_state=args.s) + all_potential_vulnerabilities.update(potential_vulnerabilities) + found += 1 -print("{} CVE referenced found in commit(s)".format(len(list(cve_found))), file=sys.stderr) -print("Total potential vulnerability found in {} commit(s)".format(found), file=sys.stderr) + if not args.c: + print(json.dumps(all_potential_vulnerabilities)) + elif args.c: + print(json.dumps(list(cve_found))) + + print("{} CVE referenced found in commit(s)".format(len(list(cve_found))), file=sys.stderr) + print("Total potential vulnerability found in {} commit(s)".format(found), file=sys.stderr) diff --git a/git_vuln_finder/__pycache__/finder.cpython-38.pyc b/git_vuln_finder/__pycache__/finder.cpython-38.pyc index be0a36cf32dcef58ba7f807104e399bf2f83b580..75166b04a83ec699f8a0b7eed039b4307f8ec0ce 100644 GIT binary patch delta 28 icmaDX^;n8Gl$V!_0SHR-`QvVG Date: Sat, 4 Jan 2020 00:07:17 +0100 Subject: [PATCH 06/42] OK, it's for GitHub pages. --- _config.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 _config.yml diff --git a/_config.yml b/_config.yml new file mode 100644 index 0000000..fff4ab9 --- /dev/null +++ b/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-minimal From 19d4e874301f750260c71cab3bc6e949911a33b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Sat, 4 Jan 2020 00:08:30 +0100 Subject: [PATCH 07/42] added .gitignore --- .gitignore | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..24b934d --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +# use glob syntax +syntax: glob + +*.elc +*.pyc +*~ +*.db + +# Virtualenv +venv +build + +# setuptools +build/* +git-vuln-finder.egg-info/* +dist/* + +# tests +.coverage +.mypy_cache/ +.cache/ + +# sphinx +docs/_build + +# Emacs +eproject.cfg + +# Temporary files (vim backups) +*.swp + +.idea/ + +# Log files: +*.log + +# Vagrant: +.vagrant/ From c9af36b8694b2552a15cb3600ef542a7b4ebf703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Sat, 4 Jan 2020 00:09:22 +0100 Subject: [PATCH 08/42] updated gitignore --- .gitignore | 2 +- README.md | 1 + .../__pycache__/finder.cpython-38.pyc | Bin 3427 -> 0 bytes git_vuln_finder/finder.py | 3 --- 4 files changed, 2 insertions(+), 4 deletions(-) delete mode 100644 git_vuln_finder/__pycache__/finder.cpython-38.pyc diff --git a/.gitignore b/.gitignore index 24b934d..119966c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ build # setuptools build/* -git-vuln-finder.egg-info/* +git_vuln_finder.egg-info/ dist/* # tests diff --git a/README.md b/README.md index 990133e..64c38fc 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ $ poetry install git-vuln-finder $ poetry shell ~~~ + ## Use as a command line tool ~~~bash diff --git a/git_vuln_finder/__pycache__/finder.cpython-38.pyc b/git_vuln_finder/__pycache__/finder.cpython-38.pyc deleted file mode 100644 index 75166b04a83ec699f8a0b7eed039b4307f8ec0ce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3427 zcmZ`+OK%&=5$>KhM?*@c<%gVLBLsl~I-!(ZB(Nh0$vS>y^H}Y|b~YQvUSh;==rxcE3AKE#_#G>kAhB%QLS9cpN+W0~IW*rsMtHmXeD!?@=vWJsx`f-ou-JjYg6J7QC`%lvy)tdehMM zwUTESR$6h%(-TCj(=5X zPq=@!HuMa%oS|C9t5$KLP~IVC0hnFj}s1 zvN2DUBY4b|+Z$#%S2axxla%Mm6S)-0xaD=C%0F<|kSpt8tZczwbZX_yUC@qJ>D0=Z z!T1Lem*BCpo$Wz3;@dsWMAFl4*Kb?9zFoQg-J>YkTjNtz-=8Ey)~(jw(5G+{4WS`* zs6`tz!q20g711U}fpw85E<^IIX{?FF5=g=4alh1)HP{5}uaN>__lnLU8O~RVXhAP(>p72a zeUA6IQRqbT_Jj-fia<9y?}bjL*Scv%%F1%(JdcM*%FV~aL@GPyV-@g}<fCni#35&*gbo z<{yBQ-$6qVl~)mmAzepwUN;fiKy(H)pbla)Lc}(}T|?VMzm8FpZd#%R-G@C}PfVmq{VUh%U-0J^M*p$I|V6# zrMm>${z8vEjGf0o=JEx4OjkdSWpA)~kWEsyv-xsnjuSBIXm3vKcDt=S9K;A`>5jB$ zD^Om}W6>KZf5h`V?sMgiMUqP89b^JB%I%A60`$mF0WY}(^a|w}Ufy!W_hG5>t5Z+` zjwRpMm_$b!Z{)kjP}5LhVEhRb^Q&kGyzJn2!+6<&=RsTIC-`y}!459kf}bS(dC%9< z)uWL?yWAkddzL{4zd&9B;DbYo45$?-PDeE{E^K`ObGRPaFEO&U^O=oY3X2ewgN(PH zTC|AY;uwm%J4tgZZkNB14Bux^C!|!)VEA7*72Vs8#8#R-Dx( zXU%Zdmh>e`fC_<1e=qdDiSfwQ zw^q0Y2^SLomY3)2MX;cUzPxhe8P1hAJRsotKD0&+-@hqpOWt+EvxLVsmb@FgByAMl zMOu5M@DFrp{np+zMA;fl_P3A#XLLubBYMv0_SD@1|9b}grtN{0<9ug(J02&RU5RfC zKF;#(moxoW=8vYPO^9q}BIzgT<|I8zv*)P__C=ib20RyASU?3DuqH^WX{}UjCrtVI zWHgEaR&Fk1iL{C*a*&B>RQ7E?ij$$LmlS7R25?i2QeRf|^c?!yIUnMR#MyMUO2f8E zZG9Uk_cpy$_U+Q5HjGn_jQhE=dnbGvp=rl?p7c|WJB8j7Z(UU@{Dl;8Pj>OSv;`=V zGI0%NiTAW^c0Sn`Khz^{z+dEp7>0~-J7Fr+R+m>d@iQ&|p_X!GCgY*FtH(|nj{r5s zQt@*wqBB|i*nHFOE|JBxkgOf*Rr9x5*mY5UE9<(<0S2?sB~D@7^F7F zndGTV;-P^bF5>+JH7$Xgd{Ih@hnTTbIjdVbs|#m!f=u+gdcz%DwZ@qlzJqMW%pUaf zM`%c(5iQhJ1Z@MzrrW0NSRuWRcg<81T}$+D4Znur=xXEP)ol}q=n*|yvl>=8zb?$@ zk48UTvJC=i4@&Y%-;QtzWoU5|!i5*NA{36Q1nm?yuCWc25<15nNn_OU&BDG=Lae>% z^?RQ^X#e5;_6N_NvX}4w;77Bz>2J?kp05AW9KHAs)>Gjxf3f$;qX+lyK7OEl9UJj* zsO*@r^SV$&=yXjn>I7z}vxXnJGh5 vFL7a0=Kj#svjw}Oj7^68r@H3kn`rd);QSZajvZ`7-wm7Qe+Ej Date: Sat, 4 Jan 2020 00:14:44 +0100 Subject: [PATCH 09/42] Updated README. --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 64c38fc..5100ed8 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ added automatically in the output. # Requirements -- jq (sudo apt install jq) +- jq (``sudo apt install jq``) # Installation @@ -22,6 +22,8 @@ $ poetry install git-vuln-finder $ poetry shell ~~~ +You can also use ``pip``. + ## Use as a command line tool @@ -30,6 +32,9 @@ $ pipx install git-vuln-finder $ git-vuln-finder --help ~~~ +``pipx`` installs scripts (system wide available) provided by Python packages into +separate virtualenvs to shield them from your system and each other. + # Usage @@ -170,7 +175,7 @@ ploit|malicious|directory traversal |\bRCE\b|\bdos\b|\bXSRF \b|\bXSS\b|clickjack This software is free software and licensed under the AGPL version 3. -Copyright (c) 2019 Alexandre Dulaunoy - https://github.com/adulau/ +Copyright (c) 2019-2020 Alexandre Dulaunoy - https://github.com/adulau/ # Acknowledgment From caed1e9031230701146ec065f61b8241305771e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Sat, 4 Jan 2020 00:17:21 +0100 Subject: [PATCH 10/42] deleted useless file --- .../__pycache__/__init__.cpython-38.pyc | Bin 342 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 git_vuln_finder/__pycache__/__init__.cpython-38.pyc diff --git a/git_vuln_finder/__pycache__/__init__.cpython-38.pyc b/git_vuln_finder/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 6d105f5f47f121308cd25c57d7d652e6b67d6b25..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 342 zcmYk0OAf*y5Qd>oUyVEO(1rE_#&`lZ#tjQeEvO{WHk2AYhR5(qZe4i=SI)FC5iyo*%E+vc!m1*JP8f3>I`7+{|L>l0u1YBJ VMSXh?mZlP|mMfgq3_vEd-(M97RvG{R From ba558d34e41764e3f66948f3d83a491f536d7f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Sun, 5 Jan 2020 13:08:57 +0100 Subject: [PATCH 11/42] typo --- bin/finder.py | 4 ++-- git_vuln_finder/finder.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/finder.py b/bin/finder.py index 7ac005e..d710c56 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -83,7 +83,7 @@ def main(): defaultpattern for commit in commits: if isinstance(defaultpattern, typing.Pattern): - ret = find_vuln(commit, pattern=defaultpattern, versbose=args.v) + ret = find_vuln(commit, pattern=defaultpattern, verbose=args.v) if ret: rcommit = ret['commit'] _, potential_vulnerabilities = summary(rcommit, @@ -97,7 +97,7 @@ def main(): found += 1 elif isinstance(defaultpattern, list): for p in defaultpattern: - ret = find_vuln(commit, pattern=p, versbose=args.v) + ret = find_vuln(commit, pattern=p, verbose=args.v) if ret: rcommit = ret['commit'] _, potential_vulnerabilities = summary(rcommit, diff --git a/git_vuln_finder/finder.py b/git_vuln_finder/finder.py index 16ecfb4..3c9be83 100644 --- a/git_vuln_finder/finder.py +++ b/git_vuln_finder/finder.py @@ -78,10 +78,10 @@ def get_patterns(patterns_path=PATTERNS_PATH): return patterns -def find_vuln(commit, pattern, versbose=False): +def find_vuln(commit, pattern, verbose=False): m = pattern.search(commit.message) if m: - if versbose: + if verbose: print("Match found: {}".format(m.group(0)), file=sys.stderr) print(commit.message, file=sys.stderr) print("---", file=sys.stderr) From a597d1e6fc03dd902c1f2602f05eb1d98860c87a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 07:44:35 +0100 Subject: [PATCH 12/42] fixed problem of definition of tagmap --- bin/finder.py | 10 ++- git_vuln_finder/__init__.py | 10 +-- git_vuln_finder/pattern.py | 76 +++++++++++++++++++ .../{finder.py => vulnerability.py} | 67 +--------------- 4 files changed, 91 insertions(+), 72 deletions(-) create mode 100644 git_vuln_finder/pattern.py rename git_vuln_finder/{finder.py => vulnerability.py} (69%) diff --git a/bin/finder.py b/bin/finder.py index d710c56..bbefd76 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -73,8 +73,8 @@ def main(): repo_heads_names = [h.name for h in repo_heads] print(repo_heads_names, file=sys.stderr) origin = repo.remotes.origin.url + tagmap = {} if args.t: - tagmap = {} for t in repo.tags: tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) @@ -86,8 +86,10 @@ def main(): ret = find_vuln(commit, pattern=defaultpattern, verbose=args.v) if ret: rcommit = ret['commit'] - _, potential_vulnerabilities = summary(rcommit, + _, potential_vulnerabilities = summary(repo, + rcommit, branch, + tagmap, defaultpattern, origin=origin, vuln_match=ret['match'], @@ -100,8 +102,10 @@ def main(): ret = find_vuln(commit, pattern=p, verbose=args.v) if ret: rcommit = ret['commit'] - _, potential_vulnerabilities = summary(rcommit, + _, potential_vulnerabilities = summary(repo, + rcommit, branch, + tagmap, p, origin=origin, vuln_match=ret['match'], diff --git a/git_vuln_finder/__init__.py b/git_vuln_finder/__init__.py index 646d685..8b18bb6 100644 --- a/git_vuln_finder/__init__.py +++ b/git_vuln_finder/__init__.py @@ -1,6 +1,6 @@ -from git_vuln_finder.finder import build_pattern -from git_vuln_finder.finder import get_patterns -from git_vuln_finder.finder import find_vuln -from git_vuln_finder.finder import summary -from git_vuln_finder.finder import extract_cve +from git_vuln_finder.pattern import build_pattern +from git_vuln_finder.pattern import get_patterns +from git_vuln_finder.vulnerability import find_vuln +from git_vuln_finder.vulnerability import summary +from git_vuln_finder.vulnerability import extract_cve diff --git a/git_vuln_finder/pattern.py b/git_vuln_finder/pattern.py new file mode 100644 index 0000000..e853249 --- /dev/null +++ b/git_vuln_finder/pattern.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Finding potential software vulnerabilities from git commit messages +# +# Software is free software released under the "GNU Affero General Public License v3.0" +# +# This software is part of cve-search.org +# +# Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be + + +import os +import re + + +PATTERNS_PATH="./git_vuln_finder/patterns" + + +def build_pattern(pattern_file): + fp = open(pattern_file, "r") + rex = "" + try: + prefix_fp = open(pattern_file + ".prefix", "r") + rex += prefix_fp.read() + prefix_fp.close() + except: + pass + + for line in fp.readlines(): + rex += line.rstrip() + "|" + rex = rex[:-1] # We remove the extra '| + fp.close() + + try: + suffix_fp = open(pattern_file + ".suffix", "r") + rex += suffix_fp.read() + suffix_fp.close() + except: + pass + + return rex + + +def get_patterns(patterns_path=PATTERNS_PATH): + patterns = {} + for root, dirs, files in os.walk(patterns_path): + path = root.split(os.sep) + for f in files: + if f.endswith(".prefix") or f.endswith(".suffix"): + continue + npath = root[len(patterns_path):].split(os.sep) + try: + npath.remove('') + except ValueError: + pass + + lang = npath[0] + severity = npath[1] + pattern_category = f + + try: # FIXME: Is there a better way? + a = patterns[lang] + except KeyError: + patterns[lang] = {} + try: + a = patterns[lang][severity] + except KeyError: + patterns[lang][severity] = {} + try: + a = patterns[lang][severity][pattern_category] + except KeyError: + rex = build_pattern(root + os.sep + f) + patterns[lang][severity][pattern_category] = re.compile(rex) + + return patterns diff --git a/git_vuln_finder/finder.py b/git_vuln_finder/vulnerability.py similarity index 69% rename from git_vuln_finder/finder.py rename to git_vuln_finder/vulnerability.py index 3c9be83..bf2d8a7 100644 --- a/git_vuln_finder/finder.py +++ b/git_vuln_finder/vulnerability.py @@ -10,74 +10,11 @@ # Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be -import os import re import sys from langdetect import detect as langdetect -PATTERNS_PATH="./git_vuln_finder/patterns" - - -def build_pattern(pattern_file): - fp = open(pattern_file, "r") - rex = "" - try: - prefix_fp = open(pattern_file + ".prefix", "r") - rex += prefix_fp.read() - prefix_fp.close() - except: - pass - - for line in fp.readlines(): - rex += line.rstrip() + "|" - rex = rex[:-1] # We remove the extra '| - fp.close() - - try: - suffix_fp = open(pattern_file + ".suffix", "r") - rex += suffix_fp.read() - suffix_fp.close() - except: - pass - - return rex - - -def get_patterns(patterns_path=PATTERNS_PATH): - patterns = {} - for root, dirs, files in os.walk(patterns_path): - path = root.split(os.sep) - for f in files: - if f.endswith(".prefix") or f.endswith(".suffix"): - continue - npath = root[len(patterns_path):].split(os.sep) - try: - npath.remove('') - except ValueError: - pass - - lang = npath[0] - severity = npath[1] - pattern_category = f - - try: # FIXME: Is there a better way? - a = patterns[lang] - except KeyError: - patterns[lang] = {} - try: - a = patterns[lang][severity] - except KeyError: - patterns[lang][severity] = {} - try: - a = patterns[lang][severity][pattern_category] - except KeyError: - rex = build_pattern(root + os.sep + f) - patterns[lang][severity][pattern_category] = re.compile(rex) - - return patterns - - def find_vuln(commit, pattern, verbose=False): m = pattern.search(commit.message) if m: @@ -93,8 +30,10 @@ def find_vuln(commit, pattern, verbose=False): return None -def summary(commit, +def summary(repo, + commit, branch, + tagmap, pattern, origin=None, vuln_match=None, From 335bdff4cba075e922e1ceb51d18a30446ef9f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 07:50:21 +0100 Subject: [PATCH 13/42] reformat a tiny with black --- bin/finder.py | 90 ++++++++++++++++++------------ git_vuln_finder/__init__.py | 1 - git_vuln_finder/pattern.py | 10 ++-- git_vuln_finder/vulnerability.py | 94 ++++++++++++++++++-------------- 4 files changed, 113 insertions(+), 82 deletions(-) diff --git a/bin/finder.py b/bin/finder.py index bbefd76..db065db 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -16,34 +16,44 @@ import sys import argparse import typing -from git_vuln_finder import ( - get_patterns, - find_vuln, - summary -) +from git_vuln_finder import get_patterns, find_vuln, summary def main(): """Point of entry for the script. """ # Parsing arguments - parser = argparse.ArgumentParser(description = "Finding potential software vulnerabilities from git commit messages.", epilog = "More info: https://github.com/cve-search/git-vuln-finder") + parser = argparse.ArgumentParser( + description="Finding potential software vulnerabilities from git commit messages.", + epilog="More info: https://github.com/cve-search/git-vuln-finder", + ) parser.add_argument("-v", help="increase output verbosity", action="store_true") parser.add_argument("-r", type=str, help="git repository to analyse") parser.add_argument("-o", type=str, help="Output format: [json]", default="json") - parser.add_argument("-s", type=str, help="State of the commit found", default="under-review") - parser.add_argument("-p", type=str, help="Matching pattern to use: [vulnpatterns, cryptopatterns, cpatterns] - the pattern 'all' is used to match all the patterns at once.", default="vulnpatterns") - parser.add_argument("-c", help="output only a list of the CVE pattern found in commit messages (disable by default)", action="store_true") - parser.add_argument("-t", help="Include tags matching a specific commit", action="store_true") + parser.add_argument( + "-s", type=str, help="State of the commit found", default="under-review" + ) + parser.add_argument( + "-p", + type=str, + help="Matching pattern to use: [vulnpatterns, cryptopatterns, cpatterns] - the pattern 'all' is used to match all the patterns at once.", + default="vulnpatterns", + ) + parser.add_argument( + "-c", + help="output only a list of the CVE pattern found in commit messages (disable by default)", + action="store_true", + ) + parser.add_argument( + "-t", help="Include tags matching a specific commit", action="store_true" + ) args = parser.parse_args() - patterns = get_patterns() vulnpatterns = patterns["en"]["medium"]["vuln"] cryptopatterns = patterns["en"]["medium"]["crypto"] cpatterns = patterns["en"]["medium"]["c"] - if args.p == "vulnpatterns": defaultpattern = vulnpatterns elif args.p == "cryptopatterns": @@ -62,13 +72,11 @@ def main(): else: repo = git.Repo(args.r) - # Initialization of the variables for the results found = 0 all_potential_vulnerabilities = {} cve_found = set() - repo_heads = repo.heads repo_heads_names = [h.name for h in repo_heads] print(repo_heads_names, file=sys.stderr) @@ -85,32 +93,36 @@ def main(): if isinstance(defaultpattern, typing.Pattern): ret = find_vuln(commit, pattern=defaultpattern, verbose=args.v) if ret: - rcommit = ret['commit'] - _, potential_vulnerabilities = summary(repo, - rcommit, - branch, - tagmap, - defaultpattern, - origin=origin, - vuln_match=ret['match'], - tags_matching=args.t, - commit_state=args.s) + rcommit = ret["commit"] + _, potential_vulnerabilities = summary( + repo, + rcommit, + branch, + tagmap, + defaultpattern, + origin=origin, + vuln_match=ret["match"], + tags_matching=args.t, + commit_state=args.s, + ) all_potential_vulnerabilities.update(potential_vulnerabilities) found += 1 elif isinstance(defaultpattern, list): for p in defaultpattern: ret = find_vuln(commit, pattern=p, verbose=args.v) if ret: - rcommit = ret['commit'] - _, potential_vulnerabilities = summary(repo, - rcommit, - branch, - tagmap, - p, - origin=origin, - vuln_match=ret['match'], - tags_matching=args.t, - commit_state=args.s) + rcommit = ret["commit"] + _, potential_vulnerabilities = summary( + repo, + rcommit, + branch, + tagmap, + p, + origin=origin, + vuln_match=ret["match"], + tags_matching=args.t, + commit_state=args.s, + ) all_potential_vulnerabilities.update(potential_vulnerabilities) found += 1 @@ -119,5 +131,11 @@ def main(): elif args.c: print(json.dumps(list(cve_found))) - print("{} CVE referenced found in commit(s)".format(len(list(cve_found))), file=sys.stderr) - print("Total potential vulnerability found in {} commit(s)".format(found), file=sys.stderr) + print( + "{} CVE referenced found in commit(s)".format(len(list(cve_found))), + file=sys.stderr, + ) + print( + "Total potential vulnerability found in {} commit(s)".format(found), + file=sys.stderr, + ) diff --git a/git_vuln_finder/__init__.py b/git_vuln_finder/__init__.py index 8b18bb6..2bd2216 100644 --- a/git_vuln_finder/__init__.py +++ b/git_vuln_finder/__init__.py @@ -1,4 +1,3 @@ - from git_vuln_finder.pattern import build_pattern from git_vuln_finder.pattern import get_patterns from git_vuln_finder.vulnerability import find_vuln diff --git a/git_vuln_finder/pattern.py b/git_vuln_finder/pattern.py index e853249..8d08d53 100644 --- a/git_vuln_finder/pattern.py +++ b/git_vuln_finder/pattern.py @@ -14,7 +14,7 @@ import os import re -PATTERNS_PATH="./git_vuln_finder/patterns" +PATTERNS_PATH = "./git_vuln_finder/patterns" def build_pattern(pattern_file): @@ -29,7 +29,7 @@ def build_pattern(pattern_file): for line in fp.readlines(): rex += line.rstrip() + "|" - rex = rex[:-1] # We remove the extra '| + rex = rex[:-1] # We remove the extra '| fp.close() try: @@ -49,9 +49,9 @@ def get_patterns(patterns_path=PATTERNS_PATH): for f in files: if f.endswith(".prefix") or f.endswith(".suffix"): continue - npath = root[len(patterns_path):].split(os.sep) + npath = root[len(patterns_path) :].split(os.sep) try: - npath.remove('') + npath.remove("") except ValueError: pass @@ -59,7 +59,7 @@ def get_patterns(patterns_path=PATTERNS_PATH): severity = npath[1] pattern_category = f - try: # FIXME: Is there a better way? + try: # FIXME: Is there a better way? a = patterns[lang] except KeyError: patterns[lang] = {} diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index bf2d8a7..d460d50 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -23,74 +23,88 @@ def find_vuln(commit, pattern, verbose=False): print(commit.message, file=sys.stderr) print("---", file=sys.stderr) ret = {} - ret['commit'] = commit - ret['match'] = m.groups() + ret["commit"] = commit + ret["match"] = m.groups() return ret else: return None -def summary(repo, - commit, - branch, - tagmap, - pattern, - origin=None, - vuln_match=None, - tags_matching=False, - commit_state="under-review" +def summary( + repo, + commit, + branch, + tagmap, + pattern, + origin=None, + vuln_match=None, + tags_matching=False, + commit_state="under-review", ): potential_vulnerabilities = {} rcommit = commit cve = extract_cve(rcommit.message) if origin is not None: origin = origin - if origin.find('github.com'): - origin_github_api = origin.split(':')[1] - (org_name, repo_name) = origin_github_api.split('/', 1) - if repo_name.find('.git$'): - repo_name = re.sub(r".git$","", repo_name) - origin_github_api = 'https://api.github.com/repos/{}/{}/commits/{}'.format(org_name, repo_name, rcommit.hexsha) + if origin.find("github.com"): + origin_github_api = origin.split(":")[1] + (org_name, repo_name) = origin_github_api.split("/", 1) + if repo_name.find(".git$"): + repo_name = re.sub(r".git$", "", repo_name) + origin_github_api = "https://api.github.com/repos/{}/{}/commits/{}".format( + org_name, repo_name, rcommit.hexsha + ) else: - origin = 'git origin unknown' + origin = "git origin unknown" # deduplication if similar commits on different branches if rcommit.hexsha in potential_vulnerabilities: - potential_vulnerabilities[rcommit.hexsha]['branches'].append(branch) + potential_vulnerabilities[rcommit.hexsha]["branches"].append(branch) else: potential_vulnerabilities[rcommit.hexsha] = {} - potential_vulnerabilities[rcommit.hexsha]['message'] = rcommit.message - potential_vulnerabilities[rcommit.hexsha]['language'] = langdetect(rcommit.message) - potential_vulnerabilities[rcommit.hexsha]['commit-id'] = rcommit.hexsha - potential_vulnerabilities[rcommit.hexsha]['summary'] = rcommit.summary - potential_vulnerabilities[rcommit.hexsha]['stats'] = rcommit.stats.total - potential_vulnerabilities[rcommit.hexsha]['author'] = rcommit.author.name - potential_vulnerabilities[rcommit.hexsha]['author-email'] = rcommit.author.email - potential_vulnerabilities[rcommit.hexsha]['authored_date'] = rcommit.authored_date - potential_vulnerabilities[rcommit.hexsha]['committed_date'] = rcommit.committed_date - potential_vulnerabilities[rcommit.hexsha]['branches'] = [] - potential_vulnerabilities[rcommit.hexsha]['branches'].append(branch) - potential_vulnerabilities[rcommit.hexsha]['pattern-selected'] = pattern.pattern - potential_vulnerabilities[rcommit.hexsha]['pattern-matches'] = vuln_match - potential_vulnerabilities[rcommit.hexsha]['origin'] = origin + potential_vulnerabilities[rcommit.hexsha]["message"] = rcommit.message + potential_vulnerabilities[rcommit.hexsha]["language"] = langdetect( + rcommit.message + ) + potential_vulnerabilities[rcommit.hexsha]["commit-id"] = rcommit.hexsha + potential_vulnerabilities[rcommit.hexsha]["summary"] = rcommit.summary + potential_vulnerabilities[rcommit.hexsha]["stats"] = rcommit.stats.total + potential_vulnerabilities[rcommit.hexsha]["author"] = rcommit.author.name + potential_vulnerabilities[rcommit.hexsha]["author-email"] = rcommit.author.email + potential_vulnerabilities[rcommit.hexsha][ + "authored_date" + ] = rcommit.authored_date + potential_vulnerabilities[rcommit.hexsha][ + "committed_date" + ] = rcommit.committed_date + potential_vulnerabilities[rcommit.hexsha]["branches"] = [] + potential_vulnerabilities[rcommit.hexsha]["branches"].append(branch) + potential_vulnerabilities[rcommit.hexsha]["pattern-selected"] = pattern.pattern + potential_vulnerabilities[rcommit.hexsha]["pattern-matches"] = vuln_match + potential_vulnerabilities[rcommit.hexsha]["origin"] = origin if origin_github_api: - potential_vulnerabilities[commit.hexsha]['origin-github-api'] = origin_github_api - potential_vulnerabilities[rcommit.hexsha]['tags'] = [] + potential_vulnerabilities[commit.hexsha][ + "origin-github-api" + ] = origin_github_api + potential_vulnerabilities[rcommit.hexsha]["tags"] = [] if tags_matching: if repo.commit(rcommit).hexsha in tagmap: - potential_vulnerabilities[rcommit.hexsha]['tags'] = tagmap[repo.commit(rcommit).hexsha] - if cve: potential_vulnerabilities[rcommit.hexsha]['cve'] = cve + potential_vulnerabilities[rcommit.hexsha]["tags"] = tagmap[ + repo.commit(rcommit).hexsha + ] if cve: - potential_vulnerabilities[rcommit.hexsha]['state'] = "cve-assigned" + potential_vulnerabilities[rcommit.hexsha]["cve"] = cve + if cve: + potential_vulnerabilities[rcommit.hexsha]["state"] = "cve-assigned" else: - potential_vulnerabilities[rcommit.hexsha]['state'] = commit_state + potential_vulnerabilities[rcommit.hexsha]["state"] = commit_state return rcommit.hexsha, potential_vulnerabilities def extract_cve(commit): cve_found = set() - cve_find = re.compile(r'CVE-[1-2]\d{1,4}-\d{1,7}', re.IGNORECASE) + cve_find = re.compile(r"CVE-[1-2]\d{1,4}-\d{1,7}", re.IGNORECASE) m = cve_find.findall(commit) if m: for v in m: From 851d8946fd127cbb040b337a9f8e4bce8da865de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 08:08:08 +0100 Subject: [PATCH 14/42] '# FIXME: Is there a better way?': yes --- git_vuln_finder/pattern.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/git_vuln_finder/pattern.py b/git_vuln_finder/pattern.py index 8d08d53..2d64d3c 100644 --- a/git_vuln_finder/pattern.py +++ b/git_vuln_finder/pattern.py @@ -12,6 +12,14 @@ import os import re +from collections import defaultdict + + +def tree(): + """Autovivification. + Call it a tree or call it 'patterns'. + """ + return defaultdict(tree) PATTERNS_PATH = "./git_vuln_finder/patterns" @@ -43,7 +51,7 @@ def build_pattern(pattern_file): def get_patterns(patterns_path=PATTERNS_PATH): - patterns = {} + patterns = tree() for root, dirs, files in os.walk(patterns_path): path = root.split(os.sep) for f in files: @@ -59,18 +67,7 @@ def get_patterns(patterns_path=PATTERNS_PATH): severity = npath[1] pattern_category = f - try: # FIXME: Is there a better way? - a = patterns[lang] - except KeyError: - patterns[lang] = {} - try: - a = patterns[lang][severity] - except KeyError: - patterns[lang][severity] = {} - try: - a = patterns[lang][severity][pattern_category] - except KeyError: - rex = build_pattern(root + os.sep + f) - patterns[lang][severity][pattern_category] = re.compile(rex) + rex = build_pattern(root + os.sep + f) + patterns[lang][severity][pattern_category] = re.compile(rex) return patterns From 1460433429d94c788264e5e71d0f21ca161b19ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 08:22:37 +0100 Subject: [PATCH 15/42] it should be safe to do that --- git_vuln_finder/vulnerability.py | 1 - 1 file changed, 1 deletion(-) diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index d460d50..d1df6d0 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -94,7 +94,6 @@ def summary( ] if cve: potential_vulnerabilities[rcommit.hexsha]["cve"] = cve - if cve: potential_vulnerabilities[rcommit.hexsha]["state"] = "cve-assigned" else: potential_vulnerabilities[rcommit.hexsha]["state"] = commit_state From 73d5349b08fc81688b46b97582abf48878b58d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 15:50:50 +0100 Subject: [PATCH 16/42] in place merge number of CVE found via summary function --- bin/finder.py | 12 +++++++----- git_vuln_finder/vulnerability.py | 8 ++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bin/finder.py b/bin/finder.py index db065db..0df0c86 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -75,7 +75,7 @@ def main(): # Initialization of the variables for the results found = 0 all_potential_vulnerabilities = {} - cve_found = set() + all_cve_found = set() repo_heads = repo.heads repo_heads_names = [h.name for h in repo_heads] @@ -94,7 +94,7 @@ def main(): ret = find_vuln(commit, pattern=defaultpattern, verbose=args.v) if ret: rcommit = ret["commit"] - _, potential_vulnerabilities = summary( + _, potential_vulnerabilities, cve_found = summary( repo, rcommit, branch, @@ -106,13 +106,14 @@ def main(): commit_state=args.s, ) all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) found += 1 elif isinstance(defaultpattern, list): for p in defaultpattern: ret = find_vuln(commit, pattern=p, verbose=args.v) if ret: rcommit = ret["commit"] - _, potential_vulnerabilities = summary( + _, potential_vulnerabilities, cve_found = summary( repo, rcommit, branch, @@ -124,15 +125,16 @@ def main(): commit_state=args.s, ) all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) found += 1 if not args.c: print(json.dumps(all_potential_vulnerabilities)) elif args.c: - print(json.dumps(list(cve_found))) + print(json.dumps(list(all_cve_found))) print( - "{} CVE referenced found in commit(s)".format(len(list(cve_found))), + "{} CVE referenced found in commit(s)".format(len(list(all_cve_found))), file=sys.stderr, ) print( diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index d1df6d0..ad4eec6 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -43,7 +43,7 @@ def summary( ): potential_vulnerabilities = {} rcommit = commit - cve = extract_cve(rcommit.message) + cve, cve_found = extract_cve(rcommit.message) if origin is not None: origin = origin if origin.find("github.com"): @@ -98,7 +98,7 @@ def summary( else: potential_vulnerabilities[rcommit.hexsha]["state"] = commit_state - return rcommit.hexsha, potential_vulnerabilities + return rcommit.hexsha, potential_vulnerabilities, cve_found def extract_cve(commit): @@ -108,6 +108,6 @@ def extract_cve(commit): if m: for v in m: cve_found.add(v) - return m + return m, cve_found else: - return None + return None, set() From 552937a7bb1872955bef37452a5b6cac75202ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:29:30 +0100 Subject: [PATCH 17/42] Make the whole thing importable in a Python code. --- bin/finder.py | 80 +------------ git_vuln_finder/__init__.py | 1 + git_vuln_finder/vulnerability.py | 81 ++++++++++++- poetry.lock | 199 ++++++++++++++++++++++++++++++- pyproject.toml | 1 + 5 files changed, 286 insertions(+), 76 deletions(-) diff --git a/bin/finder.py b/bin/finder.py index 0df0c86..f8c692c 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -10,13 +10,11 @@ # Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be -import git import json import sys import argparse -import typing -from git_vuln_finder import get_patterns, find_vuln, summary +from git_vuln_finder import find, find_vuln, summary def main(): @@ -49,84 +47,18 @@ def main(): ) args = parser.parse_args() - patterns = get_patterns() - vulnpatterns = patterns["en"]["medium"]["vuln"] - cryptopatterns = patterns["en"]["medium"]["crypto"] - cpatterns = patterns["en"]["medium"]["c"] - - if args.p == "vulnpatterns": - defaultpattern = vulnpatterns - elif args.p == "cryptopatterns": - defaultpattern = cryptopatterns - elif args.p == "cpatterns": - defaultpattern = cpatterns - elif args.p == "all": - defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] - else: + if args.p not in ["vulnpatterns", "cryptopatterns", "cpatterns", "all"]: parser.print_usage() parser.exit() if not args.r: parser.print_usage() parser.exit() - else: - repo = git.Repo(args.r) - # Initialization of the variables for the results - found = 0 - all_potential_vulnerabilities = {} - all_cve_found = set() - - repo_heads = repo.heads - repo_heads_names = [h.name for h in repo_heads] - print(repo_heads_names, file=sys.stderr) - origin = repo.remotes.origin.url - tagmap = {} - if args.t: - for t in repo.tags: - tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) - - for branch in repo_heads_names: - commits = list(repo.iter_commits(branch)) - defaultpattern - for commit in commits: - if isinstance(defaultpattern, typing.Pattern): - ret = find_vuln(commit, pattern=defaultpattern, verbose=args.v) - if ret: - rcommit = ret["commit"] - _, potential_vulnerabilities, cve_found = summary( - repo, - rcommit, - branch, - tagmap, - defaultpattern, - origin=origin, - vuln_match=ret["match"], - tags_matching=args.t, - commit_state=args.s, - ) - all_potential_vulnerabilities.update(potential_vulnerabilities) - all_cve_found.update(cve_found) - found += 1 - elif isinstance(defaultpattern, list): - for p in defaultpattern: - ret = find_vuln(commit, pattern=p, verbose=args.v) - if ret: - rcommit = ret["commit"] - _, potential_vulnerabilities, cve_found = summary( - repo, - rcommit, - branch, - tagmap, - p, - origin=origin, - vuln_match=ret["match"], - tags_matching=args.t, - commit_state=args.s, - ) - all_potential_vulnerabilities.update(potential_vulnerabilities) - all_cve_found.update(cve_found) - found += 1 + all_potential_vulnerabilities, all_cve_found, found = find( + args.r, tags_matching=args.t, commit_state=args.s, verbose=args.v, + defaultpattern=args.p + ) if not args.c: print(json.dumps(all_potential_vulnerabilities)) diff --git a/git_vuln_finder/__init__.py b/git_vuln_finder/__init__.py index 2bd2216..8dd73fc 100644 --- a/git_vuln_finder/__init__.py +++ b/git_vuln_finder/__init__.py @@ -1,5 +1,6 @@ from git_vuln_finder.pattern import build_pattern from git_vuln_finder.pattern import get_patterns +from git_vuln_finder.vulnerability import find from git_vuln_finder.vulnerability import find_vuln from git_vuln_finder.vulnerability import summary from git_vuln_finder.vulnerability import extract_cve diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index ad4eec6..9d80b56 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -9,11 +9,90 @@ # # Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be - +import git import re import sys +import typing from langdetect import detect as langdetect +from git_vuln_finder import get_patterns + + +def find(repo, tags_matching=False, commit_state="under-review", verbose=False, defaultpattern="all"): + # Initialization of the variables for the results + repo = git.Repo(repo) + found = 0 + all_potential_vulnerabilities = {} + all_cve_found = set() + + # Initialization of the patterns + patterns = get_patterns() + vulnpatterns = patterns["en"]["medium"]["vuln"] + cryptopatterns = patterns["en"]["medium"]["crypto"] + cpatterns = patterns["en"]["medium"]["c"] + + if defaultpattern == "vulnpatterns": + defaultpattern = vulnpatterns + elif defaultpattern == "cryptopatterns": + defaultpattern = cryptopatterns + elif defaultpattern == "cpatterns": + defaultpattern = cpatterns + elif defaultpattern == "all": + defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] + + repo_heads = repo.heads + repo_heads_names = [h.name for h in repo_heads] + print(repo_heads_names, file=sys.stderr) + origin = repo.remotes.origin.url + tagmap = {} + if tags_matching: + for t in repo.tags: + tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) + + for branch in repo_heads_names: + commits = list(repo.iter_commits(branch)) + defaultpattern + for commit in commits: + if isinstance(defaultpattern, typing.Pattern): + ret = find_vuln(commit, pattern=defaultpattern, verbose=verbose) + if ret: + rcommit = ret["commit"] + _, potential_vulnerabilities, cve_found = summary( + repo, + rcommit, + branch, + tagmap, + defaultpattern, + origin=origin, + vuln_match=ret["match"], + tags_matching=tags_matching, + commit_state=commit_state, + ) + all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) + found += 1 + elif isinstance(defaultpattern, list): + for p in defaultpattern: + ret = find_vuln(commit, pattern=p, verbose=verbose) + if ret: + rcommit = ret["commit"] + _, potential_vulnerabilities, cve_found = summary( + repo, + rcommit, + branch, + tagmap, + p, + origin=origin, + vuln_match=ret["match"], + tags_matching=tags_matching, + commit_state=commit_state, + ) + all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) + found += 1 + + return all_potential_vulnerabilities, all_cve_found, found + def find_vuln(commit, pattern, verbose=False): m = pattern.search(commit.message) diff --git a/poetry.lock b/poetry.lock index 02cd4ef..ff2dd23 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,35 @@ +[[package]] +category = "dev" +description = "Atomic file writes." +marker = "sys_platform == \"win32\"" +name = "atomicwrites" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.3.0" + +[[package]] +category = "dev" +description = "Classes Without Boilerplate" +name = "attrs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "19.3.0" + +[package.extras] +azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] +dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] +docs = ["sphinx", "zope.interface"] +tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] + +[[package]] +category = "dev" +description = "Cross-platform colored terminal text." +marker = "sys_platform == \"win32\"" +name = "colorama" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "0.4.3" + [[package]] category = "dev" description = "Code coverage measurement for Python" @@ -53,6 +85,22 @@ version = "3.0.5" [package.dependencies] gitdb2 = ">=2.0.0" +[[package]] +category = "dev" +description = "Read metadata from Python packages" +marker = "python_version < \"3.8\"" +name = "importlib-metadata" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +version = "1.3.0" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "importlib-resources"] + [[package]] category = "main" description = "Language detection library ported from Google's language-detection." @@ -72,6 +120,14 @@ optional = false python-versions = "*" version = "0.6.1" +[[package]] +category = "dev" +description = "More routines for operating on iterables, beyond itertools" +name = "more-itertools" +optional = false +python-versions = ">=3.5" +version = "8.0.2" + [[package]] category = "dev" description = "Optional static typing for Python" @@ -112,6 +168,42 @@ six = ">=1.7" coverage_plugin = ["coverage (>=4.4.1)"] doc = ["Sphinx (>=1.6.5)", "sphinx-rtd-theme", "mock"] +[[package]] +category = "dev" +description = "Core utilities for Python packages" +name = "packaging" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.0" + +[package.dependencies] +pyparsing = ">=2.0.2" +six = "*" + +[[package]] +category = "dev" +description = "plugin and hook calling mechanisms for python" +name = "pluggy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.13.1" + +[package.dependencies] +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +category = "dev" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +name = "py" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "1.8.1" + [[package]] category = "dev" description = "Python style guide checker" @@ -128,6 +220,39 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.1.1" +[[package]] +category = "dev" +description = "Python parsing module" +name = "pyparsing" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.4.6" + +[[package]] +category = "dev" +description = "pytest: simple powerful testing with Python" +name = "pytest" +optional = false +python-versions = ">=3.5" +version = "5.3.2" + +[package.dependencies] +atomicwrites = ">=1.0" +attrs = ">=17.4.0" +colorama = "*" +more-itertools = ">=4.0.0" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.5.0" +wcwidth = "*" + +[package.dependencies.importlib-metadata] +python = "<3.8" +version = ">=0.12" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + [[package]] category = "main" description = "Python 2 and 3 compatibility utilities" @@ -160,11 +285,47 @@ optional = false python-versions = "*" version = "3.7.4.1" +[[package]] +category = "dev" +description = "Measures number of Terminal column cells of wide-character codes" +name = "wcwidth" +optional = false +python-versions = "*" +version = "0.1.8" + +[[package]] +category = "dev" +description = "Backport of pathlib-compatible object wrapper for zip files" +marker = "python_version < \"3.8\"" +name = "zipp" +optional = false +python-versions = ">=2.7" +version = "0.6.0" + +[package.dependencies] +more-itertools = "*" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pathlib2", "contextlib2", "unittest2"] + [metadata] -content-hash = "4fd05852a9f3844298b1c0dbc4ab61ddbb77f4a42602c42982e19e531a7883d6" +content-hash = "9b6cd9aab07a97a57d8b9c67c7dfc9d859f1c743ef2a901004eea7bd976dd040" python-versions = "^3.6" [metadata.files] +atomicwrites = [ + {file = "atomicwrites-1.3.0-py2.py3-none-any.whl", hash = "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4"}, + {file = "atomicwrites-1.3.0.tar.gz", hash = "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"}, +] +attrs = [ + {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, + {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, +] +colorama = [ + {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, + {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, +] coverage = [ {file = "coverage-5.0.1-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:c90bda74e16bcd03861b09b1d37c0a4158feda5d5a036bb2d6e58de6ff65793e"}, {file = "coverage-5.0.1-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:bb3d29df5d07d5399d58a394d0ef50adf303ab4fbf66dfd25b9ef258effcb692"}, @@ -214,6 +375,10 @@ gitpython = [ {file = "GitPython-3.0.5-py3-none-any.whl", hash = "sha256:c155c6a2653593ccb300462f6ef533583a913e17857cfef8fc617c246b6dc245"}, {file = "GitPython-3.0.5.tar.gz", hash = "sha256:9c2398ffc3dcb3c40b27324b316f08a4f93ad646d5a6328cafbb871aa79f5e42"}, ] +importlib-metadata = [ + {file = "importlib_metadata-1.3.0-py2.py3-none-any.whl", hash = "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f"}, + {file = "importlib_metadata-1.3.0.tar.gz", hash = "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45"}, +] langdetect = [ {file = "langdetect-1.0.7.zip", hash = "sha256:91a170d5f0ade380db809b3ba67f08e95fe6c6c8641f96d67a51ff7e98a9bf30"}, ] @@ -221,6 +386,10 @@ mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] +more-itertools = [ + {file = "more-itertools-8.0.2.tar.gz", hash = "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d"}, + {file = "more_itertools-8.0.2-py3-none-any.whl", hash = "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564"}, +] mypy = [ {file = "mypy-0.750-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:de9ec8dba773b78c49e7bec9a35c9b6fc5235682ad1fc2105752ae7c22f4b931"}, {file = "mypy-0.750-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:3294821b5840d51a3cd7a2bb63b40fc3f901f6a3cfb3c6046570749c4c7ef279"}, @@ -245,6 +414,18 @@ nose2 = [ {file = "nose2-0.9.1-py2.py3-none-any.whl", hash = "sha256:31d8beb00aed3ccc6efb1742bb90227d883e471715188249f594310676e0ef0e"}, {file = "nose2-0.9.1.tar.gz", hash = "sha256:0ede156fd7974fa40893edeca0b709f402c0ccacd7b81b22e76f73c116d1b999"}, ] +packaging = [ + {file = "packaging-20.0-py2.py3-none-any.whl", hash = "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb"}, + {file = "packaging-20.0.tar.gz", hash = "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.8.1-py2.py3-none-any.whl", hash = "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0"}, + {file = "py-1.8.1.tar.gz", hash = "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa"}, +] pycodestyle = [ {file = "pycodestyle-2.5.0-py2.py3-none-any.whl", hash = "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56"}, {file = "pycodestyle-2.5.0.tar.gz", hash = "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"}, @@ -253,6 +434,14 @@ pyflakes = [ {file = "pyflakes-2.1.1-py2.py3-none-any.whl", hash = "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0"}, {file = "pyflakes-2.1.1.tar.gz", hash = "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"}, ] +pyparsing = [ + {file = "pyparsing-2.4.6-py2.py3-none-any.whl", hash = "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"}, + {file = "pyparsing-2.4.6.tar.gz", hash = "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f"}, +] +pytest = [ + {file = "pytest-5.3.2-py3-none-any.whl", hash = "sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4"}, + {file = "pytest-5.3.2.tar.gz", hash = "sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa"}, +] six = [ {file = "six-1.13.0-py2.py3-none-any.whl", hash = "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd"}, {file = "six-1.13.0.tar.gz", hash = "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"}, @@ -288,3 +477,11 @@ typing-extensions = [ {file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"}, {file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"}, ] +wcwidth = [ + {file = "wcwidth-0.1.8-py2.py3-none-any.whl", hash = "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603"}, + {file = "wcwidth-0.1.8.tar.gz", hash = "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"}, +] +zipp = [ + {file = "zipp-0.6.0-py2.py3-none-any.whl", hash = "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335"}, + {file = "zipp-0.6.0.tar.gz", hash = "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e"}, +] diff --git a/pyproject.toml b/pyproject.toml index 95f779a..fcd1a24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ gitpython = "^3.0.5" mypy = "^0.750" flake8 = "^3.7.9" nose2 = "^0.9.1" +pytest = "^5.3.2" [build-system] requires = ["poetry>=0.12"] From ceb2c63458f24997dfb897b278014110b5ead9ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:29:45 +0100 Subject: [PATCH 18/42] added basic test --- tests/test_finder.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/test_finder.py diff --git a/tests/test_finder.py b/tests/test_finder.py new file mode 100644 index 0000000..f990708 --- /dev/null +++ b/tests/test_finder.py @@ -0,0 +1,9 @@ + + +from git_vuln_finder import find + + +def test_find_vuln(): + all_potential_vulnerabilities, all_cve_found, found = find("/home/cedric/git/curl") + + assert len(list(all_cve_found)) == 63 From 86b58987d330854f244289c36523dae5fac769f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:32:50 +0100 Subject: [PATCH 19/42] added GitHub workflow --- .github/workflows/pythonapp.yml | 35 +++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/pythonapp.yml diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml new file mode 100644 index 0000000..1389c87 --- /dev/null +++ b/.github/workflows/pythonapp.yml @@ -0,0 +1,35 @@ +name: Python application + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + pip install poetry + poetry install + + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Test with pytest + run: | + poetry run pytest + env: + testing: actions From e12d2537935f93bed6f28f5a5d220daf052088c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:33:55 +0100 Subject: [PATCH 20/42] removed nose2 --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fcd1a24..0401c82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ gitpython = "^3.0.5" [tool.poetry.dev-dependencies] mypy = "^0.750" flake8 = "^3.7.9" -nose2 = "^0.9.1" pytest = "^5.3.2" [build-system] From 81786106d027e34136de2a0ffef15be6d4b0b970 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:43:43 +0100 Subject: [PATCH 21/42] checkout a repo --- .github/workflows/pythonapp.yml | 5 +++++ tests/test_finder.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 1389c87..10bb338 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -28,6 +28,11 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Checkout curl for the tests + uses: curl/curl@7.67.0 + with: + path: test_repos/curl + - name: Test with pytest run: | poetry run pytest diff --git a/tests/test_finder.py b/tests/test_finder.py index f990708..4bd9e29 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -4,6 +4,6 @@ from git_vuln_finder import find def test_find_vuln(): - all_potential_vulnerabilities, all_cve_found, found = find("/home/cedric/git/curl") + all_potential_vulnerabilities, all_cve_found, found = find("test_repos/curl") assert len(list(all_cve_found)) == 63 From b4ead9123e4366cae389763ac7b3f47af817d72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:49:27 +0100 Subject: [PATCH 22/42] fixed tag of curl --- .github/workflows/pythonapp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 10bb338..b69f517 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -29,7 +29,7 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Checkout curl for the tests - uses: curl/curl@7.67.0 + uses: curl/curl@curl-7_67_0 with: path: test_repos/curl From 29cbb7c98225bd73fb0c15e64d8136d393001012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:52:45 +0100 Subject: [PATCH 23/42] checkout curl with actions/checkout@v2 --- .github/workflows/pythonapp.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index b69f517..4386848 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -29,8 +29,10 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Checkout curl for the tests - uses: curl/curl@curl-7_67_0 + uses: actions/checkout@v2 with: + repository: curl/curl + ref: curl-7_67_0 path: test_repos/curl - name: Test with pytest From 86e9f1ff5b1fb1ea625954e3b1b01c4e4a0049db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 16:58:30 +0100 Subject: [PATCH 24/42] wip --- tests/test_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_finder.py b/tests/test_finder.py index 4bd9e29..77ef6a9 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -4,6 +4,6 @@ from git_vuln_finder import find def test_find_vuln(): - all_potential_vulnerabilities, all_cve_found, found = find("test_repos/curl") + all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl") assert len(list(all_cve_found)) == 63 From 4307c411739cfa26364af08c59d6cfd5fa58540b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 17:00:36 +0100 Subject: [PATCH 25/42] wip --- tests/test_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_finder.py b/tests/test_finder.py index 77ef6a9..07ee774 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -4,6 +4,6 @@ from git_vuln_finder import find def test_find_vuln(): - all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl") + all_potential_vulnerabilities, all_cve_found, found = find("/home/runner/work/git-vuln-finder/git-vuln-finder/test_repos/curl/") assert len(list(all_cve_found)) == 63 From a61719bf16b18f0951c43e116b8842c7b691139f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 17:04:10 +0100 Subject: [PATCH 26/42] Updated README> --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 5100ed8..59dd589 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,14 @@ $ poetry shell You can also use ``pip``. +~~~python +Python 3.8.0 (default, Dec 11 2019, 21:43:13) +[GCC 9.2.1 20191008] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> from git_vuln_finder import find +>>> all_potential_vulnerabilities, all_cve_found, found = find("~/git/curl") +~~~ + ## Use as a command line tool From a4a17ba265e3bf75e39ac8dc8a0007d0a4775de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:10:09 +0100 Subject: [PATCH 27/42] checkout version curl-7_67_0 of curl for the tests. --- .github/workflows/pythonapp.yml | 7 ------- tests/test_finder.py | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 4386848..1389c87 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -28,13 +28,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Checkout curl for the tests - uses: actions/checkout@v2 - with: - repository: curl/curl - ref: curl-7_67_0 - path: test_repos/curl - - name: Test with pytest run: | poetry run pytest diff --git a/tests/test_finder.py b/tests/test_finder.py index 07ee774..35cb2a2 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -3,7 +3,7 @@ from git_vuln_finder import find -def test_find_vuln(): - all_potential_vulnerabilities, all_cve_found, found = find("/home/runner/work/git-vuln-finder/git-vuln-finder/test_repos/curl/") +def test_find_vuln(clone_curl): + all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl/") assert len(list(all_cve_found)) == 63 From 1aec0290af72f91db6eedd648fba0bb637e3dea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:10:30 +0100 Subject: [PATCH 28/42] checkout version curl-7_67_0 of curl for the tests. --- tests/conftest.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5267a2e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ + +import os +import pytest + +from git import Repo + + +@pytest.fixture(scope='session') +def clone_curl(): + """Clone the repository of curl for the tests.""" + git_url = 'https://github.com/curl/curl.git' + repo_dir = './test_repos/curl' + repo = Repo.clone_from(url=git_url, to_path=repo_dir) + repo.checkout('curl-7_67_0') + + def teardown(): + os.unlink(repo_dir) + + return repo_dir From 2e1d646f994bbf6d2cc2a24833c0b21a938583e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:11:43 +0100 Subject: [PATCH 29/42] checkout version curl-7_67_0 of curl for the tests. --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 5267a2e..2b6f692 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ def clone_curl(): git_url = 'https://github.com/curl/curl.git' repo_dir = './test_repos/curl' repo = Repo.clone_from(url=git_url, to_path=repo_dir) - repo.checkout('curl-7_67_0') + repo.heads['curl-7_67_0'].checkout() def teardown(): os.unlink(repo_dir) From 43c8c432b896e1b98b059c90d53fef7239762288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:12:57 +0100 Subject: [PATCH 30/42] updated gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 119966c..88e45aa 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ dist/* .coverage .mypy_cache/ .cache/ +.test_repos/ # sphinx docs/_build From d2b88a3e6cb714774700f59eb10a627eb60ea519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:16:47 +0100 Subject: [PATCH 31/42] updated README. --- README.md | 4 ++-- tests/conftest.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 59dd589..f17f628 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ added automatically in the output. # Installation -## Use in your Python software +## Use it as a library ~~~bash $ poetry install git-vuln-finder @@ -33,7 +33,7 @@ Type "help", "copyright", "credits" or "license" for more information. ~~~ -## Use as a command line tool +## Use it as a command line tool ~~~bash $ pipx install git-vuln-finder diff --git a/tests/conftest.py b/tests/conftest.py index 2b6f692..e37d33b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,7 @@ def clone_curl(): git_url = 'https://github.com/curl/curl.git' repo_dir = './test_repos/curl' repo = Repo.clone_from(url=git_url, to_path=repo_dir) - repo.heads['curl-7_67_0'].checkout() + #repo.heads['curl-7_67_0'].checkout() def teardown(): os.unlink(repo_dir) From 864def653a0abdccb094d4fda982bdecb7be6807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:24:30 +0100 Subject: [PATCH 32/42] updated the tests --- tests/test_finder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_finder.py b/tests/test_finder.py index 35cb2a2..4625ea2 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -6,4 +6,5 @@ from git_vuln_finder import find def test_find_vuln(clone_curl): all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl/") - assert len(list(all_cve_found)) == 63 + #assert len(list(all_cve_found)) == 64 + assert "CVE-2018-1000122" in all_cve_found From d43bddd65a20808ef3ff81c99cf8ef1dcdb19e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:28:24 +0100 Subject: [PATCH 33/42] added badge with temporary address --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f17f628..8b14f9d 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ ![git-vuln-finder logo](https://raw.githubusercontent.com/cve-search/git-vuln-finder/f22077452c37e110bff0564e1f7b34637dc726c3/doc/logos/git-vuln-finder-small.png) +[![Workflow](https://github.com/cedricbonhomme/git-vuln-finder/workflows/Python%20application/badge.svg?style=flat-square)](https://github.com/cedricbonhomme/git-vuln-finder/actions?query=workflow%3A%22Python+application%22) + Finding potential software vulnerabilities from git commit messages. The output format is a JSON with the associated commit which could contain a fix regarding a software vulnerability. The search is based on a set of regular From aab859b2632850d0ad29285e76f04752d40b9caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 18:30:27 +0100 Subject: [PATCH 34/42] Updated README. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8b14f9d..73b8afd 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ $ poetry install git-vuln-finder $ poetry shell ~~~ -You can also use ``pip``. +You can also use ``pip``. Then just import it: ~~~python Python 3.8.0 (default, Dec 11 2019, 21:43:13) @@ -46,7 +46,7 @@ $ git-vuln-finder --help separate virtualenvs to shield them from your system and each other. -# Usage +### Usage ~~~bash usage: git-vuln-finder [-h] [-v] [-r R] [-o O] [-s S] [-p P] [-c] [-t] From 30af30f571a693e3c62d48b4ed08994f7e3a4089 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 21:28:31 +0100 Subject: [PATCH 35/42] added a simple comment --- git_vuln_finder/vulnerability.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index d1df6d0..326e1fe 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -16,6 +16,9 @@ from langdetect import detect as langdetect def find_vuln(commit, pattern, verbose=False): + """Find a potential vulnerability from a commit message thanks to a regex + pattern. + """ m = pattern.search(commit.message) if m: if verbose: From e10ab62c718a8a2e541db20384333c4be5cd546c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 22:02:28 +0100 Subject: [PATCH 36/42] updated README. --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 6 ++++-- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 73b8afd..c1d67bb 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,43 @@ Python 3.8.0 (default, Dec 11 2019, 21:43:13) Type "help", "copyright", "credits" or "license" for more information. >>> from git_vuln_finder import find >>> all_potential_vulnerabilities, all_cve_found, found = find("~/git/curl") + +>>> [commit for commit, summary in all_potential_vulnerabilities.items() if summary['state'] == 'cve-assigned'] +['9069838b30fb3b48af0123e39f664cea683254a5', 'facb0e4662415b5f28163e853dc6742ac5fafb3d', +... snap ... + '8a75dbeb2305297640453029b7905ef51b87e8dd', '1dc43de0dccc2ea7da6dddb7b98f8d7dcf323914', '192c4f788d48f82c03e9cef40013f34370e90737', '2eb8dcf26cb37f09cffe26909a646e702dbcab66', 'fa1ae0abcde5df8d0b3283299e3f246bedf7692c', 'c11c30a8c8d727dcf5634fa0cc6ee0b4b77ddc3d', '75ca568fa1c19de4c5358fed246686de8467c238', 'a20daf90e358c1476a325ea665d533f7a27e3364', '042cc1f69ec0878f542667cb684378869f859911'] + >>> print(json.dumps(all_potential_vulnerabilities['9069838b30fb3b48af0123e39f664cea683254a5'], sort_keys=True, indent=4, separators=(",", ": "))) + { + "author": "Daniel Stenberg", + "author-email": "daniel@haxx.se", + "authored_date": 1567544372, + "branches": [ + "master" + ], + "commit-id": "9069838b30fb3b48af0123e39f664cea683254a5", + "committed_date": 1568009674, + "cve": [ + "CVE-2019-5481", + "CVE-2019-5481" + ], + "language": "en", + "message": "security:read_data fix bad realloc()\n\n... that could end up a double-free\n\nCVE-2019-5481\nBug: https://curl.haxx.se/docs/CVE-2019-5481.html\n", + "origin": "https://github.com/curl/curl.git", + "origin-github-api": "https://api.github.com/repos///github.com/curl/curl/commits/9069838b30fb3b48af0123e39f664cea683254a5", + "pattern-matches": [ + "double-free" + ], + "pattern-selected": "(?i)(double[-| ]free|buffer overflow|double free|race[-| ]condition)", + "state": "cve-assigned", + "stats": { + "deletions": 4, + "files": 1, + "insertions": 2, + "lines": 6 + }, + "summary": "security:read_data fix bad realloc()", + "tags": [] + } ~~~ @@ -181,22 +218,33 @@ ploit|malicious|directory traversal |\bRCE\b|\bdos\b|\bXSRF \b|\bXSS\b|clickjack } ~~~ + +#Running the tests + +~~~bash +$ pytest +~~~ + + # License and author(s) This software is free software and licensed under the AGPL version 3. Copyright (c) 2019-2020 Alexandre Dulaunoy - https://github.com/adulau/ + # Acknowledgment - Thanks to [Jean-Louis Huynen](https://github.com/gallypette) for the discussions about the crypto vulnerability patterns. - Thanks to [Sebastien Tricaud](https://github.com/stricaud) for the discussions regarding native language, commit messages and external patterns. + # Contributing We welcome contributions for the software and especially additional vulnerability patterns. Every contributors will be added in the [AUTHORS file](./AUTHORS) and collectively own this open source software. The contributors acknowledge the [Developer Certificate of Origin](https://developercertificate.org/). + # References - [Notes](https://gist.github.com/adulau/dce5a6ca5c65017869bb01dfee576303#file-finding-vuln-git-commit-messages-md) diff --git a/pyproject.toml b/pyproject.toml index 0401c82..d98e39f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,15 +24,17 @@ keywords = [ ] classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 4 - Beta Copy", "Environment :: Console", "Intended Audience :: Developers", + "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "Topic :: Security", "Operating System :: OS Independent", + "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)" + "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)" ] include = [ From df6c4c163eab777f063973abcd2ebae4c719f6be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 22:30:14 +0100 Subject: [PATCH 37/42] moved find function used for the cli in a file run.py --- .gitignore | 2 +- README.md | 8 ++- bin/finder.py | 7 ++- git_vuln_finder/__init__.py | 2 +- git_vuln_finder/run.py | 97 ++++++++++++++++++++++++++++++++ git_vuln_finder/vulnerability.py | 80 -------------------------- tests/test_finder.py | 2 +- 7 files changed, 111 insertions(+), 87 deletions(-) create mode 100644 git_vuln_finder/run.py diff --git a/.gitignore b/.gitignore index 88e45aa..4a23fa9 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,7 @@ dist/* .coverage .mypy_cache/ .cache/ -.test_repos/ +test_repos/ # sphinx docs/_build diff --git a/README.md b/README.md index c1d67bb..befd4c7 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Type "help", "copyright", "credits" or "license" for more information. ['9069838b30fb3b48af0123e39f664cea683254a5', 'facb0e4662415b5f28163e853dc6742ac5fafb3d', ... snap ... '8a75dbeb2305297640453029b7905ef51b87e8dd', '1dc43de0dccc2ea7da6dddb7b98f8d7dcf323914', '192c4f788d48f82c03e9cef40013f34370e90737', '2eb8dcf26cb37f09cffe26909a646e702dbcab66', 'fa1ae0abcde5df8d0b3283299e3f246bedf7692c', 'c11c30a8c8d727dcf5634fa0cc6ee0b4b77ddc3d', '75ca568fa1c19de4c5358fed246686de8467c238', 'a20daf90e358c1476a325ea665d533f7a27e3364', '042cc1f69ec0878f542667cb684378869f859911'] + >>> print(json.dumps(all_potential_vulnerabilities['9069838b30fb3b48af0123e39f664cea683254a5'], sort_keys=True, indent=4, separators=(",", ": "))) { "author": "Daniel Stenberg", @@ -79,8 +80,9 @@ $ pipx install git-vuln-finder $ git-vuln-finder --help ~~~ -``pipx`` installs scripts (system wide available) provided by Python packages into -separate virtualenvs to shield them from your system and each other. +You can also use pip. +``pipx`` installs scripts (system wide available) provided by Python packages +into separate virtualenvs to shield them from your system and each other. ### Usage @@ -106,6 +108,7 @@ optional arguments: More info: https://github.com/cve-search/git-vuln-finder ~~~ + # Patterns git-vuln-finder comes with 3 default patterns which can be selected to find the potential vulnerabilities described in the commit messages such as: @@ -114,6 +117,7 @@ git-vuln-finder comes with 3 default patterns which can be selected to find the - [`cryptopatterns`](https://github.com/cve-search/git-vuln-finder/blob/master/patterns/en/medium/crypto) is a vulnerability pattern for cryptographic errors mentioned in commit messages. - [`cpatterns`](https://github.com/cve-search/git-vuln-finder/blob/master/patterns/en/medium/c) is a set of standard vulnerability patterns see for C/C++-like languages. + ## A sample partial output from Curl git repository ~~~bash diff --git a/bin/finder.py b/bin/finder.py index f8c692c..581a67b 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -56,8 +56,11 @@ def main(): parser.exit() all_potential_vulnerabilities, all_cve_found, found = find( - args.r, tags_matching=args.t, commit_state=args.s, verbose=args.v, - defaultpattern=args.p + args.r, + tags_matching=args.t, + commit_state=args.s, + verbose=args.v, + defaultpattern=args.p, ) if not args.c: diff --git a/git_vuln_finder/__init__.py b/git_vuln_finder/__init__.py index 8dd73fc..a2572bc 100644 --- a/git_vuln_finder/__init__.py +++ b/git_vuln_finder/__init__.py @@ -1,6 +1,6 @@ from git_vuln_finder.pattern import build_pattern from git_vuln_finder.pattern import get_patterns -from git_vuln_finder.vulnerability import find from git_vuln_finder.vulnerability import find_vuln from git_vuln_finder.vulnerability import summary from git_vuln_finder.vulnerability import extract_cve +from git_vuln_finder.run import find diff --git a/git_vuln_finder/run.py b/git_vuln_finder/run.py new file mode 100644 index 0000000..181633b --- /dev/null +++ b/git_vuln_finder/run.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Finding potential software vulnerabilities from git commit messages +# +# Software is free software released under the "GNU Affero General Public License v3.0" +# +# This software is part of cve-search.org +# +# Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be + +import sys +import git +import typing +from git_vuln_finder import get_patterns, find_vuln, summary + + +def find( + repo, + tags_matching=False, + commit_state="under-review", + verbose=False, + defaultpattern="all", +): + # Initialization of the variables for the results + repo = git.Repo(repo) + found = 0 + all_potential_vulnerabilities = {} + all_cve_found = set() + + # Initialization of the patterns + patterns = get_patterns() + vulnpatterns = patterns["en"]["medium"]["vuln"] + cryptopatterns = patterns["en"]["medium"]["crypto"] + cpatterns = patterns["en"]["medium"]["c"] + + if defaultpattern == "vulnpatterns": + defaultpattern = vulnpatterns + elif defaultpattern == "cryptopatterns": + defaultpattern = cryptopatterns + elif defaultpattern == "cpatterns": + defaultpattern = cpatterns + elif defaultpattern == "all": + defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] + + repo_heads = repo.heads + repo_heads_names = [h.name for h in repo_heads] + print(repo_heads_names, file=sys.stderr) + origin = repo.remotes.origin.url + tagmap = {} + if tags_matching: + for t in repo.tags: + tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) + + for branch in repo_heads_names: + commits = list(repo.iter_commits(branch)) + defaultpattern + for commit in commits: + if isinstance(defaultpattern, typing.Pattern): + ret = find_vuln(commit, pattern=defaultpattern, verbose=verbose) + if ret: + rcommit = ret["commit"] + _, potential_vulnerabilities, cve_found = summary( + repo, + rcommit, + branch, + tagmap, + defaultpattern, + origin=origin, + vuln_match=ret["match"], + tags_matching=tags_matching, + commit_state=commit_state, + ) + all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) + found += 1 + elif isinstance(defaultpattern, list): + for p in defaultpattern: + ret = find_vuln(commit, pattern=p, verbose=verbose) + if ret: + rcommit = ret["commit"] + _, potential_vulnerabilities, cve_found = summary( + repo, + rcommit, + branch, + tagmap, + p, + origin=origin, + vuln_match=ret["match"], + tags_matching=tags_matching, + commit_state=commit_state, + ) + all_potential_vulnerabilities.update(potential_vulnerabilities) + all_cve_found.update(cve_found) + found += 1 + + return all_potential_vulnerabilities, all_cve_found, found diff --git a/git_vuln_finder/vulnerability.py b/git_vuln_finder/vulnerability.py index 616f4e8..4581618 100644 --- a/git_vuln_finder/vulnerability.py +++ b/git_vuln_finder/vulnerability.py @@ -9,90 +9,10 @@ # # Copyright (c) 2019-2020 Alexandre Dulaunoy - a@foo.be -import git import re import sys -import typing from langdetect import detect as langdetect -from git_vuln_finder import get_patterns - - -def find(repo, tags_matching=False, commit_state="under-review", verbose=False, defaultpattern="all"): - # Initialization of the variables for the results - repo = git.Repo(repo) - found = 0 - all_potential_vulnerabilities = {} - all_cve_found = set() - - # Initialization of the patterns - patterns = get_patterns() - vulnpatterns = patterns["en"]["medium"]["vuln"] - cryptopatterns = patterns["en"]["medium"]["crypto"] - cpatterns = patterns["en"]["medium"]["c"] - - if defaultpattern == "vulnpatterns": - defaultpattern = vulnpatterns - elif defaultpattern == "cryptopatterns": - defaultpattern = cryptopatterns - elif defaultpattern == "cpatterns": - defaultpattern = cpatterns - elif defaultpattern == "all": - defaultpattern = [vulnpatterns, cryptopatterns, cpatterns] - - repo_heads = repo.heads - repo_heads_names = [h.name for h in repo_heads] - print(repo_heads_names, file=sys.stderr) - origin = repo.remotes.origin.url - tagmap = {} - if tags_matching: - for t in repo.tags: - tagmap.setdefault(repo.commit(t).hexsha, []).append(str(t)) - - for branch in repo_heads_names: - commits = list(repo.iter_commits(branch)) - defaultpattern - for commit in commits: - if isinstance(defaultpattern, typing.Pattern): - ret = find_vuln(commit, pattern=defaultpattern, verbose=verbose) - if ret: - rcommit = ret["commit"] - _, potential_vulnerabilities, cve_found = summary( - repo, - rcommit, - branch, - tagmap, - defaultpattern, - origin=origin, - vuln_match=ret["match"], - tags_matching=tags_matching, - commit_state=commit_state, - ) - all_potential_vulnerabilities.update(potential_vulnerabilities) - all_cve_found.update(cve_found) - found += 1 - elif isinstance(defaultpattern, list): - for p in defaultpattern: - ret = find_vuln(commit, pattern=p, verbose=verbose) - if ret: - rcommit = ret["commit"] - _, potential_vulnerabilities, cve_found = summary( - repo, - rcommit, - branch, - tagmap, - p, - origin=origin, - vuln_match=ret["match"], - tags_matching=tags_matching, - commit_state=commit_state, - ) - all_potential_vulnerabilities.update(potential_vulnerabilities) - all_cve_found.update(cve_found) - found += 1 - - return all_potential_vulnerabilities, all_cve_found, found - def find_vuln(commit, pattern, verbose=False): """Find a potential vulnerability from a commit message thanks to a regex diff --git a/tests/test_finder.py b/tests/test_finder.py index 4625ea2..a4f36d0 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -3,7 +3,7 @@ from git_vuln_finder import find -def test_find_vuln(clone_curl): +def test_find_vuln(): all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl/") #assert len(list(all_cve_found)) == 64 From a98bc112277e90abbd36672923e8b1ed5136734f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 22:33:21 +0100 Subject: [PATCH 38/42] added comments --- bin/finder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/finder.py b/bin/finder.py index 581a67b..8bce93a 100644 --- a/bin/finder.py +++ b/bin/finder.py @@ -55,6 +55,7 @@ def main(): parser.print_usage() parser.exit() + # Launch the process all_potential_vulnerabilities, all_cve_found, found = find( args.r, tags_matching=args.t, @@ -63,11 +64,13 @@ def main(): defaultpattern=args.p, ) + # Output the result as json. Can be piped to another software. if not args.c: print(json.dumps(all_potential_vulnerabilities)) elif args.c: print(json.dumps(list(all_cve_found))) + # Output the result to stderr. print( "{} CVE referenced found in commit(s)".format(len(list(all_cve_found))), file=sys.stderr, From c26f4eb2c11cb4e79eb6700d153b053a890c6114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 22:35:23 +0100 Subject: [PATCH 39/42] won't use mypy here... for the moment... --- poetry.lock | 153 +------------------------------------------------ pyproject.toml | 1 - 2 files changed, 1 insertion(+), 153 deletions(-) diff --git a/poetry.lock b/poetry.lock index ff2dd23..3d54d35 100644 --- a/poetry.lock +++ b/poetry.lock @@ -30,17 +30,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.4.3" -[[package]] -category = "dev" -description = "Code coverage measurement for Python" -name = "coverage" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" -version = "5.0.1" - -[package.extras] -toml = ["toml"] - [[package]] category = "dev" description = "Discover and load entry points from installed packages." @@ -128,46 +117,6 @@ optional = false python-versions = ">=3.5" version = "8.0.2" -[[package]] -category = "dev" -description = "Optional static typing for Python" -name = "mypy" -optional = false -python-versions = ">=3.5" -version = "0.750" - -[package.dependencies] -mypy-extensions = ">=0.4.0,<0.5.0" -typed-ast = ">=1.4.0,<1.5.0" -typing-extensions = ">=3.7.4" - -[package.extras] -dmypy = ["psutil (>=4.0)"] - -[[package]] -category = "dev" -description = "Experimental type system extensions for programs checked with the mypy typechecker." -name = "mypy-extensions" -optional = false -python-versions = "*" -version = "0.4.3" - -[[package]] -category = "dev" -description = "unittest2 with plugins, the succesor to nose" -name = "nose2" -optional = false -python-versions = "*" -version = "0.9.1" - -[package.dependencies] -coverage = ">=4.4.1" -six = ">=1.7" - -[package.extras] -coverage_plugin = ["coverage (>=4.4.1)"] -doc = ["Sphinx (>=1.6.5)", "sphinx-rtd-theme", "mock"] - [[package]] category = "dev" description = "Core utilities for Python packages" @@ -269,22 +218,6 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "2.0.5" -[[package]] -category = "dev" -description = "a fork of Python 2 and 3 ast modules with type comment support" -name = "typed-ast" -optional = false -python-versions = "*" -version = "1.4.0" - -[[package]] -category = "dev" -description = "Backported and Experimental Type Hints for Python 3.5+" -name = "typing-extensions" -optional = false -python-versions = "*" -version = "3.7.4.1" - [[package]] category = "dev" description = "Measures number of Terminal column cells of wide-character codes" @@ -310,7 +243,7 @@ docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] testing = ["pathlib2", "contextlib2", "unittest2"] [metadata] -content-hash = "9b6cd9aab07a97a57d8b9c67c7dfc9d859f1c743ef2a901004eea7bd976dd040" +content-hash = "9faea1409b72575568ba911a1729a6cc7994603342a0d5ae7b045ed15cd091f8" python-versions = "^3.6" [metadata.files] @@ -326,39 +259,6 @@ colorama = [ {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, ] -coverage = [ - {file = "coverage-5.0.1-cp27-cp27m-macosx_10_12_x86_64.whl", hash = "sha256:c90bda74e16bcd03861b09b1d37c0a4158feda5d5a036bb2d6e58de6ff65793e"}, - {file = "coverage-5.0.1-cp27-cp27m-macosx_10_13_intel.whl", hash = "sha256:bb3d29df5d07d5399d58a394d0ef50adf303ab4fbf66dfd25b9ef258effcb692"}, - {file = "coverage-5.0.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:1ca43dbd739c0fc30b0a3637a003a0d2c7edc1dd618359d58cc1e211742f8bd1"}, - {file = "coverage-5.0.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:591506e088901bdc25620c37aec885e82cc896528f28c57e113751e3471fc314"}, - {file = "coverage-5.0.1-cp27-cp27m-win32.whl", hash = "sha256:a50b0888d8a021a3342d36a6086501e30de7d840ab68fca44913e97d14487dc1"}, - {file = "coverage-5.0.1-cp27-cp27m-win_amd64.whl", hash = "sha256:c792d3707a86c01c02607ae74364854220fb3e82735f631cd0a345dea6b4cee5"}, - {file = "coverage-5.0.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f425f50a6dd807cb9043d15a4fcfba3b5874a54d9587ccbb748899f70dc18c47"}, - {file = "coverage-5.0.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:25b8f60b5c7da71e64c18888f3067d5b6f1334b9681876b2fb41eea26de881ae"}, - {file = "coverage-5.0.1-cp35-cp35m-macosx_10_12_x86_64.whl", hash = "sha256:7362a7f829feda10c7265b553455de596b83d1623b3d436b6d3c51c688c57bf6"}, - {file = "coverage-5.0.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:fcd4459fe35a400b8f416bc57906862693c9f88b66dc925e7f2a933e77f6b18b"}, - {file = "coverage-5.0.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:40fbfd6b044c9db13aeec1daf5887d322c710d811f944011757526ef6e323fd9"}, - {file = "coverage-5.0.1-cp35-cp35m-win32.whl", hash = "sha256:7f2675750c50151f806070ec11258edf4c328340916c53bac0adbc465abd6b1e"}, - {file = "coverage-5.0.1-cp35-cp35m-win_amd64.whl", hash = "sha256:24bcfa86fd9ce86b73a8368383c39d919c497a06eebb888b6f0c12f13e920b1a"}, - {file = "coverage-5.0.1-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:eeafb646f374988c22c8e6da5ab9fb81367ecfe81c70c292623373d2a021b1a1"}, - {file = "coverage-5.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:2ca2cd5264e84b2cafc73f0045437f70c6378c0d7dbcddc9ee3fe192c1e29e5d"}, - {file = "coverage-5.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2cc707fc9aad2592fc686d63ef72dc0031fc98b6fb921d2f5395d9ab84fbc3ef"}, - {file = "coverage-5.0.1-cp36-cp36m-win32.whl", hash = "sha256:04b961862334687549eb91cd5178a6fbe977ad365bddc7c60f2227f2f9880cf4"}, - {file = "coverage-5.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:232f0b52a5b978288f0bbc282a6c03fe48cd19a04202df44309919c142b3bb9c"}, - {file = "coverage-5.0.1-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:cfce79ce41cc1a1dc7fc85bb41eeeb32d34a4cf39a645c717c0550287e30ff06"}, - {file = "coverage-5.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:46c9c6a1d1190c0b75ec7c0f339088309952b82ae8d67a79ff1319eb4e749b96"}, - {file = "coverage-5.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1cbb88b34187bdb841f2599770b7e6ff8e259dc3bb64fc7893acf44998acf5f8"}, - {file = "coverage-5.0.1-cp37-cp37m-win32.whl", hash = "sha256:ff3936dd5feaefb4f91c8c1f50a06c588b5dc69fba4f7d9c79a6617ad80bb7df"}, - {file = "coverage-5.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:65bead1ac8c8930cf92a1ccaedcce19a57298547d5d1db5c9d4d068a0675c38b"}, - {file = "coverage-5.0.1-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:348630edea485f4228233c2f310a598abf8afa5f8c716c02a9698089687b6085"}, - {file = "coverage-5.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:960d7f42277391e8b1c0b0ae427a214e1b31a1278de6b73f8807b20c2e913bba"}, - {file = "coverage-5.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0101888bd1592a20ccadae081ba10e8b204d20235d18d05c6f7d5e904a38fc10"}, - {file = "coverage-5.0.1-cp38-cp38m-win32.whl", hash = "sha256:c0fff2733f7c2950f58a4fd09b5db257b00c6fec57bf3f68c5bae004d804b407"}, - {file = "coverage-5.0.1-cp38-cp38m-win_amd64.whl", hash = "sha256:5f622f19abda4e934938e24f1d67599249abc201844933a6f01aaa8663094489"}, - {file = "coverage-5.0.1-cp39-cp39m-win32.whl", hash = "sha256:2714160a63da18aed9340c70ed514973971ee7e665e6b336917ff4cca81a25b1"}, - {file = "coverage-5.0.1-cp39-cp39m-win_amd64.whl", hash = "sha256:b7dbc5e8c39ea3ad3db22715f1b5401cd698a621218680c6daf42c2f9d36e205"}, - {file = "coverage-5.0.1.tar.gz", hash = "sha256:5ac71bba1e07eab403b082c4428f868c1c9e26a21041436b4905c4c3d4e49b08"}, -] entrypoints = [ {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, @@ -390,30 +290,6 @@ more-itertools = [ {file = "more-itertools-8.0.2.tar.gz", hash = "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d"}, {file = "more_itertools-8.0.2-py3-none-any.whl", hash = "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564"}, ] -mypy = [ - {file = "mypy-0.750-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:de9ec8dba773b78c49e7bec9a35c9b6fc5235682ad1fc2105752ae7c22f4b931"}, - {file = "mypy-0.750-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:3294821b5840d51a3cd7a2bb63b40fc3f901f6a3cfb3c6046570749c4c7ef279"}, - {file = "mypy-0.750-cp35-cp35m-win_amd64.whl", hash = "sha256:6992133c95a2847d309b4b0c899d7054adc60481df6f6b52bb7dee3d5fd157f7"}, - {file = "mypy-0.750-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:41696a7d912ce16fdc7c141d87e8db5144d4be664a0c699a2b417d393994b0c2"}, - {file = "mypy-0.750-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:c87ac7233c629f305602f563db07f5221950fe34fe30af072ac838fa85395f78"}, - {file = "mypy-0.750-cp36-cp36m-win_amd64.whl", hash = "sha256:83fa87f556e60782c0fc3df1b37b7b4a840314ba1ac27f3e1a1e10cb37c89c17"}, - {file = "mypy-0.750-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:30e123b24931f02c5d99307406658ac8f9cd6746f0d45a3dcac2fe5fbdd60939"}, - {file = "mypy-0.750-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:02d9bdd3398b636723ecb6c5cfe9773025a9ab7f34612c1cde5c7f2292e2d768"}, - {file = "mypy-0.750-cp37-cp37m-win_amd64.whl", hash = "sha256:088f758a50af31cf8b42688118077292370c90c89232c783ba7979f39ea16646"}, - {file = "mypy-0.750-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4f42675fa278f3913340bb8c3371d191319704437758d7c4a8440346c293ecb2"}, - {file = "mypy-0.750-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:f385a0accf353ca1bca4bbf473b9d83ed18d923fdb809d3a70a385da23e25b6a"}, - {file = "mypy-0.750-cp38-cp38-win_amd64.whl", hash = "sha256:54d205ccce6ed930a8a2ccf48404896d456e8b87812e491cb907a355b1a9c640"}, - {file = "mypy-0.750-py3-none-any.whl", hash = "sha256:28e9fbc96d13397a7ddb7fad7b14f373f91b5cff538e0772e77c270468df083c"}, - {file = "mypy-0.750.tar.gz", hash = "sha256:6ecbd0e8e371333027abca0922b0c2c632a5b4739a0c61ffbd0733391e39144c"}, -] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] -nose2 = [ - {file = "nose2-0.9.1-py2.py3-none-any.whl", hash = "sha256:31d8beb00aed3ccc6efb1742bb90227d883e471715188249f594310676e0ef0e"}, - {file = "nose2-0.9.1.tar.gz", hash = "sha256:0ede156fd7974fa40893edeca0b709f402c0ccacd7b81b22e76f73c116d1b999"}, -] packaging = [ {file = "packaging-20.0-py2.py3-none-any.whl", hash = "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb"}, {file = "packaging-20.0.tar.gz", hash = "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8"}, @@ -450,33 +326,6 @@ smmap2 = [ {file = "smmap2-2.0.5-py2.py3-none-any.whl", hash = "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde"}, {file = "smmap2-2.0.5.tar.gz", hash = "sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a"}, ] -typed-ast = [ - {file = "typed_ast-1.4.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e"}, - {file = "typed_ast-1.4.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b"}, - {file = "typed_ast-1.4.0-cp35-cp35m-win32.whl", hash = "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4"}, - {file = "typed_ast-1.4.0-cp35-cp35m-win_amd64.whl", hash = "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"}, - {file = "typed_ast-1.4.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631"}, - {file = "typed_ast-1.4.0-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233"}, - {file = "typed_ast-1.4.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1"}, - {file = "typed_ast-1.4.0-cp36-cp36m-win32.whl", hash = "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a"}, - {file = "typed_ast-1.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c"}, - {file = "typed_ast-1.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a"}, - {file = "typed_ast-1.4.0-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e"}, - {file = "typed_ast-1.4.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d"}, - {file = "typed_ast-1.4.0-cp37-cp37m-win32.whl", hash = "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36"}, - {file = "typed_ast-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0"}, - {file = "typed_ast-1.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66"}, - {file = "typed_ast-1.4.0-cp38-cp38-manylinux1_i686.whl", hash = "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2"}, - {file = "typed_ast-1.4.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47"}, - {file = "typed_ast-1.4.0-cp38-cp38-win32.whl", hash = "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161"}, - {file = "typed_ast-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e"}, - {file = "typed_ast-1.4.0.tar.gz", hash = "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34"}, -] -typing-extensions = [ - {file = "typing_extensions-3.7.4.1-py2-none-any.whl", hash = "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d"}, - {file = "typing_extensions-3.7.4.1-py3-none-any.whl", hash = "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575"}, - {file = "typing_extensions-3.7.4.1.tar.gz", hash = "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2"}, -] wcwidth = [ {file = "wcwidth-0.1.8-py2.py3-none-any.whl", hash = "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603"}, {file = "wcwidth-0.1.8.tar.gz", hash = "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8"}, diff --git a/pyproject.toml b/pyproject.toml index d98e39f..e9e1976 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,6 @@ langdetect = "^1.0.7" gitpython = "^3.0.5" [tool.poetry.dev-dependencies] -mypy = "^0.750" flake8 = "^3.7.9" pytest = "^5.3.2" From 7a6a8c67e049ae2e621ec8d00c067b2e0cc2b2b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 22:38:22 +0100 Subject: [PATCH 40/42] fixed wrong spdx --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e9e1976..d853e15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Finding potential software vulnerabilities from git commit messag authors = [ "Alexandre Dulaunoy " ] -license = "GPL-3.0-or-later" +license = "AGPL-3.0-or-later" readme = "README.md" From 865ee48d21f2ac7b1adaa04a95a123c1412c09d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 22:56:18 +0100 Subject: [PATCH 41/42] forgot the fixture --- tests/test_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_finder.py b/tests/test_finder.py index a4f36d0..4625ea2 100644 --- a/tests/test_finder.py +++ b/tests/test_finder.py @@ -3,7 +3,7 @@ from git_vuln_finder import find -def test_find_vuln(): +def test_find_vuln(clone_curl): all_potential_vulnerabilities, all_cve_found, found = find("./test_repos/curl/") #assert len(list(all_cve_found)) == 64 From e69c951e163f4a955344134068a614a6d0ffe422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Bonhomme?= Date: Mon, 6 Jan 2020 22:59:50 +0100 Subject: [PATCH 42/42] typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index befd4c7..e660ed4 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ ploit|malicious|directory traversal |\bRCE\b|\bdos\b|\bXSRF \b|\bXSS\b|clickjack ~~~ -#Running the tests +# Running the tests ~~~bash $ pytest