mirror of
https://github.com/adulau/napkin-text-analysis.git
synced 2024-11-24 19:07:08 +00:00
chg: [cli] black the napkin binary
This commit is contained in:
parent
a2a074436e
commit
8541ae3192
1 changed files with 198 additions and 107 deletions
131
bin/napkin.py
131
bin/napkin.py
|
@ -15,20 +15,77 @@ version = "0.9"
|
||||||
parser = argparse.ArgumentParser(description="Extract statistical analysis of text")
|
parser = argparse.ArgumentParser(description="Extract statistical analysis of text")
|
||||||
parser.add_argument('-v', help="verbose output")
|
parser.add_argument('-v', help="verbose output")
|
||||||
parser.add_argument('-f', help="file to analyse")
|
parser.add_argument('-f', help="file to analyse")
|
||||||
parser.add_argument('-t', help="maximum value for the top list (default is 100) -1 is no limit", default=100)
|
parser.add_argument(
|
||||||
parser.add_argument('-s', help="display the overall statistics (default is False)", default=False, action='store_true')
|
'-t',
|
||||||
parser.add_argument('-o', help="output format (default is csv), json, readable", default="csv")
|
help="maximum value for the top list (default is 100) -1 is no limit",
|
||||||
parser.add_argument('-l', help="language used for the analysis (default is en)", default="en")
|
default=100,
|
||||||
parser.add_argument('-i', help="Use stdin instead of a filename", default=False, action='store_true')
|
)
|
||||||
parser.add_argument('--verbatim', help="Don't use the lemmatized form, use verbatim. (default is the lematized form)", default=False, action='store_true')
|
parser.add_argument(
|
||||||
parser.add_argument('--no-flushdb', help="Don't flush the redisdb, useful when you want to process multiple files and aggregate the results. (by default the redis database is flushed at each run)", default=False, action='store_true')
|
'-s',
|
||||||
parser.add_argument('--binary', help="set output in binary instead of UTF-8 (default)", default=False, action='store_true')
|
help="display the overall statistics (default is False)",
|
||||||
parser.add_argument('--analysis', help="Limit output to a specific analysis (verb, noun, hashtag, mention, digit, url, oov, labels, punct). (Default is all analysis are displayed)", default='all')
|
default=False,
|
||||||
parser.add_argument('--disable-parser', help="disable parser component in Spacy", default=False, action='store_true')
|
action='store_true',
|
||||||
parser.add_argument('--disable-tagger', help="disable tagger component in Spacy", default=False, action='store_true')
|
)
|
||||||
parser.add_argument('--token-span', default= None, help='Find the sentences where a specific token is located')
|
parser.add_argument(
|
||||||
parser.add_argument('--table-format', help="set tabulate format (default is fancy_grid)", default="fancy_grid")
|
'-o', help="output format (default is csv), json, readable", default="csv"
|
||||||
parser.add_argument('--full-labels', help="store each label value in a ranked set (default is False)", action='store_true', default=False)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'-l', help="language used for the analysis (default is en)", default="en"
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'-i', help="Use stdin instead of a filename", default=False, action='store_true'
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--verbatim',
|
||||||
|
help="Don't use the lemmatized form, use verbatim. (default is the lematized form)",
|
||||||
|
default=False,
|
||||||
|
action='store_true',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--no-flushdb',
|
||||||
|
help="Don't flush the redisdb, useful when you want to process multiple files and aggregate the results. (by default the redis database is flushed at each run)",
|
||||||
|
default=False,
|
||||||
|
action='store_true',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--binary',
|
||||||
|
help="set output in binary instead of UTF-8 (default)",
|
||||||
|
default=False,
|
||||||
|
action='store_true',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--analysis',
|
||||||
|
help="Limit output to a specific analysis (verb, noun, hashtag, mention, digit, url, oov, labels, punct). (Default is all analysis are displayed)",
|
||||||
|
default='all',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--disable-parser',
|
||||||
|
help="disable parser component in Spacy",
|
||||||
|
default=False,
|
||||||
|
action='store_true',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--disable-tagger',
|
||||||
|
help="disable tagger component in Spacy",
|
||||||
|
default=False,
|
||||||
|
action='store_true',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--token-span',
|
||||||
|
default=None,
|
||||||
|
help='Find the sentences where a specific token is located',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--table-format',
|
||||||
|
help="set tabulate format (default is fancy_grid)",
|
||||||
|
default="fancy_grid",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'--full-labels',
|
||||||
|
help="store each label value in a ranked set (default is False)",
|
||||||
|
action='store_true',
|
||||||
|
default=False,
|
||||||
|
)
|
||||||
# parser.add_argument('--geolocation', help="export geolocation (default is False)", action='store_true', default=False)
|
# parser.add_argument('--geolocation', help="export geolocation (default is False)", action='store_true', default=False)
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
@ -41,7 +98,9 @@ if args.f is None and not args.i:
|
||||||
# args.full_labels = True
|
# args.full_labels = True
|
||||||
|
|
||||||
if not args.binary:
|
if not args.binary:
|
||||||
redisdb = redis.Redis(host="localhost", port=6379, db=5, encoding='utf-8', decode_responses=True)
|
redisdb = redis.Redis(
|
||||||
|
host="localhost", port=6379, db=5, encoding='utf-8', decode_responses=True
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
redisdb = redis.Redis(host="localhost", port=6379, db=5)
|
redisdb = redis.Redis(host="localhost", port=6379, db=5)
|
||||||
|
|
||||||
|
@ -90,20 +149,52 @@ if args.i:
|
||||||
|
|
||||||
detect_lang = cld3.get_language(text)
|
detect_lang = cld3.get_language(text)
|
||||||
if detect_lang[0] != args.l:
|
if detect_lang[0] != args.l:
|
||||||
sys.exit("Language detected ({}) is different than the NLP used ({})".format(detect_lang[0], args.l))
|
sys.exit(
|
||||||
|
"Language detected ({}) is different than the NLP used ({})".format(
|
||||||
|
detect_lang[0], args.l
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
doc = nlp(text)
|
doc = nlp(text)
|
||||||
|
|
||||||
analysis = ["verb", "noun", "hashtag", "mention",
|
analysis = [
|
||||||
"digit", "url", "oov", "labels",
|
"verb",
|
||||||
"punct", "email"]
|
"noun",
|
||||||
|
"hashtag",
|
||||||
|
"mention",
|
||||||
|
"digit",
|
||||||
|
"url",
|
||||||
|
"oov",
|
||||||
|
"labels",
|
||||||
|
"punct",
|
||||||
|
"email",
|
||||||
|
]
|
||||||
|
|
||||||
if args.token_span and not disable:
|
if args.token_span and not disable:
|
||||||
analysis.append("span")
|
analysis.append("span")
|
||||||
|
|
||||||
redisdb.hset("stats", "token", doc.__len__())
|
redisdb.hset("stats", "token", doc.__len__())
|
||||||
|
|
||||||
labels = [ "EVENT", "PERCENT", "MONEY", "FAC", "TIME", "QUANTITY", "WORK_OF_ART", "LANGUAGE", "PRODUCT", "LOC", "LAW", "DATE", "ORDINAL", "NORP", "ORG", "CARDINAL", "GPE", "PERSON"]
|
labels = [
|
||||||
|
"EVENT",
|
||||||
|
"PERCENT",
|
||||||
|
"MONEY",
|
||||||
|
"FAC",
|
||||||
|
"TIME",
|
||||||
|
"QUANTITY",
|
||||||
|
"WORK_OF_ART",
|
||||||
|
"LANGUAGE",
|
||||||
|
"PRODUCT",
|
||||||
|
"LOC",
|
||||||
|
"LAW",
|
||||||
|
"DATE",
|
||||||
|
"ORDINAL",
|
||||||
|
"NORP",
|
||||||
|
"ORG",
|
||||||
|
"CARDINAL",
|
||||||
|
"GPE",
|
||||||
|
"PERSON",
|
||||||
|
]
|
||||||
|
|
||||||
for entity in doc.ents:
|
for entity in doc.ents:
|
||||||
redisdb.zincrby("labels", 1, entity.label_)
|
redisdb.zincrby("labels", 1, entity.label_)
|
||||||
|
|
Loading…
Reference in a new issue