adulau/napkin-text-analysis
1
0
Fork 0
mirror of https://github.com/adulau/napkin-text-analysis.git synced 2024-11-24 10:57:07 +00:00
Code Issues Projects Releases Packages Wiki Activity

chg: [cleanup] key names used in redis has been simplified

Browse source
This commit is contained in:
Alexandre Dulaunoy 2020-10-09 07:18:16 +02:00
parent 10049a69b6
commit ef5011a64f
Signed by: adulau
GPG key ID: 09E2CD4944E6CBCD
1 changed files with 25 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
bin/napkin.py
View file

@ -52,64 +52,64 @@ with open(args.f, 'r') as file:
doc = nlp(text) doc = nlp(text)
analysis = ["verb:napkin", "noun:napkin", "hashtag:napkin", "mention:napkin", analysis = ["verb", "noun", "hashtag", "mention",
"digit:napkin", "url:napking", "oov:napkin", "labels:napkin", "digit", "url", "oov", "labels",
"punct:napkin"] "punct"]
redisdb.hset("stats", "token", doc.__len__()) redisdb.hset("stats", "token", doc.__len__())
for token in doc: for token in doc:
if token.pos_ == "VERB" and not token.is_oov: if token.pos_ == "VERB" and not token.is_oov:
if not args.verbatim: if not args.verbatim:
redisdb.zincrby("verb:napkin", 1, token.lemma_) redisdb.zincrby("verb", 1, token.lemma_)
else: else:
redisdb.zincrby("verb:napkin", 1, token.text) redisdb.zincrby("verb", 1, token.text)
redisdb.hincrby("stats", "verb:napkin", 1) redisdb.hincrby("stats", "verb", 1)
continue continue
if token.pos_ == "NOUN" and not token.is_oov: if token.pos_ == "NOUN" and not token.is_oov:
if not args.verbatim: if not args.verbatim:
redisdb.zincrby("noun:napkin", 1, token.lemma_) redisdb.zincrby("noun", 1, token.lemma_)
else: else:
redisdb.zincrby("noun:napkin", 1, token.text) redisdb.zincrby("noun", 1, token.text)
redisdb.hincrby("stats", "noun:napkin", 1) redisdb.hincrby("stats", "noun", 1)
continue continue
if token.is_oov: if token.is_oov:
value = "{}".format(token) value = "{}".format(token)
if value.startswith('#'): if value.startswith('#'):
redisdb.zincrby("hashtag:napkin", 1, value[1:]) redisdb.zincrby("hashtag", 1, value[1:])
redisdb.hincrby("stats", "hashtag:napkin", 1) redisdb.hincrby("stats", "hashtag", 1)
continue continue
if value.startswith('@'): if value.startswith('@'):
redisdb.zincrby("mention:napkin", 1, value[1:]) redisdb.zincrby("mention", 1, value[1:])
redisdb.hincrby("stats", "mention:napkin", 1) redisdb.hincrby("stats", "mention", 1)
continue continue
if token.is_digit: if token.is_digit:
redisdb.zincrby("digit:napkin", 1, value) redisdb.zincrby("digit", 1, value)
redisdb.hincrby("stats", "digit:napkin", 1) redisdb.hincrby("stats", "digit", 1)
continue continue
if token.is_space: if token.is_space:
redisdb.hincrby("stats", "space:napkin", 1) redisdb.hincrby("stats", "space", 1)
continue continue
if token.like_url: if token.like_url:
redisdb.zincrby("url:napkin", 1, value) redisdb.zincrby("url", 1, value)
redisdb.hincrby("stats", "url:napkin", 1) redisdb.hincrby("stats", "url", 1)
continue continue
if token.like_email: if token.like_email:
redisdb.zincrby("email:napkin", 1, value) redisdb.zincrby("email", 1, value)
redisdb.hincrby("stats", "email:napkin", 1) redisdb.hincrby("stats", "email", 1)
continue continue
if token.is_punct: if token.is_punct:
redisdb.zincrby("punct:napkin", 1, value) redisdb.zincrby("punct", 1, value)
redisdb.hincrby("stats", "punct:napkin", 1) redisdb.hincrby("stats", "punct", 1)
continue continue
redisdb.zincrby("oov:napkin", 1, value) redisdb.zincrby("oov", 1, value)
redisdb.hincrby("stats", "oov:napkin", 1) redisdb.hincrby("stats", "oov", 1)
for entity in doc.ents: for entity in doc.ents:
redisdb.zincrby("labels:napkin", 1, entity.label_) redisdb.zincrby("labels", 1, entity.label_)
if args.o == "json": if args.o == "json":
output_json = {"format":"napkin"} output_json = {"format":"napkin"}