evaluate.py 675 B

12345678910111213141516171819202122232425262728
  1. from perplexity import *
  2. from itertools import zip_longest
  3. from typing import List
  4. from nltk.tokenize import sent_tokenize
  5. def evaluate(text):
  6. print("Evaluating the text given...")
  7. paragraphs = grouper(sent_tokenize(text), 5)
  8. results = []
  9. print(len(paragraphs))
  10. for i in range(len(paragraphs)):
  11. txt = ""
  12. for j in range(len(paragraphs[i])):
  13. txt += str(j)
  14. get_perplexity(txt)
  15. results.append(return_msg())
  16. print(results[i])
  17. return return_msg()
  18. def grouper(iterable: List, n: int, fillvalue=None) -> List:
  19. args = [iter(iterable)] * n
  20. return list(zip_longest(*args, fillvalue=fillvalue))