From 3e1b5f0f3c4016fb846ec0fa5ea0cd5f6fb0ace1 Mon Sep 17 00:00:00 2001 From: Alex Shan Date: Tue, 7 Nov 2023 19:59:07 -0800 Subject: [PATCH] Make updates to model evaluation, and add data processing class for extracting sentences of interest --- .../lemma_classifier/evaluate_models.py | 77 ++- .../lemma_classifier/prepare_dataset.py | 94 ++++ .../models/lemma_classifier/test_output.txt | 464 ++++++++++++++++++ stanza/models/lemma_classifier/utils.py | 7 + 4 files changed, 627 insertions(+), 15 deletions(-) create mode 100644 stanza/models/lemma_classifier/prepare_dataset.py create mode 100644 stanza/models/lemma_classifier/test_output.txt create mode 100644 stanza/models/lemma_classifier/utils.py diff --git a/stanza/models/lemma_classifier/evaluate_models.py b/stanza/models/lemma_classifier/evaluate_models.py index 63be08b4f0..bb1cc184fa 100644 --- a/stanza/models/lemma_classifier/evaluate_models.py +++ b/stanza/models/lemma_classifier/evaluate_models.py @@ -10,34 +10,68 @@ import stanza from typing import Any, List, Tuple from models.lemma_classifier.baseline_model import BaselineModel +import utils -def load_doc_from_conll_file(path: str): - return stanza.utils.conll.CoNLL.conll2doc(path) +def update_counts(gold_tag: str, pred_tag: str, true_pos: int, false_pos: int, false_neg: int) -> Tuple[int, int, int]: + """" + Takes in a prediction along with the counts for true positive, false positive and false negative and updates the counts + of the measurements according to the prediction. + + We measure positives, where we treat "be" as a positive and "have" as a negative. + """ + if gold_tag == "be" and pred_tag == "be": + true_pos += 1 + elif gold_tag == "be" and pred_tag == "have": + false_neg += 1 + elif gold_tag == "have" and pred_tag == "be": + false_pos += 1 + return true_pos, false_pos, false_neg def evaluate_models(eval_path: str, binary_classifier: Any, baseline_classifier: BaselineModel): """ Evaluates both the binary classifier and baseline classifier on a test file, checking the predicted lemmas for each "'s" token against the gold lemma. + + TODO: Measure precision, recall, and F1. + + Precision = true positives / true positives + false positives + Recall = true positives / true positives + false negatives + F1 = 2 * (Precision * Recall) / (Precision + Recall) """ + gold_doc = utils.load_doc_from_conll_file(eval_path) - gold_doc = load_doc_from_conll_file(eval_path) - for sentence in doc.sentences: + bin_tp, bin_fp, bin_fn = 0, 0, 0 + bl_tp, bl_fp, bl_fn = 0, 0, 0 # baseline counts + + for sentence in gold_doc.sentences: for word in sentence.words: - if word.text == "'s" and word.upos in ["VERB", "AUX"]: + if word.text == "'s" and word.upos == "AUX": # only evaluate when the UPOS tag is AUX gold_tag = word.lemma # predict binary classifier bin_predict = None # TODO # predict baseline classifier baseline_predict = baseline_classifier.predict(word.text) # TODO - # score - if gold_tag == bin_predict: - pass - if gold_tag == baseline_predict: - pass + + # score binary classifier + bin_tp, bin_fp, bin_fn = update_counts(gold_tag, bin_predict, bin_tp, bin_fp, bin_fn) + bl_tp, bl_fp, bl_fn = update_counts(gold_tag, baseline_predict, bl_tp, bl_fp, bl_fn) - return # TODO write some kind of evaluation + # compute precision, recall, f1 + bin_precision, bin_recall = bin_tp / (bin_tp + bin_fp), bin_tp / (bin_tp + bin_fn) + bin_results = {"precision": bin_precision, + "recall": bin_recall, + "f1": 2 * (bin_precision * bin_recall) / (bin_precision + bin_recall) + } + + bl_precision, bl_recall = bl_tp / (bl_tp + bl_fp), bl_tp / (bl_tp + bl_fn) + bl_results = {"precision": bl_precision, + "recall": bl_recall, + "f1": 2 * (bl_precision * bl_recall) / (bl_precision + bl_recall) + } + + return bin_results, bl_results def main(): @@ -45,16 +79,29 @@ def main(): Runs a test on the EN_GUM test set """ coNLL_path = os.path.join(os.path.dirname(__file__), "en_gum-ud-train.conllu") - doc = load_doc_from_conll_file(coNLL_path) + print(f"Attempting to find token 's in file {coNLL_path}...") + doc = utils.load_doc_from_conll_file(coNLL_path) count = 0 + be_count, have_count = 0, 0 for sentence in doc.sentences: for word in sentence.words: - if word.text == "'s" and word.upos in ["VERB", "AUX"]: - print("Found") + if word.text == "'s" and word.upos == "AUX": + print("---------------------------") print(word) + print("---------------------------") + if word.lemma == "have": + have_count += 1 + if word.lemma == "be": + be_count += 1 count += 1 - print(f"Count was {count}.") + print(f"The number of 's found was {count}.") + print(f"There were {have_count} occurrences of the lemma being 'have'.") + print(f"There were {be_count} occurrences of the lemma being 'be'.") + + # bl_model = BaselineModel("'s", "be") + # bin_results, bl_results = evaluate_models(coNLL_path, None, bl_model) + # print(bin_results, bl_results) if __name__ == "__main__": diff --git a/stanza/models/lemma_classifier/prepare_dataset.py b/stanza/models/lemma_classifier/prepare_dataset.py new file mode 100644 index 0000000000..32d0f48a6d --- /dev/null +++ b/stanza/models/lemma_classifier/prepare_dataset.py @@ -0,0 +1,94 @@ +import stanza +import utils +import os +from typing import List, Tuple, Any + +""" +The code in this file processes a CoNLL dataset by taking its sentences and filtering out all sentences that do not contain the target token. +Furthermore, it will store tuples of the Stanza document object, the position index of the target token, and its lemma. +""" + + +class DataProcessor(): + + def __init__(self, target_word: str, target_upos: List[str]): + self.target_word = target_word + self.target_upos = target_upos + + def find_all_occurrences(self, sentence) -> List[int]: + """ + Finds all occurrences of self.target_word in tokens and returns the index(es) of such occurrences. + """ + occurrences = [] + for idx, token in enumerate(sentence.words): + if token.text == self.target_word and token.upos in self.target_upos: + occurrences.append(idx) + return occurrences + + def process_document(self, doc, keep_condition: callable, save_name: str) -> None: + """ + Takes any sentence from `doc` that meets the condition of `keep_condition` and writes its tokens, index of target word, and lemma to `save_name` + + Sentences that meet `keep_condition` and contain `self.target_word` multiple times have each instance in a different example in the output file. + + Args: + doc (Stanza.doc): Document object that represents the file to be analyzed + keep_condition (callable): A function that outputs a boolean representing whether to analyze (True) or not analyze the sentence for a target word. + save_name (str): Path to the file for storing output + """ + if os.path.exists(save_name): + raise ValueError(f"Output path {save_name} already exists. Aborting...") + with open(save_name, "w+", encoding="utf-8") as output_f: + for sentence in doc.sentences: + # for each sentence, we need to determine if it should be added to the output file. + # if the sentence fulfills the keep_condition, then we will save it along with the target word's index and its corresponding lemma + if keep_condition(sentence): + tokens = [token.text for token in sentence.words] + indexes = self.find_all_occurrences(sentence) + for idx in indexes: + # for each example found, we write the tokens along with the target word index and lemma + output_f.write(f'{" ".join(tokens)} {idx} {sentence.words[idx].lemma}\n') + + def read_processed_data(self, file_name: str) -> List[dict]: + """ + Reads the output file from `process_document()` and outputs a list that contains the sentences of interest. Each object within the list + contains a map with three (key, val) pairs: + + "words" is a list that contains the tokens of the sentence + "index" is an integer representing which token in "words" the lemma annotation corresponds to + "lemma" is a string that is the lemma of the target word in the sentence. + + """ + output = [] + with open(file_name, "r", encoding="utf-8") as f: + for line in f.readlines(): + obj = {} + split = line.split() + words, index, lemma = split[:-2], int(split[-2]), split[-1] + + obj["words"] = words + obj["index"] = index + obj["lemma"] = lemma + + output.append(obj) + + return output + + +if __name__ == "__main__": + + coNLL_path = os.path.join(os.path.dirname(__file__), "en_gum-ud-train.conllu") + doc = utils.load_doc_from_conll_file(coNLL_path) + + processor = DataProcessor(target_word="'s", target_upos=["AUX"]) + output_path = os.path.join(os.path.dirname(__file__), "test_output.txt") + + def keep_sentence(sentence): + for word in sentence.words: + if word.text == "'s" and word.upos == "AUX": + return True + return False + + processor.process_document(doc, keep_sentence, output_path) + + print(processor.read_processed_data(output_path)) diff --git a/stanza/models/lemma_classifier/test_output.txt b/stanza/models/lemma_classifier/test_output.txt new file mode 100644 index 0000000000..17c822b6f5 --- /dev/null +++ b/stanza/models/lemma_classifier/test_output.txt @@ -0,0 +1,464 @@ +He 's garnered praise for his performance as Charles Ryder in Julian Jarrold ’s adaptation of Evelyn Waugh 's Brideshead Revisited ( 2008 ) , and as Ozymandias in the American neo - noir superhero film Watchmen ( 2009 ) , based on DC Comics ' limited series of the same name . 1 be +The temperature of this w- water here , is sixty - four degrees , the temperature of this steam , is , I know it 's hotter than that . 25 be +Yeah , it 's about a hundred and eleven degrees . 3 be +Oh , and by the way guys , what do we call something that 's hard , like this ice here , or a table , or a rock ? 14 be +And finally , what do we call something that 's loose , and floating around like this steam here ? 9 be +Well folks , now I 'm going to show you something else that happens when you change the temperature , and that 's where these balloons come in . 22 be +It 's four . 1 be +That 's another thing too , is I kinda had a b- general idea , of kinda how to do it , just watching him . 1 be +Then , down there , um , it 's mandatory . 8 be +I mean , it — that 's just kinda how it happens , you know , because he kinda has to tell you ... 6 be +And that 's kinda where you go by , to — 2 be +And that 's where you kinda — kinda need a little guide , of where you trim . 2 be +That 's not bad , but sometimes you can get it really bad . 1 be +A lot of times , like , I 'll get done , and I 'll think I 'm done , and I 'll look at – look down at the horse 's hoof , and it 's still , it 's too long . 36 be +A lot of times , like , I 'll get done , and I 'll think I 'm done , and I 'll look at – look down at the horse 's hoof , and it 's still , it 's too long . 40 be +And that 's as far as we got . 2 be +You know , I mean , you get into the big horses , the — like the Clydesdales , Shires , that 's a d- — a whole different thing . 22 be +Or ponies , that 's a whole different thing . 4 be +That 's to Judy . 1 be +He 's wonderful , hunh . 1 be +I mean , he – he 's dead now . 6 be +That 's — on the way . 1 be +Look , it 's a bread baking pan . 3 be +Now , That 's from Mrs. Santa . 3 be +She 's been saying for months that you would never wear it . 1 be +But , I thought that 's the largest size they have . 5 be +That 's big enough . 1 be +That 's great . 1 be +That 's a great idea . 1 be +That one 's to Dad . 2 be +If it 's too small , Mom can take it . 2 be +No , it 's not too small . 3 be +It 's perfect . 1 be +Oh here 's one you can open . 2 be +Well , who 's this from ? 3 be +Who 's Mitchell Roberts ? 1 be +Who 's the party being s- served ? 1 be +Okay , let 's see , who 's here now . 7 be +That 's correct . 1 be +That 's correct . 1 be +So I said that 's not as per our agreement , he said yes it is , he handed me my check , rolled up his window , and drove off . 4 be +And he 's there to state , he 's — he saw that , and was witness to that . 2 be +I do n- I do n't think it 's relevant that he rolled up his window and drove off . 8 be +That 's correct . 1 be +So uh your uncle then wrote uh Builders uh a letter , that 's what I 'm reading now . 13 be +That 's correct , that 's correct . 1 be +That 's correct , that 's correct . 5 be +That 's it sir . 1 be +That 's the reason why I had him fill out that portion of , or he wanted to write those checks , was so I would save that money , so I would do that job . 1 be +That 's anything above that . 1 be +This is in the Luther book some of you are reading , and um , if you , this evening when you go home , if you wan na meditate it on more , on – on it more , it 's on page three eleven . 41 be +It 's on page three eleven . 1 be +Is based uh , alright , what is it , it 's based , as it is , on the concept , that human nature is totally depraved , and of itself not capable of any good will . 11 be +Well that 's as clear as you can get it . 2 be +And as you come — as Erasmus comes to this , he 's going to say , well let 's look at the Scriptures . 12 be +I do n't think Kathy can have kids , she 's got a bad back . 10 have +She 's all , heavy . 1 be +I mean , it 's not impossible . 4 be +Tend to believe it 's not as bad as sh- th- – 4 be +What 's Bill doing now ? 1 be +I do n't know what he 's doing today . 6 be +That 's just part of the deal . 1 be +Well I know , that 's what starving is . 5 be +Cause that 's what she does with other people . 2 be +Actually it 's more of a dinner , because it 's – 2 be +N- that 's alright . 2 be +No , it 's okay . 3 be +I usually d- have , yeah , that 's good . 8 be +She 's got a sore throat . 1 have +Now it 's gotten back to eight , hunh ? 2 be +That said that , that 's not potty training , that 's mother training . 5 be +That said that , that 's not potty training , that 's mother training . 11 be +She 's trying . 1 be +It 's Mister Spock , and Cap- 1 be +Oh sh- like oh , he 's gon na go back again in other words . 6 be +So then I said , " but look at Barb , she 's healthy now " , and Carolyn looked at her , and sort of looked away , and I thought , oh boy . 12 be +He 's teething . 1 be +That 's scary . 1 be +Maybe it 's good he did n't stay with me . 2 be +He 's lucky that he did n't – 1 be +That 's dangerou- – I mean s- – 1 be +And that 's where it 's fatal . 2 be +And that 's where it 's fatal . 5 be +Well that 's what the – I mean that 's what – 2 be +Well that 's what the – I mean that 's what – 9 be +No I 'm pretty – I 'm pretty sure , I 'm pretty sure that 's part of it , but uh it 's just that you have to watch out , cause then , if it gets too hot it 's bad for the brain . 15 be +No I 'm pretty – I 'm pretty sure , I 'm pretty sure that 's part of it , but uh it 's just that you have to watch out , cause then , if it gets too hot it 's bad for the brain . 23 be +No I 'm pretty – I 'm pretty sure , I 'm pretty sure that 's part of it , but uh it 's just that you have to watch out , cause then , if it gets too hot it 's bad for the brain . 41 be +It 's like a car , it overheats . 1 be +Who 's all the babinos ? 1 be +That 's the one that 's ten days older than – 1 be +That 's the one that 's ten days older than – 5 be +That 's Michael 's – those are Michael 's nephews – 1 be +That 's Jose – 1 be +And then Gino 's the one in the middle . 3 be +That 's his sister . 1 be +Aw he 's a bit older . 2 be +Yeah he 's three . 2 be +He 's o- he 's three . 1 be +He 's o- he 's three . 4 be +No , he 's okay then . 3 be +He 's a gestalt child . 1 be +It 's just two different ways . 1 be +It 's all my fault . 1 be +So it 's just a d- like a different style . 2 be +It 's funny cause , h- he 'll be like , he 'll get up in the morning , he 'll sit in front of the TV with his bottle in one hand , just watching TV , like half asleep , you know , he 's a – 1 be +It 's funny cause , h- he 'll be like , he 'll get up in the morning , he 'll sit in front of the TV with his bottle in one hand , just watching TV , like half asleep , you know , he 's a – 46 be +That 's low , Issac – 1 be +That 's low . 1 be +It 's true . 1 be +And the way it 's marketed , and the way we 're d- — we develop needs for it . 4 be +And it 's just like rubble , f- like from , just below the knees , down , was just kind of rubble . 2 be +That 's true . 1 be +That 's true . 1 be +Cause , who 's gon na be able to hold , all that knowledge , and make the connections ? 3 be +It 's like both sides . 1 be +Who d- is n't necessarily inclined to deal with the details that much , who 's guiding , those who are willing to sit there . 15 be +I guess that 's the problem . 3 be +That 's right . 1 be +And — and uh , you know , I think , I think that , one of the reasons that there 's been no focus is because , oh , everybody accepted the scientific method as the best tool that we have , and was kinda letting the scientific method be the — be the leader . 21 have +Cause he had down twenty - five plus eighteen , and what can you do , when he 's already got it down ? 18 have +Well , I 'll call him probably late this afternoon , to make sure he 's alright . 15 be +But that 's better than pins , and surgery . 2 be +Yeah , it 's more for a surgery . 3 be +But , still , it 's more expensive . 5 be +I think it 'll do fine with a cast , especially since it 's a spiral fracture . 13 be +And it 's not displaced . 2 be +And it 's not displaced . 2 be +Well , as long as she 's — as long as she 's quiet . 12 be +I do n't know if he 's in the barn , from home . 6 be +It 's Marcia . 1 be +Um , it looks like she 's got a spiral fracture , of um , of her tibia , which is the bone between the knee and the hock , and , um , it 's f- it 's really looks , pretty stable . 6 be +Um , it looks like she 's got a spiral fracture , of um , of her tibia , which is the bone between the knee and the hock , and , um , it 's f- it 's really looks , pretty stable . 35 be +Um , it looks like she 's got a spiral fracture , of um , of her tibia , which is the bone between the knee and the hock , and , um , it 's f- it 's really looks , pretty stable . 38 be +It 's not displaced very far anyway , and it 's um , pretty much , you know , locked in place , so to speak . 1 be +It 's not displaced very far anyway , and it 's um , pretty much , you know , locked in place , so to speak . 10 be +It 's gon na be about a hundred and fifty - three , total . 1 be +And , the only , then it would be , the only extra would be , in about six to eight weeks , we 'd have her come in again , we would tranquilize her , and take the X - rays , and see if it 's healed , enough to take off . 47 be +And that 's generally about thirty - four to forty , somewhere in there . 2 be +You know , a pin , you know , if we did a pin , it 's probably , it would probably be even a hundred more but , so it 's it 's that 's about the — at this point the , the least expensive alternative that probably will work for her . 35 be +She 's doing the Karate Kid , Nathan . 1 be +She 's not even looking at me . 1 be +She 's just looking , like — 1 be +That 's what I 'm talking about . 1 be +That 's what I was try — That 's what I was trying to say . 1 be +That 's what I was try — That 's what I was trying to say . 8 be +That 's what it was . 1 be +Well , then that 's fine . 4 be +Cause that 's the same way you 're multiplying there , square root of nine , that — and square root of nine equals three . 2 be +Where 's the test ? 1 be +And that 's easy , you can do that . 2 be +Well , I mean , that 's just wasting time . 6 be +Cause if it 's not . 3 be +Tha- – that 's right . 3 be +And that 's one ? 2 be +Is that all that 's left ? 4 be +It 's not real . 1 be +It 's just a dream . " 1 be +Still , when your best friend 's in trouble you help out , right ? 6 be +It 's like , your mom and dad murdered you — " 1 be +My mother 's more particular -- 2 be +" What 's your name ? " Carroll said . 2 be +" My father , he 's an anthropologist , " she said . 5 be +It 's very soft stone that you can easily carve . 1 be +This means the experimenter does n't know which ants belong to which treatment , so it 's impossible to influence the results with ' observer bias ' . 16 be +That 's as of now . 1 be +I suspect that 's not true . " 3 be +Santorum criticized the response of the press to the phenomenon in a 2011 radio interview , saying , " It 's offensive beyond , you know , anything that any public figure or anybody in America should tolerate , and the mainstream media laughs about it . " 20 be +It 's been around forever . 1 have +It 's not for us to take a stand on any political issues . 1 be +The only way to get at what 's different is to make comparisons with close relatives . 7 be +In terms of their glow , I suspect that it 's not theirs - although luminescence is common in anemone relatives , they do n't usually make light themselves . 10 be +( that 's how I got the pictures of the muscles etc in the paper ) . 2 be +I am curious about the population structure of the " fields " of anemones - the group to which Edwardsiella andrillae belongs includes many species that reproduce asexually , and it 's possible that the fields are " clones " produced asexually rather than the result of sexual reproduction . 31 be +It 's portrayed as though it 's dark , black and evil . 1 be +It 's portrayed as though it 's dark , black and evil . 6 be +What 's up ? 1 be +It 's anything but that . 1 be +What 's your greatest non-fixed cost ? 1 be +That 's about it . 1 be +Who 's buying the paper , who 's selling the paper . 1 be +Who 's buying the paper , who 's selling the paper . 7 be +Because I want someone who 's going to treat people nicely and well . 5 be +And he 's been right . 2 have +And I have made him hire these girls , and ultimately he 's been right 12 have +Could I have a sample , not necessarily one that 's filled out , but just one that you -- 10 be +And it 's an amalgamation of questions . 2 be +What 's your favorite question ? 1 be +No , because I really ca n't call and tell them I 'm yeah , so - and - so 's applying here for a position . 20 be +I think that a vibrant public domain is very important to a healthy world , and so I thought : here 's a way to help the cause . 21 be +That 's good and well and fine with us . 1 be +It 's quite a large Canadian tournament , and so we went as the Gliders team . 1 be +Cause that 's one of our problems really , to compete . 2 be +It 's not the same . 1 be +No , it 's really not , so it 's really important to be able to get as a many international trips throughout the year to continue our improvement . 3 be +No , it 's really not , so it 's really important to be able to get as a many international trips throughout the year to continue our improvement . 9 be +It 's really strange occurrence , over the years . 1 be +So the World Cup is ... it 's good to be able to do well at the World Cup , to be placed , but it also means that you get a really good opportunity to know where you 're at in that two year gap between the Paralympics . 7 be +So you can come back home and revisit what you need to do and , you know , where the team 's at . 21 be +It 's a funny phenomenon . 1 be +It 's ironic . 1 be +Do you know what it 's like to be chased by the Ghost of Failure while staring through Victory 's door ? 5 be +Cheer the team today ; ( your boys in orange and blue ) Let them hear you shout as they fight for what 's mightily due 23 be +That 's all it said . 1 be +It 's cool ... it 's like another family . 1 be +It 's cool ... it 's like another family . 5 be +It 's opened up a new fan base , so to speak . 1 be +There 's a lot of unfunny jokes that are told in a back room , that 's why they stay and die in that back room and do n’t go out in The Onion . 16 be +It 's very beautiful I think . 1 be +It 's like from the sky . 1 be +It 's almost like God to some extent . 1 be +I think it 's a combination of the beauty , of the imagination , of the freedom . 3 be +In the staccato part it 's like a music : a lot of break , very very short sentences . 5 be +It 's a book of opening , Overtures I called it , and there is no end to the stories . 1 be +It 's a little bit like Achilles and the turtle . 1 be +Here 's what we know . 1 be +And that 's on the main Stardust @ home website [ see below ] ? 2 be +It 's what makes America great . " 1 be +Rathbun asserted that Scientology officials , " ... plied the Sheriff 's detective with a grand conspiracy theory , characterizing me as the ' anti-christ ' of the church of Scientology . ... The next thing Daniel knows he 's in a cell where he sat for the next 30 hours . " 39 be +" So far , it 's the Internet : 1 , Scientology : 0 . But it 's a long game . " - Matthew Ingram 5 be +" So far , it 's the Internet : 1 , Scientology : 0 . But it 's a long game . " - Matthew Ingram 17 be +" He 's hanging out with the celebrities , and has kind of become the same sort of celebrity he was interested in documenting earlier in his career " , Wilkins said . 2 be +" That 's why he did the Campbell Soup cans or the Marilyn pictures , these iconic products of American culture whether they be in film , video or actually products we consumed . 2 be +" I think everybody knows Andy Warhol 's name , even non-art people , that 's a name they might know because he was such a personality " , Water said . 15 be +" It 's Warhol . 2 be +If it 's 200 nodes strong , they will try to poach the leader to the government . 2 be +If the environmental agency group reports one number and your neighbors report another number , of course , you 're going to trust the one that you participated yourself , even though it 's a lower quality sensor . 33 be +Because of that , it 's very rare in Taiwan because we have an expanding civic space . 5 be +It 's impossible to stay there forever . 1 be +That 's electricity generators , power plants , over there . 1 be +The beauty is this , it 's all open hardware . 6 be +It 's all open source . 1 be +It 's on GitHub . 1 be +That 's a long time . 1 be +It 's a long time , as Congresswoman Dean said , calling out for his mother , extending love to his family . 1 be +That 's really nice . 1 be +It 's great to be at Trump Tower . 1 be +It 's great to be in a wonderful city , New York . 1 be +And it 's an honor to have everybody here . 2 be +There 's been no crowd like this . 1 have +I do n't think it 's gon na happen . 5 be +It 's true , and these are the best and the finest . 1 be +It 's coming from more than Mexico . 1 be +It 's coming from all over South and Latin America , and it 's coming probably — probably — from the Middle East . 1 be +It 's coming from all over South and Latin America , and it 's coming probably — probably — from the Middle East . 13 be +Because we have no protection and we have no competence , we do n't know what 's happening . 16 be +And it 's got to stop and it 's got to stop fast . 2 have +And it 's got to stop and it 's got to stop fast . 8 have +It 's never below zero . 1 be +That 's right . 1 be +But the real number , the real number is anywhere from 18 to 19 and maybe even 21 percent , and nobody talks about it , because it 's a statistic that 's full of nonsense . 28 be +But the real number , the real number is anywhere from 18 to 19 and maybe even 21 percent , and nobody talks about it , because it 's a statistic that 's full of nonsense . 32 be +I just feel awkward being like " what 's up guys , " like , I do n't know when my neighbors are out in their backyard . 8 be +Sorry , hi , it 's me now . 5 be +I think it 's hormonal . 3 be +So I do n't really know what 's going on . 7 be +I think it 's hormonal , but anyways , this is just real life and I 've really been struggling with my appearance lately . 3 be +It 's been a really sore spot for me . 1 have +And I also think I put way too much emphasis on what I look like , and the fact that I 'm putting this on the internet and that I , like , get like this for the internet , is huge for me , it 's always been huge for me , but also it 's really hard when I 'm editing . 46 have +And I also think I put way too much emphasis on what I look like , and the fact that I 'm putting this on the internet and that I , like , get like this for the internet , is huge for me , it 's always been huge for me , but also it 's really hard when I 'm editing . 56 be +I have a really hard time , like just grappling with my appearance at times , and that 's a really toxic thing for your mental health . 18 be +And I know that 's not the case and ugly is such a subjective word . 4 be +And also I feel like it 's a word meant to like , oppress people , particularly women , but I 've just been having a hard time with the way I look . 6 be +So it 's just , it 's been rough . 6 have +And I think I 'm zooming out right now in my head and realizing that it 's been two , almost two years of this shit , and that I obviously need to be more gentle with myself and I think I 'm in the process of doing that , and this week , particularly I 've been moving my body a little bit more and it 's been feeling really good . 16 have +And I think I 'm zooming out right now in my head and realizing that it 's been two , almost two years of this shit , and that I obviously need to be more gentle with myself and I think I 'm in the process of doing that , and this week , particularly I 've been moving my body a little bit more and it 's been feeling really good . 67 have +Your girl 's got trauma like , it 's way past due . 2 have +Your girl 's got trauma like , it 's way past due . 8 be +It 's like a lot of bravery involved to do something like this and to kind of want to better yourself and to take these steps for you and your mental health . 1 be +And it probably does n't seem that way , by the way I feel myself , but it 's always just something that 's in here , do you know what I mean ? 18 be +And it probably does n't seem that way , by the way I feel myself , but it 's always just something that 's in here , do you know what I mean ? 23 be +Conversations for first thing in the morning what the heck , it 's not even 11:00 a.m. , but hi , just coming right out of the gate , just with what 's fresh on my mind . 12 be +Conversations for first thing in the morning what the heck , it 's not even 11:00 a.m. , but hi , just coming right out of the gate , just with what 's fresh on my mind . 32 be +Um today 's Wednesday , it 's quite gloomy out . 2 be +Um today 's Wednesday , it 's quite gloomy out . 6 be +It 's like really thick and stretchy and comfy and soft . 1 be +I think it 'll be good for me and it 's gon na be relaxing . 10 be +She 's nice . 1 be +It 's cheaper ! 1 be +That 's just like a local cafe . 1 be +But that 's okay . 2 be +Anyways now it 's like 5:19 p.m. 3 be +Yes it 's a I provide a service . 2 be +That 's not what I — that 's not what I said . 1 be +That 's not what I — that 's not what I said . 7 be +That 's what you implied . 1 be +It 's delicious . 1 be +Hey , what 's up everyone ? 3 be +And I remember at one point I checked my temperature 's only like 98.7 , but at another point it was like 101 or something really high that was obviously a fever . 10 be +That 's what the feeling was . 1 be +I guess that 's like what happens when you lose your taste buds , I do n't know , olfactory nerves , I do n't know . 3 be +First of all , that 's like super inaccurate . 5 be +So , you can kind of distinguish if it 's like sweet or salty or tart um , but you ca n't really taste it to its fullest if that makes any sense . 9 be +I do n't really know exactly how it tastes because I can only like I can kind of taste the cheese , um the saltiness , but it 's more so um I 'm able to feel the textures . 28 be +She 's not here yet . 1 be +It 's a new day , a little rainy outside . 1 be +But I actually like when it 's gloomy for studying because then I feel like there 's nothing else you could be doing , so you might as well just stay inside and work . 6 be +It 's currently 2:25 a.m. on Wednesday . 1 be +It 's raining . 1 be +A lot of the notes that I had taken from our last meeting and being more of a cause or like leading to memory 's importance for each of the sources , I — I essentially changed it to um " it 's the one persistent element of our existence that gives meaning to the inheritance of memory " . 42 be +I 'll try my best and am just gon na take the test , move on , it 's just one more test . 18 be +That 's lovely . 1 be +She also surprised me on my birthday , so now it 's my turn . 11 be +Okay , it 's a 16 minute walk . 3 be +Well , that 's exciting ! 3 be +Not at all , I just think it 's really nice outside . 8 be +She 's turning 20 ! 1 be +That one 's the vegan one ? 2 be +Alright , it 's recording so you can start now ! 3 be +So it 's Shawntas Way here today , and today I 'm going to be talking to you guys about number one : my new product line . 2 be +Um , it 's not like super cold w- where I am just yet , but it 's getting there , so ignore my hair right now . 3 be +Um , it 's not like super cold w- where I am just yet , but it 's getting there , so ignore my hair right now . 17 be +So it 's the Angelic Coco Curl Creator Cream , and it has coconut plus rose in it . 2 be +It 's so moisturizing . 1 be +It 's really really thick . 1 be +You guys can see on the camera it 's super thick . 8 be +It 's a cream , but it has a really buttery feel to it . 1 be +And it 's of course white , hence the name " Angelic Coco " . 2 be +As you guys know , that 's all I talk about on my channel most of the time . 6 be +It 's very natural , and it does n't have a strong scent . 1 be +I 'm just so excited to finally , like , give it to you guys and know that it 's gon na contribute to the actual health of your hair . 19 be +I have a gel as well , so it 's not just a cream . 9 be +So the next product is the Angelic Clear Coarse Jelly with aloe plus rosemary , and it 's a curl - defining and light - to - medium whole jelly , and it 's enriched with aloe vera juice , which is amazing . 17 be +So the next product is the Angelic Clear Coarse Jelly with aloe plus rosemary , and it 's a curl - defining and light - to - medium whole jelly , and it 's enriched with aloe vera juice , which is amazing . 33 be +But a lot of times we kinda , like , deal with it because , you know , the gel is what 's gon na keep our hair in place and make it actually last . 22 be +So we just slept in today because it 's our last day here in Glacier and we 've been getting up at five o'clock every morning so that we could get into the park and get hiking early . 8 be +That 's how the paths are , so I have to clean them every day um but yeah . 1 be +So that 's fun just a heads up , hiking in Glacier , it 's not muddy it 's just the dusty dirt paths are very dusty . 2 be +So that 's fun just a heads up , hiking in Glacier , it 's not muddy it 's just the dusty dirt paths are very dusty . 14 be +So that 's fun just a heads up , hiking in Glacier , it 's not muddy it 's just the dusty dirt paths are very dusty . 18 be +So these have been awesome for hiking and I highly recommend them , especially if you 're looking for like a lighter shoe that 's not heavy like hiking boots or anything . 24 be +So it 's our last day and we are just driving the uh – Going to the Sun Road and we ’re just gon na stop along wherever we want to go . 2 be +So that 's good ! 2 be +It 's just the other end of it , um , so we 're just gon na hike a little bit on that . 1 be +It is overcast today and they said there could be some rain , so we 're probably gon na go far and that 's okay . 23 be +That 's so cool . 1 be +It 's one of the best roads in the US I feel , it 's amazing . 1 be +It 's one of the best roads in the US I feel , it 's amazing . 14 be +Alright , I think we made it about a mile on the loop and I think we 're gon na turn around because thing is it 's all uphi- uphill and we 're just we 're being lazy today . 26 be +It 's just a really big like traffic jam here on the road . 1 be +Because we have n't seen any yet and it 's our last day so we were really hoping to see some today and we did ! 9 be +So that 's so cool I 'm so glad we found . 2 be +It 's not even – it 's not three - two , it 's like real good . 1 be +It 's not even – it 's not three - two , it 's like real good . 6 be +It 's not even – it 's not three - two , it 's like real good . 13 be +It 's real beer , not three two , yes . 1 be +It 's the real deal 1 be +Everything 's bigger in Montana . 1 be +Everything 's bigger in Montana . 1 be +But anyways we 're just down by this river here that 's like right here in Columbia Falls and we 're going to eat our ice cream and enjoy our last evening here . 11 be +Alright you guys , it 's our last day here and at Glacier National Park . 5 be +It 's like lipstick central . 1 be +It 's actually pretty crazy in here . 1 be +Like that 's definitely what I want . 2 be +But I also need to clear a lot of these out because I know a lot of these lipsticks are at least over like two three years old and that 's like not okay . 30 be +It 's like the most beautiful red lipstick . 1 be +It 's beautiful , so I ca n't part with it . 1 be +Who 's at the door Hershey ? 1 be +Hershey it 's your first vlog . 2 be +One hundred percent this is definitely definitely expired , but it 's just with lipsticks it 's kind of hard to tell . 11 be +One hundred percent this is definitely definitely expired , but it 's just with lipsticks it 's kind of hard to tell . 16 be +Like — like usually with foundations when it 's expired , you will see the oil separating and all that , but this just looks — it looks fine . 8 be +Yeah that 's gon na have to go in the expired pile . 2 be +I 'm a just put all the Dose of Color in a separate pile cuz that 's kind of weird . 16 be +My gosh it 's hard to think back . 3 be +That 's gon na have to go in the expired pile . 1 be +It 's not looking good though . 1 be +It 's all like super separated . 1 be +That 's two years . 1 be +That 's like not healthy for you ? 1 be +I do n't know if you can actually tell by looking at it but she 's old . 15 be +That 's interesting . 1 be +This is , like ... it 's not a pizza vlog , what is this ? 6 be +You never know what 's gon na happen on Saturdays for me . 4 be +That 's what I did ! 1 be +It 's weird , like , teaching them because they teach me everything about cooking ! 1 be +So , it 's like , uhh , that motion . 3 be +If it falls on the floor , it 's okay ? 8 be +I mean it 's your pizza , but throw it right over ... 3 be +That 's great ! 1 be +The pizza sauce is in Nonna 's blood , that 's where it started . 10 be +What 's pizza without cheese ? 1 be +Oh , yeah , it 's fun ! 5 be +It 's just , like , a little twist . 1 be +It 's a good one . 1 be +It 's nice and crunchy ! 1 be +It 's the first week that I 've been actually able to say the words " I 'm pregnant " and have it not feel really weird . 1 be +It was quite a shock and it 's such a huge huge thing . 7 be +It 's life - changing , and so for the first couple of weeks that 's all I could focus on , was how life - changing it was and it was really scary . 1 be +It 's life - changing , and so for the first couple of weeks that 's all I could focus on , was how life - changing it was and it was really scary . 15 be +I do n't want this to be too , like , all over the place but I decided I do want to document this some way , whether I – at first I was like , I do n't need to copy it on YouTube but I definitely wanted to document this in some kind of video form , just because it 's such a huge thing . 62 be +I mean you , you feel quite bad when you feel like that , because it 's such a happy thing and now , you know , I – you know , I know I 've kind of accepted . 16 be +And I am happy to tell people , but at first when you 're trying to get your head around it , it 's very daunting and you just feel awful thinking . 23 be +I want chips , I want white toast , it 's pretty much all I eat all day . 10 be +Pasta if I want something hot maybe , I 've had soup , but essentially starchy carbs and that 's quite normal it seems . 19 be +That 's crazy , right ? 1 be +Not the best representation of my city I guess , because it 's instant coffee . 12 be +So the Sauvignon Blanc , it 's naturally high - acidic grape , and then if you smell it you pick up grassy aromas , lots of passion fruits , gooseberry flavors in the palate . 6 be +And then the Chardonnay - Pinot Noir , it is a blended wine of 60 % Chardonnay and 40 % Pinot Noir , so it 's more like a dry rosé style of wine with lots of strawberry flavors , hints of yellow apple , and it 's 2018 vintage , which is unwooded . 25 be +And then the Chardonnay - Pinot Noir , it is a blended wine of 60 % Chardonnay and 40 % Pinot Noir , so it 's more like a dry rosé style of wine with lots of strawberry flavors , hints of yellow apple , and it 's 2018 vintage , which is unwooded . 47 be +It 's very fresh and crisp . 1 be +Yes , in South Africa Chardonnay and Pinot Noir , it 's a popular wine . 11 be +Can I ask you , with your white wines , how – how long – when 's the best time to drink them ? 16 be +It 's open now drink now wines , as I said they are lifestyle wines , easy drinking , everyday drinking styles . 1 be +So it 's a high grade grape that is made from Pinot Noir and the Hermitage . 2 be +This is a 2017 vintage and then the Shiraz , it 's known for its biasing characteristics , that what you pick up on the palette on the nose to the palate , spicy peppery flavor , has lots of great fruits and this is 2017 vintage . 11 be +And then the Cab , the Cabernet is known for its thick uh skin and it 's known for when it 's blended , it always acts as a backbone to the wine . 16 be +And then the Cab , the Cabernet is known for its thick uh skin and it 's known for when it 's blended , it always acts as a backbone to the wine . 21 be +So , so with th- with these people , it 's better to wait for these as well , even though they 're your everyday – 10 be +That 's good ! 1 be +He 's lying , you drank – you drank most of them . 1 be +What 's your recap ? 1 be +No which one 's your favorite wine ? 3 be +What 's in there ? 1 be +So it 's a white wine . 2 be +Yeah , it 's a white wine . 3 be +He 's driving , not me . 1 be +The building is typical , it 's a former warehouse . 6 be +While it 's not as cold as in the northern states during the winter , it does freeze sometimes , and temperatures in the 30s ° F ( that 's around or slightly above 0 ° C ) are not unheard of . 2 be +While it 's not as cold as in the northern states during the winter , it does freeze sometimes , and temperatures in the 30s ° F ( that 's around or slightly above 0 ° C ) are not unheard of . 29 be +With over 40,000 inhabitants , though , it 's not just a nature reserve . 8 be +Ronald McDonald showing how it 's done 5 be +The rules are even more strict for foreign visitors , so even if you see a local in shorts it 's not OK for everyone . 20 be +It 's a romantic city for a weekend break . 1 be +If it 's minimal , they may be simply having a bad day . 2 be +Do n't be nosy or gossipy , but if they are happy with someone one day and hateful with them the next , that 's a sign of them having a lot of fair weather friends . 24 be +That 's a sign of arrogance , since it is very hard to be a truly good friend to someone who 's stuck on themselves . 1 be +That 's a sign of arrogance , since it is very hard to be a truly good friend to someone who 's stuck on themselves . 21 be +If it 's inherently negative , then they 're either over - zealous , ignorant of other people or what to avoid those that contradict their fantasy land that caters to them and them only . 2 be +Many times prideful people have a serious ' my - way 's - the - only - way ' attitude . 11 be +What 's their personality like ? 1 be +If it 's harsh , they are . 2 be +If they 're nice to your face , but talk bad about you behind your back like it 's their favourite hobby , then they probably have a problem with pride . 18 be +For this reason , it 's always better to practice ballet in a ballet studio under the guidance of a good instructor , who will be able to correct your positioning and make sure that you 're dancing properly . 5 be +Basil needs warm air and sun to do well , so it 's often easiest to start the seeds indoors instead of risking that they 'll get damaged by frost . 12 be +Dampen the mixture with water so it 's ready to provide the right environment for the basil seeds to germinate . 7 be +When you see the first green tendrils push up through the soil , it 's time to remove the plastic wrap . 14 be +When the plants become a few inches tall and their leaves mature , it 's time to transplant them to a larger container . 14 be +It 's best to put basil somewhere where it will get a good deal of sunshine and have well - drained soil . 1 be +If you want to plant the basil in a container , make sure it 's large enough to accommodate the number of plants you 're growing ; 14 be +Anyone who 's spent a lot of time on a farm with chickens is probably familiar with this trick . 2 have +While it 's not uncommon to transport a chicken by its feet , this upside - down hold risks breaking the chicken 's hip . 2 be +It 's not completely clear how much stress this hypnosis causes . 1 be +Here 's a great vegan cupcake recipe to use as a base for whatever flavored icing you want to add to it . 1 be +And olive oil works fine if it 's all that 's left in the pantry . 7 be +And olive oil works fine if it 's all that 's left in the pantry . 10 be +After your initial greeting and exchange of pleasantries , you 'll probably want to ask how the person 's doing ( just like you might in the real world ) . 18 be +For example , after we 've opened up by asking about this person 's interest in the band in his / her username , it 's logical and reasonable to ask about this person 's musical likes and dislikes . 25 be +For instance , if you 're asked what you 're up to , rather than saying , " looking for people to hit on on Facebook " you might find that it 's a better idea to give a sarcastic answer like " writing the great American novel " or " drowning my sorrows " . 32 be +It 's also a good idea to not wear your Sunday best . 1 be +It 's important to use sanitary , clean tubes for your glowsticks . 1 be +It 's not like the glow will get up and run away from you , but still . 1 be +If it 's not already glowing , something went wrong . 2 be +For something that lasts hours , go to the next method ( which is a lot easier to facilitate if you have access to a laboratory , but it 's still worth mentioning ) . 29 be +It 's uncompleted though , because the word is part of the term ' aviation ' 1 be +It 's a completed term , but it is supposed to come from ' Flap ! ' 1 be +For example we write it from left to right , in Arabic it 's right to left , Chinese is in columns , and so on . 13 be +It 's much more fun when there are other people that can understand your language . 1 be +Write down a list of everything you are packing , in particular if it 's valuable . 14 be +When you have a looming project that 's bogging you down and making you unproductive in other areas , tackle it first . 7 be +You do n't need flying broomsticks or magic balls to play this version of the game - it 's tailor - made for muggles like us . 18 be +Another variation is where the referee places the snitch somewhere in the field ( in this case it 's a small ball ) . 18 be +If you bought packaged quinoa , this step is n't always necessary , but it 's better to be safe than sorry . 15 be +The lime green Skittles add a sourness to the drink , so if you want one that 's sweet , leave the green ones out and add them to their own bottle for a sour lime vodka . 17 be diff --git a/stanza/models/lemma_classifier/utils.py b/stanza/models/lemma_classifier/utils.py new file mode 100644 index 0000000000..feb4c7a85a --- /dev/null +++ b/stanza/models/lemma_classifier/utils.py @@ -0,0 +1,7 @@ +import stanza + +def load_doc_from_conll_file(path: str): + """" + loads in a Stanza document object from a path to a CoNLL file containing annotated sentences. + """ + return stanza.utils.conll.CoNLL.conll2doc(path) \ No newline at end of file