diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..9248de4
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+env/
+.vscode
+*pycache*
+*.bin
+*.pt
+*cached*
+*checkpoint*
+analysis/output/*
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..9006e3f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,43 @@
+# The r/Jokes Dataset: a Large Scale Humor Collection
+Code and Datasets from the paper, ["The r/Jokes Dataset: a Large Scale Humor Collection"](TODO) by Orion Weller and Kevin Seppi
+
+Dataset files are located in `data/{train/dev/test}.tsv` for the regression task, while the full unsplit data can be found in `data/preprocessed.tsv`.  These files will need to be unzipped after cloning the repo.
+
+For related projects, see our work on [Humor Detection (separating the humorous jokes from the non-humorous)](https://github.com/orionw/RedditHumorDetection) or other cited work.
+
+** **We do not endorse these jokes. Please view at your own risk** **
+
+## License
+The data is under the [Reddit License and Terms of Service](https://www.reddit.com/wiki/api-terms) and users must follow the Reddit User Agreement and Privacy Policy, as well as remove any posts if asked to by the original user.  For more details on this, please see the link above.  
+
+# Usage
+## Load the Required Packages
+0. Run `pip3 install -r requirements.txt`
+1. Gather the NLTK packages by running `bash download_nltk_packages.sh`.  This downloads the packages `averaged_perceptron_tagger`, `words`, `stopwords`, `maxent_ne_chunker`, used for analysis/preprocessing.
+
+## Reproduce the current dataset (updated to Jan 1st 2020)
+### We chunk this process into three parts to avoid networking errors
+0. Run `python3 gather_reddit_pushshift.py` after `cd prepare_data` to gather the Reddit post ids.
+1. Run `python3 preprocess.py --update` to update the Reddit post IDs with the full post.
+2. Run `python3 preprocess.py --preprocess` to preprocess the Reddit posts into final datasets
+
+## Reproduce plots and analysis from the paper
+0. Run `cd analysis`
+1. Run `python3 time_statistics.py` to gather the statistics that display over time
+2. Run `python3 dataset_statistics.py` to gather the overall dataset statistics
+3. See plots in the `./plots` folder
+
+## Re-gather All Jokes and Extend With Newer Jokes 
+0. Run the first two commands in the `Reproduce` section above
+1. Update the code in the `preprocess` function of the `preprocess.py` file to NOT remove all jokes after 2020 (line 89).  Then run `python3 preprocess.py --preprocess`
+
+# Reference:
+If you found this repository helpful, please cite the following paper:
+```
+@ARTICLE{rjokesData2020,
+  title={The r/Jokes Dataset: a Large Scale Humor Collection},
+  author={Weller, Orion and Seppi, Kevin},
+  journal={"Proceedings of the 2020 Conference of Language Resources and Evaluation"},
+  month=May,
+  year = "2020",
+}
diff --git a/analysis/dataset_statistics.py b/analysis/dataset_statistics.py
new file mode 100644
index 0000000..015331f
--- /dev/null
+++ b/analysis/dataset_statistics.py
@@ -0,0 +1,209 @@
+import os
+import json
+import string
+import random
+
+import pandas as pd
+import numpy as np
+from wordcloud import WordCloud
+from sklearn.feature_extraction.text import CountVectorizer
+import matplotlib.pyplot as plt
+import seaborn as sns
+import nltk
+from nltk.corpus import stopwords
+from nltk.tokenize import word_tokenize 
+from nltk.stem import PorterStemmer
+from nltk.stem import LancasterStemmer
+from nltk import ne_chunk, pos_tag, word_tokenize
+from nltk.tree import Tree
+
+from sklearn.decomposition import LatentDirichletAllocation as LDA
+
+random.seed(42)
+np.random.seed(42)
+
+EXT_TYPES = ["pdf", "png"]
+REMOVE_TOKENS = set(stopwords.words('english') + ["n't", "ve"]).union(set(string.punctuation))
+sns.set_style('whitegrid')
+
+def get_continuous_chunks(text):
+        chunked = ne_chunk(pos_tag(word_tokenize(text)))
+        continuous_chunk = []
+        current_chunk = []
+        for i in chunked:
+                if type(i) == Tree:
+                        current_chunk.append(" ".join([token for token, pos in i.leaves()]))
+                elif current_chunk:
+                        named_entity = " ".join(current_chunk)
+                        if named_entity not in continuous_chunk:
+                                continuous_chunk.append(named_entity)
+                                current_chunk = []
+                else:
+                        continue
+        return continuous_chunk
+
+
+def plot_n_most_common_words(text: pd.Series, data_name: str, n: int = 10, specific_year: str = None):
+    entities = []
+    for sentence in text:
+        entities.extend(get_continuous_chunks(sentence))
+    stemmer = PorterStemmer()
+    filtered_words = [stemmer.stem(w) for w in entities if not w in REMOVE_TOKENS]
+    # Initialise the count vectorizer with the English stop words
+    count_vectorizer = CountVectorizer(stop_words='english')
+    # Fit and transform the processed titles
+    count_data = count_vectorizer.fit_transform(filtered_words)
+    # Visualise the 10 most common words
+    words = count_vectorizer.get_feature_names()
+    total_counts = np.zeros(len(words))
+    for t in count_data:
+        total_counts+=t.toarray()[0]
+
+    get_lda(count_data, count_vectorizer)
+
+    count_dict = (zip(words, total_counts))
+    count_dict = sorted(count_dict, key=lambda x:x[1], reverse=True)[0:n]
+    words = [w[0] for w in count_dict]
+    counts = [w[1] for w in count_dict]
+    x_pos = np.arange(len(words)) 
+    
+    for ext_type in EXT_TYPES:
+        plt.figure(2, figsize=(15, 15/1.6180))
+        plt.subplot(title='{} most common words'.format(n))
+        sns.set_context("notebook", font_scale=1.25, rc={"lines.linewidth": 2.5})
+        sns.barplot(x_pos, counts, palette='husl')
+        plt.xticks(x_pos, words, rotation=90) 
+        plt.xlabel('words')
+        plt.ylabel('counts')
+        plt.savefig(os.path.join(os.path.realpath('..'), "plots", data_name, 
+                    "{}top_wordcounts.{}".format("" if specific_year is None else str(specific_year) + "_", ext_type)))
+        plt.close()
+
+
+def run_wordcloud(text: list, data_name: str):
+    word_tokens = []
+    try:
+        for index, sentence in enumerate(text):
+            if pd.isnull(sentence):
+                print("index is null", index)
+                continue
+            word_tokens.extend(word_tokenize(sentence))
+    except Exception as e:
+        import pdb; pdb.set_trace()
+        print(e, sentence)
+    filtered_words = [w for w in word_tokens if not w in REMOVE_TOKENS] 
+    # Create a WordCloud object
+    wordcloud = WordCloud(background_color="white", width=800, height=800, max_words=150, contour_width=3, contour_color='steelblue')
+    # Generate a word cloud
+    wordcloud.generate(" ".join(filtered_words))
+    # Visualize the word cloud
+    for ext_type in EXT_TYPES:
+        wordcloud.to_file(os.path.join(os.path.realpath('..'), "plots", data_name, 'wordCloud.{}'.format(ext_type)))
+        plt.close()
+
+
+def print_topics(model, count_vectorizer, n_top_words):
+    words = count_vectorizer.get_feature_names()
+    for topic_idx, topic in enumerate(model.components_):
+        print("\nTopic #%d:" % topic_idx)
+        print(" ".join([words[i]
+                        for i in topic.argsort()[:-n_top_words - 1:-1]]))
+
+def get_lda(count_data, count_vectorizer): 
+    # Tweak the two parameters below
+    number_topics = 5
+    number_words = 10
+    # Create and fit the LDA model
+    lda = LDA(n_components=number_topics, n_jobs=-1)
+    lda.fit(count_data)
+    print("Topics found via LDA:")
+    print_topics(lda, count_vectorizer, number_words)
+
+def get_statistics(df: pd.DataFrame, data_name: str):
+    distrib = df.copy(deep=True)
+    distrib = distrib[distrib["score"] != 0]
+    for ext_type in EXT_TYPES:
+        sns.distplot(distrib["score"])
+        plt.savefig(os.path.join(os.path.realpath('..'), "plots", data_name, 'total_score_distribution.{}'.format(ext_type)))
+        plt.close()
+
+    punch = np.array([len(sentence) for sentence in df["punchline"].values])
+    body = np.array([len(sentence) for sentence in df["body"].values])
+    joke = np.array([len(sentence) for sentence in df["joke"].values])
+
+    average_punch = np.mean(punch)
+    average_body = np.mean(body)
+    average_joke = np.mean(joke) - 1.0 # Joke Token, `AND` to join body and punchline together
+
+    std_punch = np.nanstd(punch)
+    std_body = np.nanstd(body)
+    std_joke = np.nanstd(joke)
+
+    ave_tokens_punch = np.nanmean(np.array([len(nltk.word_tokenize(sentence)) for sentence in df["punchline"].dropna().values]))
+    ave_tokens_body = np.nanmean(np.array([len(nltk.word_tokenize(sentence)) for sentence in df["body"].dropna().values]))
+    ave_tokens_joke = np.nanmean(np.array([len(nltk.word_tokenize(sentence)) for sentence in df["joke"].dropna().values]))
+
+    tokens = []
+    [tokens.extend(nltk.word_tokenize(joke)) for joke in df["joke"].dropna().values]
+    total_tokens = len(set(tokens))
+
+    stat_df = pd.DataFrame([{"ave_punchline_len": average_punch, "ave_body_len": average_body, "ave_joke_len": average_joke, "std_punch": std_punch,
+                                "std_body": std_body, "std_joke": std_joke, "total_tokens": total_tokens}])
+    stat_df.to_csv(os.path.join(os.path.realpath('..'), "plots", data_name, 'statistics.txt'))
+
+
+def plot_sentiment(df: pd.DataFrame, data_name: str):
+    for ext_type in EXT_TYPES:
+        ax = sns.lineplot(x="date", y="prop", hue="sentiment", data=df, ci=False)
+        # Find the x,y coordinates for each point
+        x_coords = []
+        y_coords = []
+        for point_pair in ax.collections:
+            for x, y in point_pair.get_offsets():
+                x_coords.append(x)
+                y_coords.append(y)
+        # create the custom error bars
+        colors = ['steelblue']*2 + ['coral']*2
+        ax.errorbar(x_coords, y_coords, yerr=df["std"],
+            ecolor=colors, fmt=' ', zorder=-1)
+        ax.savefig(os.path.join(os.path.realpath('..'), "data", data_name, "sentiment_plot.{}".format(ext_type)))
+
+
+
+def gather_data(df: pd.DataFrame, data_name: str):
+    if not os.path.isdir(os.path.join(os.path.realpath('..'), "plots", data_name)):
+        os.mkdir(os.path.join(os.path.realpath('..'), "plots", data_name))
+    run_wordcloud(df["joke"].tolist(), data_name)
+    plot_n_most_common_words(df["joke"].tolist(), data_name)
+    get_statistics(df, data_name)
+
+
+def percentiles_upvotes(df: pd.DataFrame, data_name: str) -> pd.DataFrame:
+    list_of_percentiles = []
+    for percentile in [0, 10, 25, 50, 75, 90, 100]:
+        cur_per = np.percentile(df["score"], percentile)
+        list_of_percentiles.append({"percentile": percentile, "value": cur_per})
+    percent_df = pd.DataFrame(list_of_percentiles)
+    percent_df.to_csv(os.path.join(os.path.realpath('..'), "plots", data_name, "percentiles.csv"))
+    return percent_df
+
+
+if __name__ == "__main__":
+    # NOTE: log-distribution plots are found in the preprocess.py script
+    df = pd.read_csv(os.path.join(os.path.realpath('..'), "data", "preprocessed.csv"), index_col=None, encoding="UTF-8", keep_default_na=False)
+    df["date"] = pd.to_numeric(df["date"])
+    df["score"] = pd.to_numeric(df["score"])
+    df = df[df["date"].isna() == False]
+    assert df.shape == df.dropna().shape, "was nans that are unaccounted for"
+    percentiles_upvotes(df, "all")
+    gather_data(df, "all")
+
+
+
+
+
+
+
+
+
+
diff --git a/analysis/gather_regression.py b/analysis/gather_regression.py
new file mode 100644
index 0000000..5ec63e0
--- /dev/null
+++ b/analysis/gather_regression.py
@@ -0,0 +1,53 @@
+import os
+import pandas as pd 
+import matplotlib.pyplot as plt
+import seaborn as sns
+import glob
+
+def gather_data_from_huggingface(root_dir: str, output_dir: str):
+    all_models = {}
+    for result_dir in os.listdir(root_dir):
+        model_name = result_dir.split("/")[-1]
+        model_results = []
+        checkpoint_folder = None
+        for checkpoint_folder in glob.glob(os.path.join(root_dir, result_dir, "*checkpoint-*/")):
+            if not os.path.isfile(os.path.join(checkpoint_folder, "eval_results.txt")):
+                print("No results file in path:", checkpoint_folder)
+            else:
+                results_on_check = {}
+                with open(os.path.join(checkpoint_folder, "eval_results.txt"), "r") as f:
+                    for index, line in enumerate(f):
+                        name, value = line.split("=")
+                        name, value = name.strip(), float(value.strip())
+                        results_on_check[name] = value
+            results_on_check["model"] = model_name
+            results_on_check["checkpoint"] = checkpoint_folder.split("/")[-2]
+            model_results.append(results_on_check)
+        results_df = pd.DataFrame(model_results)
+        if checkpoint_folder is not None:
+            results_df.to_csv(os.path.join(checkpoint_folder, "model_results.csv"))
+            ax = sns.lineplot(x="index", y="rmse", data=results_df.reset_index()) 
+            plt.savefig(os.path.join(root_dir, result_dir, "rmse_plot.png"))
+            plt.close()
+            all_models[model_name] = results_df
+            print("Wrote model file to ", os.path.join(result_dir, "rmse_plot.png"))
+
+    if all_models:
+        # now that all the results have been gathered, let's combine them
+        full_df = pd.concat(list(all_models.values())).reset_index()
+        full_df.to_csv(output_dir)
+        # focus on lowest RMSE (could focus on Pearson, Spearmanr etc.)
+        min_rmse = full_df.groupby("model")["rmse"].idxmin()
+        best_results = full_df.iloc[min_rmse, :]
+
+        # plot line plot here
+        ax = sns.barplot(x="model", y="rmse", data=best_results) 
+        plt.savefig(os.path.join(root_dir, "rmse_plot.png"))
+        print("Wrote full file to ", os.path.join(root_dir, "rmse_plot.png"))
+        plt.close()
+        print(best_results)
+
+
+if __name__ == "__main__":
+    gather_data_from_huggingface("output/large", "regression_results_large.csv")
+    gather_data_from_huggingface("output/base/", "regression_results_base.csv")
\ No newline at end of file
diff --git a/analysis/regression-base.sh b/analysis/regression-base.sh
new file mode 100644
index 0000000..34179d9
--- /dev/null
+++ b/analysis/regression-base.sh
@@ -0,0 +1,25 @@
+export DATA_DIR=../data
+# regression task set up
+export TASK_NAME=STS-B
+declare -a arr=("roberta" "bert" "xlnet")
+declare -a arrtwo=("roberta-base" "bert-base-uncased" "xlnet-base-cased")
+
+
+for ((i=0; i<3; i++));
+do
+    python run_glue.py \
+      --model_type "${arr[i]}"  \
+      --model_name_or_path "${arrtwo[i]}" \
+      --task_name $TASK_NAME \
+      --do_train \
+      --do_eval \
+      --do_lower_case \
+      --data_dir $DATA_DIR \
+      --eval_all_checkpoints \
+      --max_seq_length 128 \
+      --per_gpu_train_batch_size 96 \
+      --learning_rate 2e-5 \
+      --num_train_epochs 5.0 \
+      --overwrite_output_dir \
+      --output_dir output/base/"${arr[i]}"
+done
diff --git a/analysis/regression-large.sh b/analysis/regression-large.sh
new file mode 100644
index 0000000..ae3e461
--- /dev/null
+++ b/analysis/regression-large.sh
@@ -0,0 +1,26 @@
+export DATA_DIR=../data
+# regression task set up
+export TASK_NAME=STS-B
+declare -a arr=("roberta" "bert" "xlnet")
+# I used batch size of 96 with xlnet-large and 128 with others
+declare -a arrtwo=("roberta-large" "bert-large-uncased" "xlnet-large-cased")
+
+
+for ((i=0; i<3; i++));
+do
+    python run_glue.py \
+      --model_type "${arr[i]}"  \
+      --model_name_or_path "${arrtwo[i]}" \
+      --task_name $TASK_NAME \
+      --do_train \
+      --do_eval \
+      --do_lower_case \
+      --data_dir $DATA_DIR \
+      --eval_all_checkpoints \
+      --max_seq_length 128 \
+      --per_gpu_train_batch_size 128 \
+      --learning_rate 2e-5 \
+      --num_train_epochs 5.0 \
+      --overwrite_output_dir \
+      --output_dir output/large/"${arr[i]}"
+done
diff --git a/analysis/regression_results_base.csv b/analysis/regression_results_base.csv
new file mode 100644
index 0000000..f79fbde
--- /dev/null
+++ b/analysis/regression_results_base.csv
@@ -0,0 +1,10 @@
+,index,corr,pearson,rmse,spearmanr,model,checkpoint
+0,0,0.4443397796905355,0.4651837567754322,1.7541763,0.42349580260563885,roberta,checkpoint-12000
+1,1,0.4313989023673853,0.45019291276869466,1.6386057,0.41260489196607597,roberta,checkpoint-6000
+2,2,0.4452852768941931,0.4658201273950414,1.8800247,0.4247504263933448,roberta,checkpoint-18000
+3,0,0.4318083997703859,0.4531463200754483,1.8907644,0.4104704794653235,xlnet,checkpoint-12000
+4,1,0.42182588036243795,0.4421904039165935,1.6474583,0.4014613568082824,xlnet,checkpoint-6000
+5,2,0.43808309570939125,0.4580278612177484,1.9638016,0.4181383302010341,xlnet,checkpoint-18000
+6,0,0.44518519943639,0.46596080774994675,1.7353058,0.4244095911228332,bert,checkpoint-12000
+7,1,0.43999650071057195,0.4611161397762899,1.6263107,0.41887686164485405,bert,checkpoint-6000
+8,2,0.44383916832300385,0.4650446676294345,1.8012,0.42263366901657323,bert,checkpoint-18000
diff --git a/analysis/regression_results_large.csv b/analysis/regression_results_large.csv
new file mode 100644
index 0000000..0b55a7a
--- /dev/null
+++ b/analysis/regression_results_large.csv
@@ -0,0 +1,11 @@
+,index,corr,pearson,rmse,spearmanr,model,checkpoint
+0,0,0.4667788714460796,0.4883324584728209,1.7011992,0.4452252844193383,roberta,checkpoint-12000
+1,1,0.4544410080159367,0.47388892823620876,1.6138555,0.43499308779566465,roberta,checkpoint-6000
+2,2,0.4660406387792235,0.48559753527177185,1.8018064,0.4464837422866752,roberta,checkpoint-18000
+3,0,0.43412252294017617,0.4567995734825274,1.7385621,0.411445472397825,xlnet,checkpoint-12000
+4,1,0.37713699244381943,0.38511733976723006,1.9101037,0.3691566451204088,xlnet,checkpoint-6000
+5,2,0.44718551599712086,0.4682513100855499,1.9024026,0.42611972190869185,xlnet,checkpoint-24000
+6,3,0.44616641205790697,0.4687108119213232,1.772816,0.4236220121944907,xlnet,checkpoint-18000
+7,0,0.44932231309459303,0.47116557636851286,1.7352395,0.4274790498206732,bert,checkpoint-12000
+8,1,0.4506944418874647,0.4709748411712956,1.6188005,0.4304140426036338,bert,checkpoint-6000
+9,2,0.4523183259704836,0.47327446616808244,1.8045843,0.43136218577288477,bert,checkpoint-18000
diff --git a/analysis/run_glue.py b/analysis/run_glue.py
new file mode 100644
index 0000000..5cf1c0c
--- /dev/null
+++ b/analysis/run_glue.py
@@ -0,0 +1,662 @@
+# coding=utf-8
+# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
+# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Finetuning the library models for sequence classification on GLUE (Bert, XLM, XLNet, RoBERTa)."""
+
+from __future__ import absolute_import, division, print_function
+
+import argparse
+import glob
+import logging
+import os
+import random
+
+import numpy as np
+from scipy.stats import pearsonr, spearmanr
+from sklearn.metrics import matthews_corrcoef, f1_score, mean_squared_error
+import torch
+from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
+                              TensorDataset)
+from torch.utils.data.distributed import DistributedSampler
+
+from tqdm import tqdm, trange
+
+from transformers import (WEIGHTS_NAME, BertConfig,
+                                  BertForSequenceClassification, BertTokenizer,
+                                  RobertaConfig,
+                                  RobertaForSequenceClassification,
+                                  RobertaTokenizer,
+                                  XLMConfig, XLMForSequenceClassification,
+                                  XLMTokenizer, XLNetConfig,
+                                  XLNetForSequenceClassification,
+                                  XLNetTokenizer,
+                                  DistilBertConfig,
+                                  DistilBertForSequenceClassification,
+                                  DistilBertTokenizer)
+
+from transformers import AdamW, get_linear_schedule_with_warmup
+# from transformers import WarmupLinearSchedule as get_linear_schedule_with_warmup
+
+from transformers import glue_output_modes as output_modes
+from transformers import glue_convert_examples_to_features as convert_examples_to_features
+from transformers import DataProcessor, InputExample
+
+logger = logging.getLogger(__name__)
+
+ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLNetConfig, XLMConfig, 
+                                                                                RobertaConfig, DistilBertConfig)), ())
+
+MODEL_CLASSES = {
+    'bert': (BertConfig, BertForSequenceClassification, BertTokenizer),
+    'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
+    'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
+    'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
+    'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer)
+}
+
+
+def simple_accuracy(preds, labels):
+    return (preds == labels).mean()
+
+
+def acc_and_f1(preds, labels):
+    acc = simple_accuracy(preds, labels)
+    f1 = f1_score(y_true=labels, y_pred=preds)
+    return {
+        "acc": acc,
+        "f1": f1,
+        "acc_and_f1": (acc + f1) / 2,
+    }
+
+
+def pearson_and_spearman(preds, labels):
+    rmse = np.sqrt(mean_squared_error(preds, labels))
+    pearson_corr = pearsonr(preds, labels)[0]
+    spearman_corr = spearmanr(preds, labels)[0]
+    return {
+        "pearson": pearson_corr,
+        "spearmanr": spearman_corr,
+        "corr": (pearson_corr + spearman_corr) / 2,
+        "rmse": rmse
+    }
+
+
+def compute_metrics(task_name, preds, labels):
+    assert len(preds) == len(labels)
+    if task_name == "cola":
+        acc = simple_accuracy(preds, labels)
+        f1 = f1_score(y_true=labels, y_pred=preds)
+        mcc = matthews_corrcoef(labels, preds)
+        return {
+            "acc": acc,
+            "f1": f1,
+            "acc_and_f1": (acc + f1) / 2,
+            "mcc": mcc
+        }
+    elif task_name == "sst-2":
+        return {"acc": simple_accuracy(preds, labels)}
+    elif task_name == "mrpc":
+        return acc_and_f1(preds, labels)
+    elif task_name == "sts-b":
+        return pearson_and_spearman(preds, labels)
+    elif task_name == "qqp":
+        return acc_and_f1(preds, labels)
+    elif task_name == "mnli":
+        return {"acc": simple_accuracy(preds, labels)}
+    elif task_name == "mnli-mm":
+        return {"acc": simple_accuracy(preds, labels)}
+    elif task_name == "qnli":
+        return {"acc": simple_accuracy(preds, labels)}
+    elif task_name == "rte":
+        return {"acc": simple_accuracy(preds, labels)}
+    elif task_name == "wnli":
+        return {"acc": simple_accuracy(preds, labels)}
+    else:
+        raise KeyError(task_name)
+
+
+class StsbProcessor(DataProcessor):
+    """OVERRIDE Processor for the STS-B data set (GLUE version)."""
+
+    def get_example_from_tensor_dict(self, tensor_dict):
+        """See base class."""
+        return InputExample(tensor_dict['idx'].numpy(),
+                            tensor_dict['sentence'].numpy().decode('utf-8'),
+                            None,
+                            str(tensor_dict['label'].numpy()))
+
+    def get_train_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")
+
+    def get_dev_examples(self, data_dir):
+        """See base class."""
+        return self._create_examples(
+            self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")
+
+    def get_labels(self):
+        """See base class."""
+        return [None]
+
+    def _create_examples(self, lines, set_type):
+        """Creates examples for the training and dev sets."""
+        examples = []
+        for (i, line) in enumerate(lines):
+            try:
+                if i == 0:
+                    continue
+                guid = "%s-%s" % (set_type, line[0])
+                text_a = line[1]
+                text_b =  None
+                label = line[0]
+                examples.append(
+                    InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
+            except Exception as e:
+                import pdb; pdb.set_trace()
+                print(e)
+        return examples
+
+# define my own
+# from transformers import glue_processors as processors
+processors = {
+    "sts-b": StsbProcessor,
+}
+
+def set_seed(args):
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def train(args, train_dataset, model, tokenizer):
+    """ Train the model """
+
+    args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)
+    train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset)
+    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)
+
+    if args.max_steps > 0:
+        t_total = args.max_steps
+        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+    else:
+        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
+
+    # Prepare optimizer and schedule (linear warmup and decay)
+    no_decay = ['bias', 'LayerNorm.weight']
+    optimizer_grouped_parameters = [
+        {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay},
+        {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
+        ]
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
+    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total)
+    if args.fp16:
+        try:
+            from apex import amp
+        except ImportError:
+            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
+
+    # multi-gpu training (should be after apex fp16 initialization)
+    if args.n_gpu > 1:
+        model = torch.nn.DataParallel(model)
+
+    # Distributed training (should be after apex fp16 initialization)
+    if args.local_rank != -1:
+        model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
+                                                          output_device=args.local_rank,
+                                                          find_unused_parameters=True)
+
+    # Train!
+    logger.info("***** Running training *****")
+    logger.info("  Num examples = %d", len(train_dataset))
+    logger.info("  Num Epochs = %d", args.num_train_epochs)
+    logger.info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
+    logger.info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
+                   args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1))
+    logger.info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
+    logger.info("  Total optimization steps = %d", t_total)
+
+    global_step = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    model.zero_grad()
+    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0])
+    set_seed(args)  # Added here for reproductibility (even between python 2 and 3)
+    for _ in train_iterator:
+        epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0])
+        for step, batch in enumerate(epoch_iterator):
+            model.train()
+            batch = tuple(t.to(args.device) for t in batch)
+            inputs = {'input_ids':      batch[0],
+                      'attention_mask': batch[1],
+                      'labels':         batch[3]}
+            if args.model_type != 'distilbert':
+                inputs['token_type_ids'] = batch[2] if args.model_type in ['bert', 'xlnet'] else None  # XLM, DistilBERT and RoBERTa don't use segment_ids
+            outputs = model(**inputs)
+            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)
+
+            if args.n_gpu > 1:
+                loss = loss.mean() # mean() to average on multi-gpu parallel training
+            if args.gradient_accumulation_steps > 1:
+                loss = loss / args.gradient_accumulation_steps
+
+            if args.fp16:
+                with amp.scale_loss(loss, optimizer) as scaled_loss:
+                    scaled_loss.backward()
+            else:
+                loss.backward()
+
+            tr_loss += loss.item()
+            if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu:
+                if args.fp16:
+                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
+                else:
+                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
+
+                optimizer.step()
+                scheduler.step()  # Update learning rate schedule
+                model.zero_grad()
+                global_step += 1
+
+                if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0:
+                    # Log metrics
+                    if args.local_rank == -1 and args.evaluate_during_training:  # Only evaluate when single GPU otherwise metrics may not average well
+                        results = evaluate(args, model, tokenizer)
+                    logging_loss = tr_loss
+
+                if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0:
+                    # Save model checkpoint
+                    output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step))
+                    if not os.path.exists(output_dir):
+                        os.makedirs(output_dir)
+                    model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
+                    model_to_save.save_pretrained(output_dir)
+                    torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+            if args.tpu:
+                args.xla_model.optimizer_step(optimizer, barrier=True)
+                model.zero_grad()
+                global_step += 1
+
+            if args.max_steps > 0 and global_step > args.max_steps:
+                epoch_iterator.close()
+                break
+        if args.max_steps > 0 and global_step > args.max_steps:
+            train_iterator.close()
+            break
+
+
+    return global_step, tr_loss / global_step
+
+
+def evaluate(args, model, tokenizer, prefix=""):
+    # Loop to handle MNLI double evaluation (matched, mis-matched)
+    eval_task_names = ("mnli", "mnli-mm") if args.task_name == "mnli" else (args.task_name,)
+    eval_outputs_dirs = (args.output_dir, args.output_dir + '-MM') if args.task_name == "mnli" else (args.output_dir,)
+
+    results = {}
+    for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs):
+        eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)
+
+        if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(eval_output_dir)
+
+        args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)
+        # Note that DistributedSampler samples randomly
+        eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset)
+        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)
+
+        # multi-gpu eval
+        if args.n_gpu > 1:
+            model = torch.nn.DataParallel(model)
+
+        # Eval!
+        logger.info("***** Running evaluation {} *****".format(prefix))
+        logger.info("  Num examples = %d", len(eval_dataset))
+        logger.info("  Batch size = %d", args.eval_batch_size)
+        eval_loss = 0.0
+        nb_eval_steps = 0
+        preds = None
+        out_label_ids = None
+        for batch in tqdm(eval_dataloader, desc="Evaluating"):
+            model.eval()
+            batch = tuple(t.to(args.device) for t in batch)
+
+            with torch.no_grad():
+                inputs = {'input_ids':      batch[0],
+                          'attention_mask': batch[1],
+                          'labels':         batch[3]}
+                if args.model_type != 'distilbert':
+                    inputs['token_type_ids'] = batch[2] if args.model_type in ['bert', 'xlnet'] else None  # XLM, DistilBERT and RoBERTa don't use segment_ids
+                outputs = model(**inputs)
+                tmp_eval_loss, logits = outputs[:2]
+
+                eval_loss += tmp_eval_loss.mean().item()
+            nb_eval_steps += 1
+            if preds is None:
+                preds = logits.detach().cpu().numpy()
+                out_label_ids = inputs['labels'].detach().cpu().numpy()
+            else:
+                preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)
+                out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0)
+
+        eval_loss = eval_loss / nb_eval_steps
+        if args.output_mode == "classification":
+            preds = np.argmax(preds, axis=1)
+        elif args.output_mode == "regression":
+            preds = np.squeeze(preds)
+        result = compute_metrics(eval_task, preds, out_label_ids)
+        results.update(result)
+
+        output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt")
+        with open(output_eval_file, "w") as writer:
+            logger.info("***** Eval results {} *****".format(prefix))
+            for key in sorted(result.keys()):
+                logger.info("  %s = %s", key, str(result[key]))
+                writer.write("%s = %s\n" % (key, str(result[key])))
+
+    return results
+
+
+def load_and_cache_examples(args, task, tokenizer, evaluate=False):
+    if args.local_rank not in [-1, 0] and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    processor = processors[task]()
+    output_mode = output_modes[task]
+    # Load data features from cache or dataset file
+    cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}'.format(
+        'dev' if evaluate else 'train',
+        list(filter(None, args.model_name_or_path.split('/'))).pop(),
+        str(args.max_seq_length),
+        str(task)))
+    if os.path.exists(cached_features_file) and not args.overwrite_cache:
+        logger.info("Loading features from cached file %s", cached_features_file)
+        features = torch.load(cached_features_file)
+    else:
+        logger.info("Creating features from dataset file at %s", args.data_dir)
+        label_list = processor.get_labels()
+        if task in ['mnli', 'mnli-mm'] and args.model_type in ['roberta']:
+            # HACK(label indices are swapped in RoBERTa pretrained model)
+            label_list[1], label_list[2] = label_list[2], label_list[1] 
+        examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir)
+        """
+        Examples is a list of dicts like so:
+        {
+            "guid": "train-0", # train id
+            "label": "1",
+            "text_a": "RIP Roger Rabbit ... James Bond",
+            "text_b": null
+        }
+        """
+        features = convert_examples_to_features(examples,
+                                                tokenizer,
+                                                label_list=label_list,
+                                                max_length=args.max_seq_length,
+                                                output_mode=output_mode,
+                                                pad_on_left=bool(args.model_type in ['xlnet']),                 # pad on the left for xlnet
+                                                pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
+                                                pad_token_segment_id=4 if args.model_type in ['xlnet'] else 0,
+        )
+        if args.local_rank in [-1, 0]:
+            logger.info("Saving features into cached file %s", cached_features_file)
+            torch.save(features, cached_features_file)
+
+    if args.local_rank == 0 and not evaluate:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training process the dataset, and the others will use the cache
+
+    # Convert to Tensors and build dataset
+    all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
+    all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long)
+    all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long)
+    if output_mode == "classification":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.long)
+    elif output_mode == "regression":
+        all_labels = torch.tensor([f.label for f in features], dtype=torch.float)
+
+    dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels)
+    return dataset
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    ## Required parameters
+    parser.add_argument("--data_dir", default=None, type=str, required=True,
+                        help="The input data dir. Should contain the .tsv files (or other data files) for the task.")
+    parser.add_argument("--model_type", default=None, type=str, required=True,
+                        help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys()))
+    parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
+                        help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS))
+    parser.add_argument("--task_name", default=None, type=str, required=True,
+                        help="The name of the task to train selected in the list: " + ", ".join(processors.keys()))
+    parser.add_argument("--output_dir", default=None, type=str, required=True,
+                        help="The output directory where the model predictions and checkpoints will be written.")
+
+    ## Other parameters
+    parser.add_argument("--config_name", default="", type=str,
+                        help="Pretrained config name or path if not the same as model_name")
+    parser.add_argument("--tokenizer_name", default="", type=str,
+                        help="Pretrained tokenizer name or path if not the same as model_name")
+    parser.add_argument("--cache_dir", default="", type=str,
+                        help="Where do you want to store the pre-trained models downloaded from s3")
+    parser.add_argument("--max_seq_length", default=128, type=int,
+                        help="The maximum total input sequence length after tokenization. Sequences longer "
+                             "than this will be truncated, sequences shorter will be padded.")
+    parser.add_argument("--do_train", action='store_true',
+                        help="Whether to run training.")
+    parser.add_argument("--do_eval", action='store_true',
+                        help="Whether to run eval on the dev set.")
+    parser.add_argument("--evaluate_during_training", action='store_true',
+                        help="Rul evaluation during training at each logging step.")
+    parser.add_argument("--do_lower_case", action='store_true',
+                        help="Set this flag if you are using an uncased model.")
+
+    parser.add_argument("--per_gpu_train_batch_size", default=8, type=int,
+                        help="Batch size per GPU/CPU for training.")
+    parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int,
+                        help="Batch size per GPU/CPU for evaluation.")
+    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
+                        help="Number of updates steps to accumulate before performing a backward/update pass.")
+    parser.add_argument("--learning_rate", default=5e-5, type=float,
+                        help="The initial learning rate for Adam.")
+    parser.add_argument("--weight_decay", default=0.0, type=float,
+                        help="Weight deay if we apply some.")
+    parser.add_argument("--adam_epsilon", default=1e-8, type=float,
+                        help="Epsilon for Adam optimizer.")
+    parser.add_argument("--max_grad_norm", default=1.0, type=float,
+                        help="Max gradient norm.")
+    parser.add_argument("--num_train_epochs", default=3.0, type=float,
+                        help="Total number of training epochs to perform.")
+    parser.add_argument("--max_steps", default=-1, type=int,
+                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
+    parser.add_argument("--warmup_steps", default=0, type=int,
+                        help="Linear warmup over warmup_steps.")
+
+    parser.add_argument('--logging_steps', type=int, default=6000,
+                        help="Log every X updates steps.")
+    parser.add_argument('--save_steps', type=int, default=6000,
+                        help="Save checkpoint every X updates steps.")
+    parser.add_argument("--eval_all_checkpoints", action='store_true',
+                        help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number")
+    parser.add_argument("--no_cuda", action='store_true',
+                        help="Avoid using CUDA when available")
+    parser.add_argument('--overwrite_output_dir', action='store_true',
+                        help="Overwrite the content of the output directory")
+    parser.add_argument('--overwrite_cache', action='store_true',
+                        help="Overwrite the cached training and evaluation sets")
+    parser.add_argument('--seed', type=int, default=42,
+                        help="random seed for initialization")
+
+    parser.add_argument('--tpu', action='store_true',
+                        help="Whether to run on the TPU defined in the environment variables")
+    parser.add_argument('--tpu_ip_address', type=str, default='',
+                        help="TPU IP address if none are set in the environment variables")
+    parser.add_argument('--tpu_name', type=str, default='',
+                        help="TPU name if none are set in the environment variables")
+    parser.add_argument('--xrt_tpu_config', type=str, default='',
+                        help="XRT TPU config if none are set in the environment variables")
+
+    parser.add_argument('--fp16', action='store_true',
+                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
+    parser.add_argument('--fp16_opt_level', type=str, default='O1',
+                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
+                             "See details at https://nvidia.github.io/apex/amp.html")
+    parser.add_argument("--local_rank", type=int, default=-1,
+                        help="For distributed training: local_rank")
+    parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.")
+    parser.add_argument('--server_port', type=str, default='', help="For distant debugging.")
+    args = parser.parse_args()
+
+    if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir:
+        raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir))
+
+    # Setup distant debugging if needed
+    if args.server_ip and args.server_port:
+        # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
+        import ptvsd
+        print("Waiting for debugger attach")
+        ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
+        ptvsd.wait_for_attach()
+
+    # Setup CUDA, GPU & distributed training
+    if args.local_rank == -1 or args.no_cuda:
+        device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
+        args.n_gpu = torch.cuda.device_count()
+    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
+        torch.cuda.set_device(args.local_rank)
+        device = torch.device("cuda", args.local_rank)
+        torch.distributed.init_process_group(backend='nccl')
+        args.n_gpu = 1
+    args.device = device
+
+    if args.tpu:
+        if args.tpu_ip_address:
+            os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address
+        if args.tpu_name:
+            os.environ["TPU_NAME"] = args.tpu_name
+        if args.xrt_tpu_config:
+            os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config
+
+        assert "TPU_IP_ADDRESS" in os.environ
+        assert "TPU_NAME" in os.environ
+        assert "XRT_TPU_CONFIG" in os.environ
+
+        import torch_xla
+        import torch_xla.core.xla_model as xm
+        args.device = xm.xla_device()
+        args.xla_model = xm
+
+    # Setup logging
+    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
+                        datefmt = '%m/%d/%Y %H:%M:%S',
+                        level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN)
+    logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
+                    args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16)
+
+    # Set seed
+    set_seed(args)
+
+    # Prepare GLUE task
+    args.task_name = args.task_name.lower()
+    if args.task_name not in processors:
+        raise ValueError("Task not found: %s" % (args.task_name))
+    processor = processors[args.task_name]()
+    args.output_mode = output_modes[args.task_name]
+    label_list = processor.get_labels()
+    num_labels = len(label_list)
+
+    # Load pretrained model and tokenizer
+    if args.local_rank not in [-1, 0]:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    args.model_type = args.model_type.lower()
+    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path,
+                                          num_labels=num_labels,
+                                          finetuning_task=args.task_name,
+                                          cache_dir=args.cache_dir if args.cache_dir else None)
+    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,
+                                                do_lower_case=args.do_lower_case,
+                                                cache_dir=args.cache_dir if args.cache_dir else None)
+    model = model_class.from_pretrained(args.model_name_or_path,
+                                        from_tf=bool('.ckpt' in args.model_name_or_path),
+                                        config=config,
+                                        cache_dir=args.cache_dir if args.cache_dir else None)
+
+    if args.local_rank == 0:
+        torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab
+
+    model.to(args.device)
+
+    logger.info("Training/evaluation parameters %s", args)
+
+
+    # Training
+    if args.do_train:
+        train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False)
+        global_step, tr_loss = train(args, train_dataset, model, tokenizer)
+        logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
+
+
+    # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained()
+    if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu:
+        # Create output directory if needed
+        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
+            os.makedirs(args.output_dir)
+
+        logger.info("Saving model checkpoint to %s", args.output_dir)
+        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
+        # They can then be reloaded using `from_pretrained()`
+        model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
+        model_to_save.save_pretrained(args.output_dir)
+        tokenizer.save_pretrained(args.output_dir)
+
+        # Good practice: save your training arguments together with the trained model
+        torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))
+
+        # Load a trained model and vocabulary that you have fine-tuned
+        model = model_class.from_pretrained(args.output_dir)
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir)
+        model.to(args.device)
+
+
+    # Evaluation
+    results = {}
+    if args.do_eval and args.local_rank in [-1, 0]:
+        tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case)
+        checkpoints = [args.output_dir]
+        if args.eval_all_checkpoints:
+            checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True)))
+            logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN)  # Reduce logging
+        logger.info("Evaluate the following checkpoints: %s", checkpoints)
+        for checkpoint in checkpoints:
+            global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else ""
+            prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else ""
+            
+            model = model_class.from_pretrained(checkpoint)
+            model.to(args.device)
+            result = evaluate(args, model, tokenizer, prefix=prefix)
+            result = dict((k + '_{}'.format(global_step), v) for k, v in result.items())
+            results.update(result)
+
+    return results
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/analysis/time_statistics.py b/analysis/time_statistics.py
new file mode 100644
index 0000000..06be42b
--- /dev/null
+++ b/analysis/time_statistics.py
@@ -0,0 +1,129 @@
+import os
+from datetime import datetime
+import random
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer 
+import seaborn as sns
+from sklearn.decomposition import LatentDirichletAllocation as LDA
+
+from dataset_statistics import plot_n_most_common_words, EXT_TYPES
+  
+random.seed(42)
+np.random.seed(42)
+ 
+def sentiment_scores(sid_obj: SentimentIntensityAnalyzer, sentence: str) -> int: 
+    sentiment_dict = sid_obj.polarity_scores(sentence) 
+    if sentiment_dict['compound'] >= 0.05 : 
+        # print("positive", sentence)
+        return 1
+    elif sentiment_dict['compound'] <= - 0.05 : 
+        # print("negative", sentence)
+        return -1
+    else:
+        # print("neutral", sentence) 
+        return 0
+
+
+def get_sentiment(df, data_name):
+    sid_obj = SentimentIntensityAnalyzer() 
+    sentiments = np.array([sentiment_scores(sid_obj, sentence) for sentence in df["joke"].dropna().values])
+    results = dict(zip(*np.unique(sentiments, return_counts=True)))
+    pos_prop = results[-1] / len(sentiments)
+    neutral_prop = results[0] / len(sentiments)
+    neg_prop = results[1] / len(sentiments)
+
+    # std for a proportion is sqrt(p*q / n)
+    pos_std = np.sqrt((pos_prop * (1 - pos_prop)) / len(sentiments))
+    neutral_std = np.sqrt((neutral_prop * (1 - neutral_prop)) / len(sentiments))
+    neg_std = np.sqrt((neg_prop * (1 - neg_prop)) / len(sentiments))
+
+    results = {"positive_prop": pos_prop, "positive_std": pos_std, "neutral_prop": neutral_prop, "neutral_std": neutral_std, "negative_prop": neg_prop,
+                "negative_std": neg_std}
+
+    if "date" not in df.columns:
+        return results
+    else:
+        list_of_results = []
+        for prop_type in ["positive", "neutral", "negative"]:
+            name = prop_type + "_prop"
+            name_std = prop_type + "_std"
+            list_of_results.append({"prop": results[name], "sentiment": prop_type, "std": results[name_std], "date": df.date.iloc[0]})
+        return pd.DataFrame(list_of_results)
+
+
+def get_scores(df, data_name):
+    return df[["date", "score"]]
+
+def get_jokes(df, data_name):
+    return df[["date", "joke"]]
+
+def split_by_year(df: pd.DataFrame, func, data_name="all") -> pd.DataFrame:
+    df["date"] = df["date"].map(lambda x: pd.to_datetime(datetime.utcfromtimestamp(x)).year)
+    date_df = df.groupby("date").apply(lambda x: func(x, data_name))
+    return date_df
+
+def plot_lines_by_year(date_df: pd.DataFrame, data_name: str, name: str = "sentiment"):
+    for ext_type in EXT_TYPES:
+        ax = sns.lineplot(x="date", y="prop", hue="sentiment", data=date_df, palette=["C2", "C7", "C3"]) 
+        ax.errorbar(date_df["date"], date_df["prop"], yerr=date_df["std"], fmt='o', ecolor='black')
+        plt.xlabel('Year')
+        plt.ylabel('Proportion of Sentiment')
+        plt.savefig(os.path.join(os.path.realpath('..'), "plots", data_name, "{}.{}".format(name, ext_type)))
+        plt.close()
+
+def plot_distribution_by_year(date_df: pd.DataFrame, data_name: str, name: str = "score"):
+    date_df = date_df[date_df["score"] != 0] # zeros skew the distribution
+    for ext_type in EXT_TYPES:
+        ax = sns.violinplot(x="date", y="score", data=date_df)
+        plt.xlabel('Year')
+        plt.ylabel('Distribution of Jokes')
+        plt.savefig(os.path.join(os.path.realpath('..'), "plots", data_name, "{}.{}".format(name, ext_type)))
+        plt.close()
+
+def plot_bar_by_year(df: pd.DataFrame, data_name: str, name: str):
+    if name == "mean":
+        ave = df.groupby('date').mean().reset_index(drop=True)
+        ave.columns = ["average"]
+        max = df.groupby('date').max().reset_index()
+        max.columns = ["date", "score"]
+        data = pd.concat([max, ave], axis=1)
+    elif name == "count":
+        data = df.groupby('date').count().reset_index()
+    elif name == "max":
+        data = df.groupby('date').max().reset_index()
+    else:
+        raise NotImplementedError()
+
+    for ext_type in EXT_TYPES:
+        ax = sns.barplot(x="date", y="score", data=data)
+        for index, p in enumerate(ax.patches):
+            height = p.get_height()
+            ax.text(p.get_x()+p.get_width()/2.,
+                    height + 3,
+                    # will use average for numbering in max case
+                    '{}'.format(int(data["score"].iloc[index]) if name in ["count", "max"] else int(data["average"].iloc[index])),
+                    ha="center") 
+        plt.xlabel('Year')
+        plt.ylabel(name)
+        plt.tight_layout()
+        plt.savefig(os.path.join(os.path.realpath('..'), "plots", data_name, "{}.{}".format(name, ext_type)))
+        plt.close()
+
+
+if __name__ == "__main__" : 
+    df = pd.read_csv(os.path.join(os.path.realpath('..'), "data", "preprocessed.csv"), index_col=None, encoding="UTF-8", keep_default_na=False)
+    df["date"] = pd.to_numeric(df["date"])
+    df["score"] = pd.to_numeric(df["score"])
+    df = df[df["date"].isna() == False]
+
+    plot_bar_by_year(split_by_year(df.copy(deep=True), get_scores), "all", "max")
+    plot_bar_by_year(split_by_year(df.copy(deep=True), get_scores), "all", "count")
+    plot_lines_by_year(split_by_year(df.copy(deep=True), get_sentiment, "all"), "all", "sentiment")
+    plot_distribution_by_year(split_by_year(df.copy(deep=True), get_scores, "all"), "all", "score_distributions")
+    # plot top entities per year
+    jokes_by_year = split_by_year(df.copy(deep=True), get_jokes, "all")
+    for year in jokes_by_year["date"].unique():
+        plot_n_most_common_words(jokes_by_year[jokes_by_year["date"] == year]["joke"].tolist(), data_name="yearbyyear", specific_year=year)
diff --git a/data/dev.tsv.gz b/data/dev.tsv.gz
new file mode 100644
index 0000000..aaaf3cd
Binary files /dev/null and b/data/dev.tsv.gz differ
diff --git a/data/fullrjokes.json.gz b/data/fullrjokes.json.gz
new file mode 100644
index 0000000..a01ed70
Binary files /dev/null and b/data/fullrjokes.json.gz differ
diff --git a/data/preprocessed.csv.gz b/data/preprocessed.csv.gz
new file mode 100644
index 0000000..d945439
Binary files /dev/null and b/data/preprocessed.csv.gz differ
diff --git a/data/submissions.json.gz b/data/submissions.json.gz
new file mode 100644
index 0000000..43edf43
Binary files /dev/null and b/data/submissions.json.gz differ
diff --git a/data/test.tsv.gz b/data/test.tsv.gz
new file mode 100644
index 0000000..50f1462
Binary files /dev/null and b/data/test.tsv.gz differ
diff --git a/data/train.tsv.gz b/data/train.tsv.gz
new file mode 100644
index 0000000..c0be014
Binary files /dev/null and b/data/train.tsv.gz differ
diff --git a/download_nltk_packages.sh b/download_nltk_packages.sh
new file mode 100644
index 0000000..afc7367
--- /dev/null
+++ b/download_nltk_packages.sh
@@ -0,0 +1,4 @@
+python3 -m nltk.downloader averaged_perceptron_tagger
+python3 -m nltk.downloader words
+python3 -m nltk.downloader stopwords
+python3 -m nltk.downloader maxent_ne_chunker
diff --git a/plots/all/count.pdf b/plots/all/count.pdf
new file mode 100644
index 0000000..37af1ee
Binary files /dev/null and b/plots/all/count.pdf differ
diff --git a/plots/all/count.png b/plots/all/count.png
new file mode 100644
index 0000000..11021c5
Binary files /dev/null and b/plots/all/count.png differ
diff --git a/plots/all/logdist.pdf b/plots/all/logdist.pdf
new file mode 100644
index 0000000..c0f330f
Binary files /dev/null and b/plots/all/logdist.pdf differ
diff --git a/plots/all/logdist.png b/plots/all/logdist.png
new file mode 100644
index 0000000..64329c4
Binary files /dev/null and b/plots/all/logdist.png differ
diff --git a/plots/all/max.pdf b/plots/all/max.pdf
new file mode 100644
index 0000000..e693bd0
Binary files /dev/null and b/plots/all/max.pdf differ
diff --git a/plots/all/max.png b/plots/all/max.png
new file mode 100644
index 0000000..383626f
Binary files /dev/null and b/plots/all/max.png differ
diff --git a/plots/all/percentiles.csv b/plots/all/percentiles.csv
new file mode 100644
index 0000000..9a30ee7
--- /dev/null
+++ b/plots/all/percentiles.csv
@@ -0,0 +1,8 @@
+,percentile,value
+0,0,0.0
+1,10,0.0
+2,25,1.0
+3,50,5.0
+4,75,20.0
+5,90,103.0
+6,100,1541018587.0
diff --git a/plots/all/score_distributions.pdf b/plots/all/score_distributions.pdf
new file mode 100644
index 0000000..b8e8355
Binary files /dev/null and b/plots/all/score_distributions.pdf differ
diff --git a/plots/all/score_distributions.png b/plots/all/score_distributions.png
new file mode 100644
index 0000000..d277fea
Binary files /dev/null and b/plots/all/score_distributions.png differ
diff --git a/plots/all/sentiment.pdf b/plots/all/sentiment.pdf
new file mode 100644
index 0000000..d5b960a
Binary files /dev/null and b/plots/all/sentiment.pdf differ
diff --git a/plots/all/sentiment.png b/plots/all/sentiment.png
new file mode 100644
index 0000000..f40d8b5
Binary files /dev/null and b/plots/all/sentiment.png differ
diff --git a/plots/all/statistics.txt b/plots/all/statistics.txt
new file mode 100644
index 0000000..bec6d9e
--- /dev/null
+++ b/plots/all/statistics.txt
@@ -0,0 +1,2 @@
+,ave_punchline_len,ave_body_len,ave_joke_len,std_punch,std_body,std_joke,total_tokens
+0,47.84236006473576,191.93603856241978,239.7849140577041,25.90976722809782,502.55412523216035,501.56866351178525,256619
diff --git a/plots/all/top_wordcounts.pdf b/plots/all/top_wordcounts.pdf
new file mode 100644
index 0000000..e57f8c3
Binary files /dev/null and b/plots/all/top_wordcounts.pdf differ
diff --git a/plots/all/top_wordcounts.png b/plots/all/top_wordcounts.png
new file mode 100644
index 0000000..48e285c
Binary files /dev/null and b/plots/all/top_wordcounts.png differ
diff --git a/plots/all/total_score_distribution.pdf b/plots/all/total_score_distribution.pdf
new file mode 100644
index 0000000..1b0d3b9
Binary files /dev/null and b/plots/all/total_score_distribution.pdf differ
diff --git a/plots/all/total_score_distribution.png b/plots/all/total_score_distribution.png
new file mode 100644
index 0000000..f44d5d5
Binary files /dev/null and b/plots/all/total_score_distribution.png differ
diff --git a/plots/all/wordCloud.pdf b/plots/all/wordCloud.pdf
new file mode 100644
index 0000000..a5f6e0f
Binary files /dev/null and b/plots/all/wordCloud.pdf differ
diff --git a/plots/all/wordCloud.png b/plots/all/wordCloud.png
new file mode 100644
index 0000000..bff229e
Binary files /dev/null and b/plots/all/wordCloud.png differ
diff --git a/plots/all/wordCloudSmall.pdf b/plots/all/wordCloudSmall.pdf
new file mode 100644
index 0000000..9032a78
Binary files /dev/null and b/plots/all/wordCloudSmall.pdf differ
diff --git a/plots/yearbyyear/2008_top_wordcounts.pdf b/plots/yearbyyear/2008_top_wordcounts.pdf
new file mode 100644
index 0000000..ffcd2fb
Binary files /dev/null and b/plots/yearbyyear/2008_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2008_top_wordcounts.png b/plots/yearbyyear/2008_top_wordcounts.png
new file mode 100644
index 0000000..d1b064b
Binary files /dev/null and b/plots/yearbyyear/2008_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2009_top_wordcounts.pdf b/plots/yearbyyear/2009_top_wordcounts.pdf
new file mode 100644
index 0000000..38a89b2
Binary files /dev/null and b/plots/yearbyyear/2009_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2009_top_wordcounts.png b/plots/yearbyyear/2009_top_wordcounts.png
new file mode 100644
index 0000000..014e970
Binary files /dev/null and b/plots/yearbyyear/2009_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2010_top_wordcounts.pdf b/plots/yearbyyear/2010_top_wordcounts.pdf
new file mode 100644
index 0000000..c70c230
Binary files /dev/null and b/plots/yearbyyear/2010_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2010_top_wordcounts.png b/plots/yearbyyear/2010_top_wordcounts.png
new file mode 100644
index 0000000..c2ec9df
Binary files /dev/null and b/plots/yearbyyear/2010_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2011_top_wordcounts.pdf b/plots/yearbyyear/2011_top_wordcounts.pdf
new file mode 100644
index 0000000..3f92223
Binary files /dev/null and b/plots/yearbyyear/2011_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2011_top_wordcounts.png b/plots/yearbyyear/2011_top_wordcounts.png
new file mode 100644
index 0000000..3cd671c
Binary files /dev/null and b/plots/yearbyyear/2011_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2012_top_wordcounts.pdf b/plots/yearbyyear/2012_top_wordcounts.pdf
new file mode 100644
index 0000000..a64f1fd
Binary files /dev/null and b/plots/yearbyyear/2012_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2012_top_wordcounts.png b/plots/yearbyyear/2012_top_wordcounts.png
new file mode 100644
index 0000000..a00dd8a
Binary files /dev/null and b/plots/yearbyyear/2012_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2013_top_wordcounts.pdf b/plots/yearbyyear/2013_top_wordcounts.pdf
new file mode 100644
index 0000000..ecc7330
Binary files /dev/null and b/plots/yearbyyear/2013_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2013_top_wordcounts.png b/plots/yearbyyear/2013_top_wordcounts.png
new file mode 100644
index 0000000..12cc65b
Binary files /dev/null and b/plots/yearbyyear/2013_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2014_top_wordcounts.pdf b/plots/yearbyyear/2014_top_wordcounts.pdf
new file mode 100644
index 0000000..e9a677b
Binary files /dev/null and b/plots/yearbyyear/2014_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2014_top_wordcounts.png b/plots/yearbyyear/2014_top_wordcounts.png
new file mode 100644
index 0000000..b9cf381
Binary files /dev/null and b/plots/yearbyyear/2014_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2015_top_wordcounts.pdf b/plots/yearbyyear/2015_top_wordcounts.pdf
new file mode 100644
index 0000000..31c11a2
Binary files /dev/null and b/plots/yearbyyear/2015_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2015_top_wordcounts.png b/plots/yearbyyear/2015_top_wordcounts.png
new file mode 100644
index 0000000..ac3fc4f
Binary files /dev/null and b/plots/yearbyyear/2015_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2016_top_wordcounts.pdf b/plots/yearbyyear/2016_top_wordcounts.pdf
new file mode 100644
index 0000000..0cc937f
Binary files /dev/null and b/plots/yearbyyear/2016_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2016_top_wordcounts.png b/plots/yearbyyear/2016_top_wordcounts.png
new file mode 100644
index 0000000..6394313
Binary files /dev/null and b/plots/yearbyyear/2016_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2017_top_wordcounts.pdf b/plots/yearbyyear/2017_top_wordcounts.pdf
new file mode 100644
index 0000000..5f825f0
Binary files /dev/null and b/plots/yearbyyear/2017_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2017_top_wordcounts.png b/plots/yearbyyear/2017_top_wordcounts.png
new file mode 100644
index 0000000..17f33f6
Binary files /dev/null and b/plots/yearbyyear/2017_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2018_top_wordcounts.pdf b/plots/yearbyyear/2018_top_wordcounts.pdf
new file mode 100644
index 0000000..2fa93ea
Binary files /dev/null and b/plots/yearbyyear/2018_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2018_top_wordcounts.png b/plots/yearbyyear/2018_top_wordcounts.png
new file mode 100644
index 0000000..ed16fbb
Binary files /dev/null and b/plots/yearbyyear/2018_top_wordcounts.png differ
diff --git a/plots/yearbyyear/2019_top_wordcounts.pdf b/plots/yearbyyear/2019_top_wordcounts.pdf
new file mode 100644
index 0000000..9d2fd37
Binary files /dev/null and b/plots/yearbyyear/2019_top_wordcounts.pdf differ
diff --git a/plots/yearbyyear/2019_top_wordcounts.png b/plots/yearbyyear/2019_top_wordcounts.png
new file mode 100644
index 0000000..1855004
Binary files /dev/null and b/plots/yearbyyear/2019_top_wordcounts.png differ
diff --git a/prepare_data/gather_reddit_pushshift.py b/prepare_data/gather_reddit_pushshift.py
new file mode 100644
index 0000000..9e21afc
--- /dev/null
+++ b/prepare_data/gather_reddit_pushshift.py
@@ -0,0 +1,88 @@
+"""
+Code framework taken from:
+    https://www.osrsbox.com/blog/2019/03/18/watercooler-scraping-an-entire-subreddit-2007scape/
+"""
+
+import requests
+import json
+import re
+import time
+import datetime
+import os
+from dateutil.relativedelta import relativedelta
+
+PUSHSHIFT_REDDIT_URL = "http://api.pushshift.io/reddit"
+
+def fetchObjects(**kwargs):
+    # Default paramaters for API query
+    params = {
+        "sort_type":"created_utc",
+        "sort":"asc",
+        "size":1000,
+        "filter": ("id", "created_utc")
+        }
+
+    # Add additional paramters based on function arguments
+    for key,value in kwargs.items():
+        params[key] = value
+
+    # Print API query paramaters
+    print(params)
+
+    # Set the type variable based on function input
+    # The type can be "comment" or "submission", default is "comment"
+    type_post = "comment"
+    if 'type' in kwargs and kwargs['type'].lower() == "submission":
+        type_post = "submission"
+    
+    # Perform an API request
+    r = requests.get(PUSHSHIFT_REDDIT_URL + "/" + type_post + "/search/", params=params, timeout=30)
+
+    # Check the status code, if successful, process the data
+    if r.status_code == 200:
+        response = json.loads(r.text)
+        data = response['data']
+        sorted_data_by_id = sorted(data, key=lambda x: int(x['id'],36))
+        return sorted_data_by_id
+    
+    else:
+        return []
+
+def extract_reddit_data(**kwargs):
+    # Speficify the start timestamp, make sure we get everything
+    max_created_utc = int((datetime.datetime.now() - relativedelta(years=100)).timestamp())
+    max_id = 0
+
+    # Open a file for JSON output
+    file = open(os.path.join(os.path.realpath('..'), "data", "submissions.json"), "w")
+
+    # While loop for recursive function
+    while 1:
+        nothing_processed = True
+        # Call the recursive function
+        print("After {}".format(max_created_utc))
+        objects = fetchObjects(**kwargs,after=max_created_utc)
+        
+        # Loop the returned data, ordered by date
+        for object in objects:
+            id = int(object['id'], 36)
+            if id > max_id:
+                nothing_processed = False
+                created_utc = object['created_utc']
+                max_id = id
+                if created_utc > max_created_utc: max_created_utc = created_utc
+                # Output JSON data to the opened file
+                print(json.dumps(object, sort_keys=True, ensure_ascii=True), file=file)
+        
+        # Exit if nothing happened
+        if nothing_processed: 
+            return
+        max_created_utc -= 1
+
+        # Sleep a little before the next recursive function call
+        time.sleep(.5)
+
+# Start program by calling function with:
+# 1) Subreddit specified
+# 2) The type of data required (comment or submission)
+extract_reddit_data(subreddit="jokes",type="submission")
\ No newline at end of file
diff --git a/prepare_data/preprocess.py b/prepare_data/preprocess.py
new file mode 100644
index 0000000..6f82cc2
--- /dev/null
+++ b/prepare_data/preprocess.py
@@ -0,0 +1,170 @@
+import re
+import requests
+import json
+import praw
+import datetime
+import os
+from multiprocessing import Pool
+import time
+import csv
+
+import argparse
+import pandas as pd
+import numpy as np
+import nltk
+from sklearn.model_selection import train_test_split
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+
+reddit = praw.Reddit(client_id='client_id',
+                     client_secret='secret_key',
+                     user_agent="Desciption Here")
+
+random.seed(42)
+np.random.seed(42)
+
+def processExistingPost(doc):
+    """ A helper function to gather data from the Reddit API based on post ID """
+    # create a post object
+    post = reddit.submission(doc['id'])
+    # fetch the updated post from reddit
+    post._fetch()
+    return post
+
+
+def read_and_update_ids(file_name: str = "submissions.json", output_name: str = "fullrjokes.json"):
+    """
+    Gathers the data in the data/{file_name} file and grabs the fields we want to keep, along with the updated
+    upvote count and stores them in data/{output_name}.
+
+    If you want to include other meta-data from the reddit API, you can add it to `KEEP_KEYS` list
+
+    Args:
+        file_name: the name of the file in the `data` folder that contains the Reddit post IDs
+        output_name: the name of the file to output the gathered posts to, in the `data` folder
+    """
+    print("Reading and update jokes for each id...")
+    KEEP_KEYS = ["id", "selftext", "title", "downs", "ups", "score", "name", "created_utc"]
+    with open(os.path.join(os.path.realpath('..'), "data", file_name), "r") as f:
+        for line in f:
+            line = json.loads(line)
+            for retry in range(3):
+                try:
+                    updated_doc = processExistingPost(line)
+                except Exception as e:
+                    print("On retry number", retry)
+                    time.sleep(5)
+                    continue
+                break
+            consalidated_dict = {your_key: updated_doc.__dict__[your_key] for your_key in KEEP_KEYS}
+            with open(os.path.join(os.path.realpath('..'), "data", output_name), "a") as fout:
+                fout.write(json.dumps(consalidated_dict, default=str) + "\n")
+                print("wrote out id={}".format(updated_doc))
+
+
+def phrase_in_doc(phrase, updated_doc):
+    """ A helper function to see if the phrase is in the document body (title) or punchline (selftext) """
+    return phrase in updated_doc["title"] or phrase in updated_doc["selftext"]
+
+
+def read_filter_and_preprocess(file_name: str = "fullrjokes.json", output_name: str = "preprocessed.csv"):
+    """
+    Gathers the data in the data/{file_name} file, preprocesses it to exclude deleted/pic/video/removed/empty
+    posts and stores them in data/{output_name}.
+
+    It also creates the split for the log-upvote regression problem and stores them in `data/{train|dev|test}.tsv`
+
+    Args:
+        file_name: the name of the file in the `data` folder that contains the Reddit posts
+        output_name: the name of the file to output the gathered preprocessed files to
+    """
+    START_OF_2020 = 1577836801 # GMT time, 1/1/2020
+    list_of_jokes = []
+    skipped_deleted = 0
+    skipped_removed = 0
+    skipped_format = 0
+    TEXT_COLS = ["body", "punchline", "joke"]
+    print("Reading and preprocessing all jokes")
+    with open(os.path.join(os.path.realpath('..'), "data", file_name), "r") as f:
+        for index, line in enumerate(f):
+            updated_doc = json.loads(line)
+            if updated_doc["created_utc"] >= START_OF_2020:
+                # NOTE: if analyzing data after 2020, remove this
+                continue
+            if updated_doc["title"] == "" and updated_doc["selftext"] == "":
+                skipped_format += 1
+                continue
+            if phrase_in_doc("[deleted]", updated_doc):
+                skipped_deleted += 1
+                continue
+            if phrase_in_doc("[removed]", updated_doc):
+                skipped_removed += 1
+                continue
+            if pd.isnull(updated_doc["score"]) or type(updated_doc["score"]) == str and updated_doc["score"] == "":
+                skipped_format += 1
+                continue
+            if phrase_in_doc("[pic]", updated_doc) or phrase_in_doc("VIDEO", updated_doc):
+                skipped_format += 1
+                continue
+
+            joke = updated_doc["title"] + " " + updated_doc["selftext"]
+            list_of_jokes.append({"joke": joke, "body": updated_doc["selftext"], 
+                                    "punchline": updated_doc["title"], "score": updated_doc["score"], "date": updated_doc["created_utc"]})
+                        
+    df = pd.DataFrame(list_of_jokes)
+
+    # get rid of newlines
+    df["joke"] = df["joke"].replace(r'\n',' ',regex=True)
+    df["joke"] = df["joke"].replace(r'\r',' ',regex=True)
+    assert df["joke"].isnull().any() == False, "there were NaNs in the joke columns, ERROR"
+    assert df["score"].isnull().any() == False, "there were NaNs in the score columns, ERROR"
+    df = df.dropna()  # drop all jokes that have something funny
+    print("Writing df with shape", df.shape, "to file path", os.path.join(os.path.realpath('..'), "data", output_name))
+    print("SKIPPED: {} were removed, {} deleted, {} format".format(skipped_removed, skipped_deleted, skipped_format))
+    df.to_csv(os.path.join(os.path.realpath('..'), "data", output_name), index=None, encoding="UTF-8")
+
+    # format the data for a regression task
+    # remove data before 1/1/2016 GMT
+    year2016 = 1451606401
+    df = df[df["date"] > year2016]
+    df["score"] = np.log(df["score"]) # use a log scale, make 0's 0 still
+    df = df.replace([np.inf, -np.inf], np.nan)
+    df["score"] = df["score"].fillna(0).astype(int)
+    print("Values are between", df["score"].value_counts())
+    print("For log prediction task, {} unique jokes".format(df.shape))
+    df = df[["score", "joke"]]
+
+    # save log-distribution plots
+    plt.tight_layout()
+    ax = sns.distplot(df["score"], kde=False)
+    ax.set(xlabel='Log Upvotes', ylabel='Frequency')
+    plt.savefig(os.path.join("..", "plots", "all", "logdist.png"), bbox_inches = "tight")
+    plt.savefig(os.path.join("..", "plots", "all", "logdist.pdf"), bbox_inches = "tight")
+    plt.close()
+
+    train, dev = train_test_split(df, test_size=0.2, random_state=42)
+    dev, test = train_test_split(dev, test_size=0.5, random_state=42)
+    print("writing train, dev, and test with shapes", train.shape, dev.shape, test.shape)
+    train.to_csv(os.path.join("..", "data", "train.tsv"), sep="\t", quoting=csv.QUOTE_NONE, escapechar='\\', index=None, header=None, encoding="UTF-8")
+    dev.to_csv(os.path.join("..", "data", "dev.tsv"),  sep="\t", quoting=csv.QUOTE_NONE, escapechar='\\', index=None, header=None, encoding="UTF-8")
+    test.to_csv(os.path.join("..", "data", "test.tsv"),  sep="\t", quoting=csv.QUOTE_NONE, escapechar='\\', index=None, header=None, encoding="UTF-8")
+
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--update", action="store_true", default=False, help="updates the ids for all files in the given file_path")
+    parser.add_argument("--preprocess", action="store_true", default=False, help="preprocesses the given id for all docs in the given file_path")
+    parser.add_argument("--file_name_update", type=str, default="submissions.json", help="updates the ids for all files in the given file_name")
+    parser.add_argument("--output_name_update", type=str, default="fullrjokes.json", help="updates the ids for all files in the given output_name")
+    parser.add_argument("--file_name_preprocess", type=str, default="fullrjokes.json", help="preprocesses for all files in the given file_name")
+    parser.add_argument("--output_name_preprocess", type=str, default="preprocessed.csv", help="preprocesses for all files in the given output_name")
+    args = parser.parse_args()
+
+    if args.update:
+        read_and_update_ids(file_name=args.file_name_update, output_name=args.output_name_update)
+    if args.preprocess:
+        read_filter_and_preprocess(file_name=args.file_name_preprocess, output_name=args.output_name_preprocess)
+    if not args.update and not args.preprocess:
+        raise Exception("Did not give a command to execute...")
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..d16c1c4
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,12 @@
+pandas==0.25.3
+seaborn==0.9.0
+wordcloud==1.5.0
+scikit-learn==0.21.3
+requests==2.22.0
+transformers==2.2.0
+torch==1.2.0
+praw==6.4.0
+pymongo==3.9.0
+nltk==3.4.5
+vaderSentiment==3.2.1
+requests==2.22.0