data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import praw\n",
    "import pandas as pd\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_agent = \"redditScraperMLNLP\"\n",
    "r = praw.Reddit(user_agent=user_agent, client_id=reddit_client_id, client_secret=reddit_client_secret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "depressed_posts_titles = []\n",
    "depressed_posts_content = []\n",
    "depressed_comments = []\n",
    "depressed_dict = {\"title\": [],\n",
    "                 \"id\": [],\n",
    "                 \"num_comments\": [],\n",
    "                 \"comments\": [],\n",
    "                 \"content\": []}\n",
    "non_depressed_dict = {\"title\": [],\n",
    "                     \"id\": [],\n",
    "                     \"num_comments\": [],\n",
    "                     \"comments\": [],\n",
    "                     \"content\": []}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def depressed_data(subreddit):\n",
    "    for post in r.subreddit(subreddit).top(limit=1000):\n",
    "        post.comments.replace_more(limit=100)\n",
    "        depressed_dict['title'].append(post.title)\n",
    "        depressed_dict['id'].append(post.id)\n",
    "        depressed_dict['num_comments'].append(post.num_comments)\n",
    "        comments = post.comments.list()\n",
    "        comments_new = []\n",
    "        for comment in comments:\n",
    "            comments_new.append(comment.body)\n",
    "        depressed_dict['comments'].append(comments_new)\n",
    "        depressed_dict['content'].append(post.selftext)\n",
    "    depressed_df = pd.DataFrame(depressed_dict)\n",
    "    depressed_df.drop_duplicates(subset=['id'], inplace=True)\n",
    "    print(\"Total number of comments: \", sum(depressed_df['num_comments']))\n",
    "    print(depressed_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of comments:  121355\n",
      "                                                  title      id  num_comments  \\\n",
      "0     Reminder: NO ACTIVISM is allowed here at any t...  9er34n            55   \n",
      "1                                 Regular Check-In Post  anh7tv           431   \n",
      "2     Just saying “I have depression” feels like I’m...  apwb7w            42   \n",
      "3     does anyone else hate getting out of bed but a...  apvz7r            30   \n",
      "4                 Stop telling me happiness is a choice  aq13x0            12   \n",
      "5                       Even on this sub I’m worthless.  apugps            76   \n",
      "6     Told my therapist what was actually on my mind...  apykil            14   \n",
      "7     I sit at the Costco food court every Tuesday a...  apxr5h            11   \n",
      "8                         I’m 4 days clean from cutting  apws6i            10   \n",
      "9     Have taken a shower every day for the past wee...  apoglx            79   \n",
      "10                             I miss being able to cry  apynu5            10   \n",
      "11    After a 3 year battle with cancer my mother wa...  apub7v            16   \n",
      "12    My depression isn't just mental/chemical. My l...  apz4gd             9   \n",
      "13    I fear happiness because I'm used to it hardly...  aq1hxw             2   \n",
      "14          Try to not snooze your alarm in the morning  aq0zmy             3   \n",
      "15    Have you ever felt so helpless that you spend ...  apth38            11   \n",
      "16    Going to see a psychologist tomorrow for the f...  apzu9h             4   \n",
      "17                                       Valentines day  apwvav            22   \n",
      "18                                  Depressive episodes  aq0yub             9   \n",
      "19                               I lost my will to live  apwdeo            16   \n",
      "20    I hope that in the afterlife I can still liste...  aq2ynv             9   \n",
      "21                                  Today I made bread!  apxw4f             5   \n",
      "22                         Im hearing voices in my head  aq2s4v             2   \n",
      "23    Once you accept that you have depression it en...  apvqvb            10   \n",
      "24    Does anyone just have days they allow to go past?  apu08r            17   \n",
      "25    Having a pathetic life makes depression worse ...  apxadu             5   \n",
      "26                Goddammit I’m drunk and I miss my ex.  aq2zgh             2   \n",
      "27                                 Am I even depressed?  aq1s4h             4   \n",
      "28                                     I feel so alone.  apyz6b             5   \n",
      "29                             I finally took a shower.  aq27p4             5   \n",
      "...                                                 ...     ...           ...   \n",
      "1950                           My Dad died this morning  5vjprx            63   \n",
      "1951     Being born in a third world country is a curse  6yfdci           136   \n",
      "1952  Does anyone else just sleep all the time becau...  8iq79b            62   \n",
      "1953  I was just hysterically crying my eyes out whi...  73kurd            22   \n",
      "1954  anyone else play videogames all day cuz they c...  6td2x3           131   \n",
      "1955  18 out of 1 million lego blocks are created de...  5bbx24            61   \n",
      "1956                             Asked a girl out today  4wjfum            88   \n",
      "1957  DAE not want to sleep because you don't want t...  102iy9            90   \n",
      "1958  I turned 56 today, and with Father's Day tomor...  8rlizh           122   \n",
      "1959  Anyone here feel like the only reason you're a...  7wffxe           110   \n",
      "1960  Does anyone else just want to move to the midd...  65gf98            82   \n",
      "1961  Anyone else tired all the time, no matter how ...  51tjq3            66   \n",
      "1962                           This one kinda hit hard.   va72i            44   \n",
      "1963                                   \"I'm just tired\"  7pcobv            52   \n",
      "1964  Am I weak for wanting to die, or am I strong f...  739gwq            69   \n",
      "1965  Was anyone else thinking \"Is this the year I'l...  3z1tof           102   \n",
      "1966  Anyone else find that \"talking about it\" doesn...  8moah6           105   \n",
      "1967  I'm better. I really am. But I'd still rather ...  7cvfo9            61   \n",
      "1968  The worst part is being 20 years old, your who...  5acdm4           179   \n",
      "1969  Would anybody else be strongly considering sui...  4in57o           121   \n",
      "1970  Just cleaned the dishes. I'm so proud of mysel...  370xuy            59   \n",
      "1971                      Don't Invite Me to Your Party  2pl0ej            77   \n",
      "1972  Anyone else surprised they haven’t killed them...  afgmin            68   \n",
      "1973                    Today I opened up! Never again!  6yfxy4           196   \n",
      "1974  Does anyone else feel like the world just dema...  68k2qz            96   \n",
      "1975  Found this on tumblr, I liked it, wanted to sh...  1rxhau            60   \n",
      "1976  Anyone else not want to die but are unable to ...  a1ytiw            86   \n",
      "1977                              Today I took a shower  6t30ll            57   \n",
      "1978  has anyone else lost the ability to live in th...  agxmx9            57   \n",
      "1979  I'd never commit suicide but sometimes I wish ...  anlx5x            53   \n",
      "\n",
      "                                               comments  \\\n",
      "0     [Sorry if I'm stupid, but what's activism? I d...   \n",
      "1     [No offense, but really does anyone care?\\n\\nI...   \n",
      "2     [Confession time:\\n\\n&#x200B;\\n\\nFor this exac...   \n",
      "3     [In bed feeling this way right now lol. I have...   \n",
      "4     [Amen. My mom always says this and it pisses m...   \n",
      "5     [There just aren't very many people browsing t...   \n",
      "6     [That's great that you told you therapist and ...   \n",
      "7     [Hang in there. How’d u get suspended? And las...   \n",
      "8     [I’m proud of you for staying clean for four d...   \n",
      "9     [Sounds awesome. Dropping hygiene is usually t...   \n",
      "10    [Same! I just want to cry all the pain out, bu...   \n",
      "11    [Beautiful yet bittersweet post. Life is tough...   \n",
      "12    [Okay, well most of these things you can chang...   \n",
      "13    [All emotions are temporary. No one's happy 24...   \n",
      "14    [I knew that cold showers can help to wake you...   \n",
      "15    [Yep, basically my life for the past few weeks...   \n",
      "16    [Good luck. I wish you the best with this big ...   \n",
      "17    [I have to work that day so i get to see all t...   \n",
      "18    [This happens to me too much. I get 2 days off...   \n",
      "19    [What happened mate?, Same, I don't know you, ...   \n",
      "20    [Same, What kind of music do you listen to?, W...   \n",
      "21    [Baking helps me feel a little lighter. That i...   \n",
      "22    [I have the same feeling, like, almost every p...   \n",
      "23    [I feel the same, then you get bombarded with ...   \n",
      "24    [Same. Just replace days with years., I mostly...   \n",
      "25    [I have similar issues. I also have a lot of b...   \n",
      "26    [I don’t know you personally but you seem like...   \n",
      "27    [Yes, yes you are.\\n\\n\\nAlso, one test doctors...   \n",
      "28    [I’m right there with you. I feel the same way...   \n",
      "29    [From me and the gang, we’re all proud of you....   \n",
      "...                                                 ...   \n",
      "1950  [I'm sorry for your loss, internet hugs to you...   \n",
      "1951  [i live in a third world country too some peop...   \n",
      "1952  [Dude I wish I could sleep , I just woke up fr...   \n",
      "1953  [The weird things that make us happy...  Anyon...   \n",
      "1954  [Yes i do but they don't even bring me any enj...   \n",
      "1955  [At least you weren't born a MegaBlok mate, I ...   \n",
      "1956  [Word of advice that I discovered: Feelings ar...   \n",
      "1957  [I want to sleep, but not to wake up., I'd rat...   \n",
      "1958  [[deleted], I haven't talked to my dad in almo...   \n",
      "1959  [For me, there is a bit of fear of pulling the...   \n",
      "1960  [I always dream of doing this. Find a nice plo...   \n",
      "1961  [That fake smile everyone sees on me every fuc...   \n",
      "1962  [Damn, right in my feelings and stuff. \\n\\nTot...   \n",
      "1963  [literally the go to excuse whenever someone a...   \n",
      "1964  [You are strong, you have always been strong: ...   \n",
      "1965  [I feel you. Haven't had the courage to do so ...   \n",
      "1966  [Yep, definitely. Whenever I tell someone some...   \n",
      "1967  [I don't want to kill my self but if there was...   \n",
      "1968  [23 year old here and I feel you...\\nI keep te...   \n",
      "1969  [Yes immediately. They're the only reason I'm ...   \n",
      "1970  [Awesome!\\nOne step at a time is the best way ...   \n",
      "1971  [This literally made me cry. I've been looking...   \n",
      "1972  [I'm surprised I'm still here too. Was thinkin...   \n",
      "1973  [I feel you\\nForget other people\\nThey never f...   \n",
      "1974  [[deleted], 1. birth\\n2. school\\n3. work\\n4. d...   \n",
      "1975  [> Sometimes depression means Ignoring every p...   \n",
      "1976  [I think I'm like this too, I feel like such a...   \n",
      "1977  [That's wonderful, dude!  Showers can always f...   \n",
      "1978  [Soo relatable, but we guess we gotta try to b...   \n",
      "1979  [Same. I don’t really think about suicide. I t...   \n",
      "\n",
      "                                                content  \n",
      "0     We know that September 10 was World Suicide Pr...  \n",
      "1     Welcome to /r/depression's check-in post - a p...  \n",
      "2     I really hate it when I try to explain to peop...  \n",
      "3     i hate that i'm wasting the day away tossing a...  \n",
      "4     You dont think I'd choose to be happy over fee...  \n",
      "5     Sometimes I post something and no one cares to...  \n",
      "6     I promised on another thread that I would tell...  \n",
      "7     3 years since I started college and I’ve only ...  \n",
      "8     I’ve been 4 days clean from cutting and today ...  \n",
      "9     Hey guys, some of you might remember when I po...  \n",
      "10    I've never been good at crying. I've dealt wit...  \n",
      "11    The pain is absolutely soul crushing. It feels...  \n",
      "12    I'm 30. Never had a girlfriend. Haven't had a ...  \n",
      "13    Throughout my life, I was always let down when...  \n",
      "14    So I just wanted to post this after research t...  \n",
      "15                                                       \n",
      "16                                       Wish me luck‼️  \n",
      "17    Just another reminder of my loneliness, and us...  \n",
      "18    You guys know what I'm talking about.\\nI woke ...  \n",
      "19          I don’t know if I can keep going on in life  \n",
      "20    Music is the only thing that makes sense in th...  \n",
      "21    I have a nasty habit of buying ingredients for...  \n",
      "22    I feel as if the whole world is watching me or...  \n",
      "23    I honestly feel like if I deny my depression I...  \n",
      "24    I'm now single. Unemployed. Mortgage needs pay...  \n",
      "25    I was always melancholic as a kid but the seve...  \n",
      "26    She was manipulative and verbally/emotionally ...  \n",
      "27    * cutting myself\\n* urges to jump off from a b...  \n",
      "28    Even when I’m with people, it’s like I might a...  \n",
      "29    It's been two weeks, but this night I got myse...  \n",
      "...                                                 ...  \n",
      "1950  I was putting dishes in the sink this morning ...  \n",
      "1951  I'm from Bangladesh a third world shit-hole of...  \n",
      "1952  I feel like I spend some days just sleeping as...  \n",
      "1953  It's the first time I've had a genuine laugh i...  \n",
      "1954                                                     \n",
      "1955  Think of how simple a lego block is, and yet s...  \n",
      "1956  I didn't get a date because she was busy. But ...  \n",
      "1957                                                     \n",
      "1958  What really sucks is, since he won't talk to m...  \n",
      "1959  ...that your cowardice is the only thing that'...  \n",
      "1960  I can't realistically do this until I graduate...  \n",
      "1961  I could sleep 8 or 15 hours and still be tired...  \n",
      "1962                                                     \n",
      "1963                         I said this 22 times today  \n",
      "1964                                                     \n",
      "1965  New years kinda gets to me emotionally. I work...  \n",
      "1966  Lately, I've been noticing that in the dialogu...  \n",
      "1967  Years of effort have me a little better. \\n\\nT...  \n",
      "1968  Please let the pain stop, I can't deal w anoth...  \n",
      "1969  perhaps the only thing keeping me from doing i...  \n",
      "1970  The little victories really are the best. \\n\\n...  \n",
      "1971  I.\\n\\nWhen I feel depressed, I don’t want to h...  \n",
      "1972  Did not expect to live this long. I keep setti...  \n",
      "1973  Kinda..\\n\\nI asked my parents what they though...  \n",
      "1974  I just want to get by. I'm not even asking to ...  \n",
      "1975  \"Depression does not always mean\\nBeautiful gi...  \n",
      "1976  I'm 25 and unable to function at all just wond...  \n",
      "1977                                         Feels good  \n",
      "1978  It’s like I’m only watching myself try to get ...  \n",
      "1979  I don't want to commit suicide and I'm too muc...  \n",
      "\n",
      "[1974 rows x 5 columns]\n"
     ]
    }
   ],
   "source": [
    "depressed_data('depression')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def non_depressed_data(subreddit):\n",
    "    for post in r.subreddit(subreddit).hot(limit=1000):\n",
    "        post.comments.replace_more(limit=100)\n",
    "        non_depressed_dict['title'].append(post.title)\n",
    "        non_depressed_dict['id'].append(post.id)\n",
    "        non_depressed_dict['num_comments'].append(post.num_comments)\n",
    "        comments = post.comments.list()\n",
    "        comments_new = []\n",
    "        for comment in comments:\n",
    "            comments_new.append(comment.body)\n",
    "        non_depressed_dict['comments'].append(comments_new)\n",
    "        non_depressed_dict['content'].append(post.selftext)\n",
    "    non_depressed_df = pd.DataFrame(non_depressed_dict)\n",
    "    non_depressed_df.drop_duplicates(subset=['id'], inplace=True)\n",
    "    print(\"Total number of comments: \", sum(non_depressed_df['num_comments']))\n",
    "    print(non_depressed_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of comments:  178277\n",
      "                                                 title      id  num_comments  \\\n",
      "0            What is your high school crush doing now?  apxq09         12682   \n",
      "1    What's the pettiest reason you won't date some...  apw8vf          7161   \n",
      "2    What hobby would you like to give a try but do...  aptikm         26310   \n",
      "3    What’s the best(worst) public relations stunt ...  apuxu7          3192   \n",
      "4     Straight A students: What are your study habits?  aputqt          1260   \n",
      "5    What is something you do alone for fun that 99...  apryy7          6945   \n",
      "6    What to do if you’re bored and a lonely piece ...  apy3vi           599   \n",
      "7        What behaviors instantly kill a conversation?  apyfc5           704   \n",
      "8    What's an \"I'll never be one of those people\" ...  apsiem          2750   \n",
      "9    What childhood injustice still bothers you to ...  apz4mj           564   \n",
      "10   What do people freak out about that really isn...  apvy8z          1536   \n",
      "11   What can a normal person do to help save the b...  apz4cm           118   \n",
      "12   If you had unlimited money, what would you be ...  aq0ljm           499   \n",
      "13   Men who have been raped by or pressured/ trick...  apr9x0          2077   \n",
      "14               What’s your favorite Pokémon and why?  aq0n5s           403   \n",
      "15   If magic was real, what spell would you try fi...  apz13z           414   \n",
      "16            What will the 2020s bring in technology?  apwoqg           399   \n",
      "17          What should you NEVER say on a first date?  aq0ggw           174   \n",
      "18         What’s the cutest date you’ve ever been on?  aq1636           108   \n",
      "19         What's something you should never do naked?  aq1xa7           203   \n",
      "20   If you had a friend who spoke to you in the sa...  aq26sx            34   \n",
      "21   If the Earth had a recent update, what would t...  apwmzz           202   \n",
      "22   If you ran for president, which old social med...  apz0sc           155   \n",
      "23             What can singles do on Valentine's Day?  apu772           590   \n",
      "24   What celebrity (living or dead) would you be t...  aq2vvc            78   \n",
      "25   Redditors who have been seriously upvoted and ...  apywwz           111   \n",
      "26   What's the dumbest thing you were scared of as...  aq0kwq           174   \n",
      "27   Mall Santas of reddit, what's the worst thing ...  aq26nb            22   \n",
      "28   What childish thing do you still enjoy as an a...  aq27d5            85   \n",
      "29   What items/events would be listed in the post ...  aq0127            27   \n",
      "..                                                 ...     ...           ...   \n",
      "940  Why is the standard of female sport (generally...  aq1349            15   \n",
      "941         What does your morning routine consist of?  aq0ee9             9   \n",
      "942  [Serious]How do you build a new value system i...  aq3454             9   \n",
      "943  What is the best way to tell your mom a secret...  aq1y12            17   \n",
      "944  What jobs can humans do and will forever do be...  aq1xxq            12   \n",
      "945  Redditors who have participated in broadcasted...  apxyed             5   \n",
      "946  At what moment did you realize things don't ju...  aq12wl             3   \n",
      "947  Why are normal non modded games called vanilla...  aq33yo            13   \n",
      "948             What do you want printed on a t-shirt?  aq33xh            15   \n",
      "949  What are your strongest arguments against anti...  apzsuw            38   \n",
      "950  Who in the world right now is most accurately ...  aq1xqh             8   \n",
      "951  For those of you who have been homeless, what ...  aq12ol             7   \n",
      "952  What is a game show concept that would be grea...  aq12ns             3   \n",
      "953                          Knock knock. Who’s there?  aq12k5            27   \n",
      "954  What is the dumbest statement you've ever hear...  aq33kh            18   \n",
      "955  What's worse while playing a game? Squeakers o...  aq1xg3             8   \n",
      "956  What is your biggest regret, the choices you m...  aq33gv            15   \n",
      "957  What is the worst thing a past employer has do...  aq12b0             7   \n",
      "958  How much mb/s (download) is a fast internet co...  aq337a            18   \n",
      "959  What’s the worst Valentine’s Day you’ve ever had?  aq11vj            12   \n",
      "960                 How did you first learn about sex?  aq32x2            22   \n",
      "961     What's the greatest Youtube video of all time?  apzrjy            19   \n",
      "962  What is your most popular Reddit post of all t...  aq1wk7             8   \n",
      "963  For those of you that had high school lovers, ...  aq32sy             4   \n",
      "964  Cops of reddit, what’s the worst excuse someon...  apwyet             5   \n",
      "965  What is the creepiest thing you guys have ever...  aq32q3             0   \n",
      "966  What was your Reddit comment/post that unexpec...  aq1wdi            29   \n",
      "967          What person’s name often gets misspelled?  aq11a1            18   \n",
      "968  What would happen if you could impregnate your...  aq32h3             8   \n",
      "969                  What book(s) scared you the most?  aq1w70            18   \n",
      "\n",
      "                                              comments content  \n",
      "0    [His dad developed Huntington’s and his mom le...          \n",
      "1    [She looked like my cousin which I felt was we...          \n",
      "2    [Playing the harp! I wanted to learn so badly ...          \n",
      "3    [Telstra (Australian phone company), promoting...          \n",
      "4    [I used to re-write my class notes every eveni...          \n",
      "5    [I used to take the city bus just to have some...          \n",
      "6    [Back when I was unemployed, I used to do a lo...          \n",
      "7    [When the other side doesn't listen, just wait...          \n",
      "8    [I slept under my desk at work.\\n\\nI had a rul...          \n",
      "9    [Had a question on a test that asked: what doe...          \n",
      "10   [New things. For example, a couple years ago m...          \n",
      "11   [Plant flowers, dont spray insecticides becaus...          \n",
      "12   [Being as low key and unnoticed as I could whi...          \n",
      "13   [When I was 18, I threw a house party. My sist...          \n",
      "14   [Gengar because he’s Gengar, Bulbasaur, 001 in...          \n",
      "15   [I would stop time and take a nap. , Dongo enl...          \n",
      "16   [Deep fakes and video manipulating makes it po...          \n",
      "17   [I love you, “You smell better when you’re sle...          \n",
      "18   [Walked around the zoo, started to downpour, e...          \n",
      "19   [go to an elementary school, Cook bacon on the...          \n",
      "20   [Id punch him in the face for being an asshole...          \n",
      "21   [God's not big on documentation, so,\\n\\n\"Vario...          \n",
      "22   [We are getting close to the MySpace generatio...          \n",
      "23   [Something you'd do any other day. , Well, thi...          \n",
      "24   [Mister Rogers, Weird Al , Robin Williams, Kea...          \n",
      "25   [I stupidly made my original account name my r...          \n",
      "26   [return the slab, That if I slept with my bedr...          \n",
      "27   [This request scared me, his dad looked like a...          \n",
      "28   [Lego, I like to watch kids shows when I'm str...          \n",
      "29   [Ryan started the fire!, \"Y2K, overblown... 9/...          \n",
      "..                                                 ...     ...  \n",
      "940  [Biology , Men and women are physically differ...          \n",
      "941  [Regretting my life choices for half an hour f...          \n",
      "942  [**Attention! [Serious] Tag Notice**\\n\\n* Joke...          \n",
      "943  [She can promise she won't get mad, but she wi...          \n",
      "944  [Reproduction , Not necessarily a job but sper...          \n",
      "945  [You don't even have to take part to know that...          \n",
      "946  [When it had been years and I was still dealin...          \n",
      "947  [Chocolate ain't a game, As complex as vanilla...          \n",
      "948  [The phrase “more trees, less assholes”, \"Remi...          \n",
      "949  [Measles, mumps, polio, cholera, smallpocks......          \n",
      "950  [The simpsons, Elongated Muskrat, Whoever is c...          \n",
      "951  [Not all homeless people are bad people  Somet...          \n",
      "952  [Blackmail. The production crew does a deep di...          \n",
      "953  [FBI! OPEN UP, Nobody, take your medicine., DE...          \n",
      "954  [\"Donald Trump wont get elected\", When was 9/1...          \n",
      "955  [Music into mics. Fuck off, no one wants to he...          \n",
      "956  [I made a video that got spread which embarras...          \n",
      "957  [I remember once I used to deliver pizza's whe...          \n",
      "958  [Can you watch 4K porn? If not, it’s too slow,...          \n",
      "959  [One time in Elementary school (6th grade bc i...          \n",
      "960  [HBO in the late 80s, Cousin put his dick in m...          \n",
      "961  [How to pick up a blue chair, Grapefruit techn...          \n",
      "962  [https://reddit.app.link/SMn9OADogU, [This one...          \n",
      "963  [It wasn't easy.\\n\\nShe wasn't very into my sc...          \n",
      "964  [Not a cop but when I got pulled over I told t...          \n",
      "965                                                 []          \n",
      "966  [You know the poop knife story?\\n\\nWell, that ...          \n",
      "967  [Michael , Bendastick Summerpatch, Zack,Zach,Z...          \n",
      "968  [I'd use a condom , Get my tubes tied., Clonin...          \n",
      "969  [Salem's Lot, and by extension, Jerusalem's Lo...          \n",
      "\n",
      "[969 rows x 5 columns]\n"
     ]
    }
   ],
   "source": [
    "non_depressed_data('askreddit')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "depressed_pickle = open(\"depressed.pickle\",\"wb\")\n",
    "pickle.dump(depressed_dict, depressed_pickle)\n",
    "depressed_pickle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_depressed_pickle = open(\"depressed.pickle\",\"rb\")\n",
    "test_depressed_pickle_dict = pickle.load(test_depressed_pickle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test Total number of comments:  121355\n"
     ]
    }
   ],
   "source": [
    "depressed_df2 = pd.DataFrame(test_depressed_pickle_dict)\n",
    "depressed_df2.drop_duplicates(subset=['id'], inplace=True)\n",
    "print(\"test Total number of comments: \", sum(depressed_df2['num_comments']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "non_depressed_pickle = open(\"non_depressed.pickle\",\"wb\")\n",
    "pickle.dump(non_depressed_dict, non_depressed_pickle)\n",
    "non_depressed_pickle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test Total number of comments:  178277\n"
     ]
    }
   ],
   "source": [
    "test_non_depressed_pickle = open(\"non_depressed.pickle\",\"rb\")\n",
    "test_non_depressed_pickle_dict = pickle.load(test_non_depressed_pickle)\n",
    "non_depressed_df2 = pd.DataFrame(test_non_depressed_pickle_dict)\n",
    "non_depressed_df2.drop_duplicates(subset=['id'], inplace=True)\n",
    "print(\"test Total number of comments: \", sum(non_depressed_df2['num_comments']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}