company_profiling_classification_prob.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named brief.models",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-bf2f2e0cc55d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'/vagrant/contify-banking'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mbrief\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mBrief\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mentry\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mEntry\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0msettings\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mImportError\u001b[0m: No module named brief.models"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import os, sys\n",
    "from time import time\n",
    "os.environ.setdefault(\"DJANGO_SETTINGS_MODULE\", \"web.settings\")\n",
    "sys.path.append('/vagrant/')\n",
    "sys.path.append('/vagrant/contify-banking')\n",
    "\n",
    "from brief.models import Brief  \n",
    "from entry.models import Entry\n",
    "import settings\n",
    "from django.core.management.base import BaseCommand\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer, CountVectorizer, HashingVectorizer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from sklearn import metrics\n",
    "from sklearn.cluster import KMeans, MiniBatchKMeans\n",
    "import logging\n",
    "from optparse import OptionParser\n",
    "from time import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from random import random\n",
    "import nltk, math\n",
    "from optparse import make_option           \n",
    "from datetime import datetime, timedelta\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.naive_bayes import MultinomialNB, GaussianNB\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.feature_extraction import text "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800\n"
     ]
    }
   ],
   "source": [
    "item = Entry.objects.order_by(\"-approved_on\").filter(title__contains=\"HDFC Bank\")[:800]\n",
    "storyCols = item.values_list('title', 'title', 'title') #, 'body_html')\n",
    "storySoup = [' '.join(cols) for cols in storyCols]\n",
    "storyTitle = item.values_list('title', flat=True)\n",
    "\n",
    "itemTest= Entry.objects.order_by(\"-approved_on\").filter(title__contains=\"HDFC Bank\")[800:1000]\n",
    "storyColsTest = itemTest.values_list('title', 'title', 'title') #, 'body_html')\n",
    "storySoupTest = [' '.join(cols) for cols in storyColsTest]\n",
    "storyTitleTest = itemTest.values_list('title', flat=True)\n",
    "\n",
    "print(storyTitle.count())\n",
    "y_test= Y[800:1000]\n",
    "Y = Y[:800]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 jobs       | elapsed:    0.6s\n",
      "[Parallel(n_jobs=-1)]: Done  50 jobs       | elapsed:   39.6s\n"
     ]
    }
   ],
   "source": [
    "pipeline = Pipeline((\n",
    "    ('hasher', CountVectorizer(stop_words=stopWords, ngram_range = (1,7), max_df=0.2)),                     \n",
    "    ('vec', TfidfTransformer()),\n",
    "    ('lsa', TruncatedSVD(algorithm='arpack', n_components=150)),\n",
    "    ('norm', Normalizer(copy=False)),\n",
    "    ('clf', (GradientBoostingRegressor(n_estimators=100, max_depth=3, loss='ls'))),\n",
    "    #('clf', SGDClassifier(n_iter=10)),\n",
    "))\n",
    "   \n",
    "\n",
    "parameters = {\n",
    "    #'hasher__min_df': [x * 0.00001 for x in range(1, 10)],\n",
    "    'hasher__max_df': [x * 0.1 for x in range(1, 10,2)],\n",
    "    'hasher__ngram_range': [(1, x) for x in range(1, 10,2)],\n",
    "    #'clf__n_iter': [10],\n",
    "    'clf__n_estimators': [25 ,50 ,100],\n",
    "    'clf__max_depth': [1,2, 3, 4],\n",
    "    #'lsa__n_components': [10, 25 ,50 ,100, 150],    \n",
    "}\n",
    "\n",
    "gs = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, refit=True)\n",
    "_ = gs.fit(storySoup, Y)\n",
    "print(gs.best_params_, gs.best_score_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.63504315  0.61776118  0.50406962  0.92741287  0.8128749   0.79939012\n",
      "  0.61544762  0.75582378  0.72032121  0.93780911  0.50287033  0.47282223\n",
      "  0.38022494  0.6154341   0.78727032  0.81513181  0.90214     0.72710358\n",
      "  0.66206412  0.63862987  0.78786669  0.78786669  0.64776226  0.43443863\n",
      "  0.94795297  0.45245065  0.80541623  0.64194989  0.74947143  0.84937045\n",
      "  0.92568597  0.77384091  0.65510273  0.64662608  0.39009732  0.67123098\n",
      "  0.84940275  0.41051903  0.86306352  0.50191681  0.54683408  0.87842189\n",
      "  0.53481913  0.87842189  0.87842189  0.5349384   0.69243732  0.63741469\n",
      "  0.76402261  0.43942502  0.92232634  0.79553558  0.57629014  0.57629014\n",
      "  0.76915032  0.81952795  0.65600077  0.81399996  0.71647933  0.71647933\n",
      "  0.627683    0.77213133  0.71647933  0.71647933  0.71937508  0.71647933\n",
      "  0.71647933  0.71647933  0.66785581  0.71647933  0.71937508  0.71647933\n",
      "  0.64917887  0.90924064  0.83520879  0.96083364  0.64917887  0.74542831\n",
      "  0.8200233   0.80541623  0.47237888  0.66276231  0.79690152  0.89668849\n",
      "  0.84468925  0.56744423  0.71795323  0.89668849  0.84468925  0.87846333\n",
      "  0.461077    0.84798635  0.87846333  0.94880917  0.87846333  0.80815343\n",
      "  0.94880917  0.79687418  0.57459507  0.789861    0.68929165  0.65954897\n",
      "  0.40157817  0.80543582  0.650397    0.79526284  0.75150777  0.51976117\n",
      "  0.79526284  0.50758175  0.57662531  0.43966312  0.73129609  0.69286232\n",
      "  0.73266673  0.51976117  0.79204161  0.70872521  0.650397    0.68746323\n",
      "  0.80796298  0.84612201  0.78604723  0.77358846  0.73655751  0.55098452\n",
      "  0.78794178  0.78441964  0.72927701  0.77358846  0.45078972  0.45078972\n",
      "  0.55098452  0.72927701  0.65196351  0.70213991  0.59870695  0.8736007\n",
      "  0.42417995  0.67430804  0.40157817  0.70760966  0.67430804  0.67108709\n",
      "  0.6195063   0.41746562  0.84798635  0.80216785  0.65208885  0.50158064\n",
      "  0.79903095  0.67536949  0.76225972  0.84798635  0.96537201  0.461077\n",
      "  0.55144864  0.83302834  0.7963095   0.83302834  0.55144864  0.70946557\n",
      "  0.7905065   0.79687418  0.65072518  0.86121406  0.71055086  0.60008709\n",
      "  0.62280451  0.71894474  0.74339105  0.74149447  0.74149447  0.66306796\n",
      "  0.75430236  0.66363944  0.75091831  0.75091831  0.75091831  0.60802388\n",
      "  0.75091831  0.76706065  0.46286498  0.83264493  0.78014231  0.75091831\n",
      "  0.75091831  0.81347488  0.89696369  0.89696369  0.76831263  0.84898945\n",
      "  0.81347488  0.57908794  0.74526423  0.825307    0.83264493  0.81141846\n",
      "  0.81347488  0.76165046]\n",
      "0.67999243942\n",
      "0.00698665447086\n"
     ]
    }
   ],
   "source": [
    "pipeline = Pipeline((\n",
    "    ('hasher', CountVectorizer(stop_words=stopWords, ngram_range = (1,3), max_df=0.3)),                     \n",
    "    ('vec', TfidfTransformer()),\n",
    "    ('lsa', TruncatedSVD(algorithm='arpack', n_components=150)),\n",
    "    ('norm', Normalizer(copy=False)),\n",
    "    #('clf', (LinearSVC())),\n",
    "    ('clf', (GradientBoostingRegressor(n_estimators=100, max_depth=3, loss='ls'))),\n",
    "    #('clf', OneVsRestClassifier(LinearSVC())),\n",
    "))\n",
    "\n",
    "train = pipeline.fit_transform(storySoup, Y)\n",
    "predicted = pipeline.predict(storySoupTest)\n",
    "print(predicted)\n",
    "print(pipeline.score(storySoup, Y))\n",
    "print(pipeline.score(storySoupTest, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "score = 0\n",
    "\"\"\"\n",
    "for s in range(len(storySoupTest)):\n",
    "    if(predicted[s] != y_test[s]):     \n",
    "        print(s, predicted[s], y_test[s], storyTitleTest[s])\n",
    "    score += abs(predicted[s] - y_test[s])\n",
    "print(score) #len(pipeline.get_feature_names()))\n",
    "\"\"\"\n",
    "from sklearn.metrics import confusion_matrix\n",
    "cmx = confusion_matrix(y_test, predicted)\n",
    "print(cmx)\n",
    "#plot\n",
    "import matplotlib.pyplot as plt\n",
    "%pylab inline\n",
    "plt.matshow(cmx)\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#get true items and false items\n",
    "clusteredDict = {}\n",
    "for l in range(len(Y)):\n",
    "    try:\n",
    "        clusteredDict[Y[l]].append(storySoup[l])\n",
    "    except:\n",
    "        clusteredDict.setdefault(Y[l], [storySoup[l]])\n",
    "X_true = clusteredDict[1]\n",
    "X_flase = clusteredDict[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "fit() takes at least 2 arguments (1 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-27-d4e4f5422414>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[1;33m(\u001b[0m\u001b[1;34m'classification'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mExtraTreesClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m ])\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstorySoupTest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: fit() takes at least 2 arguments (1 given)"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "clf = Pipeline([\n",
    "    ('hasher', CountVectorizer(stop_words=stopWords, ngram_range = (1,3), max_df=0.3)),                     \n",
    "    ('vec', TfidfTransformer()),\n",
    "    ('lsa', TruncatedSVD(algorithm='arpack', n_components=150)),\n",
    "    ('norm', Normalizer(copy=False)),\n",
    "    ('feature_selection', LinearSVC()),\n",
    "    ('classification', ExtraTreesClassifier())\n",
    "])\n",
    "f = clf.fit().transform(storySoupTest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Topic extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'numpy.ndarray' object has no attribute 'feature_importances_'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-19-260e3ea0d09a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[0mclf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mGradientBoostingRegressor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_estimators\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_depth\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'ls'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtfidf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeature_importances_\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m \u001b[0mfeature_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_feature_names\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'numpy.ndarray' object has no attribute 'feature_importances_'"
     ]
    }
   ],
   "source": [
    "from sklearn.decomposition import NMF\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "X_true = clusteredDict[1]\n",
    "X_false = clusteredDict[0]\n",
    "vectorizer = TfidfVectorizer(max_df=0.20, ngram_range = (1,7), stop_words=stopWords)\n",
    "tfidf = vectorizer.fit_transform(storySoupTest)\n",
    "\n",
    "clf = GradientBoostingRegressor(n_estimators=100, max_depth=3, loss='ls').fit_transform(tfidf.toarray(), y_test)\n",
    "\n",
    "clf.feature_importances_\n",
    "feature_names = vectorizer.get_feature_names()\n",
    "\n",
    "\"\"\"\n",
    "nmf = NMF(n_components=1, random_state=1).fit(tfidf)\n",
    "\n",
    "\n",
    "for topic_idx, topic in enumerate(nmf.components_):\n",
    "    print(\"Topic #%d:\" % topic_idx)\n",
    "    print(\" \".join([feature_names[i]\n",
    "                    for i in topic.argsort()[:-5 - 1:-1]]))\n",
    "    print()\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.473*\"net\" + 0.434*\"profit\" + 0.285*\"30%\" + 0.238*\"q1\" + 0.220*\"q4\" + 0.212*\"rs\" + 0.187*\"rises\" + 0.178*\"growth\" + 0.171*\"decade\" + 0.166*\"q2\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\ntfidf = models.TfidfModel(corpus) \\ncorpus_tfidf = tfidf[corpus]\\n\\nn_topics = 60\\nlda = models.LdaModel(corpus_tfidf, id2word=dictionary, num_topics=n_topics)\\n\\nfor i in range(0, n_topics):\\n temp = lda.show_topic(i, 10)\\n terms = []\\n for term in temp:\\n     terms.append(term[1])\\n print(\" \" + str(i) + \": \"+ \", \".join(terms))\\n '"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gensim import corpora, models, similarities\n",
    "from itertools import chain\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from operator import itemgetter\n",
    "import re\n",
    "\n",
    "url_pattern = r'https?:\\/\\/(.*[\\r\\n]*)+'\n",
    "\n",
    "documents = [nltk.clean_html(document) for document in X_true]\n",
    "stoplist = stopwords.words('english')\n",
    "texts = [[word for word in document.lower().split() if word not in stoplist]\n",
    " for document in documents]\n",
    "\n",
    "dictionary = corpora.Dictionary(texts)\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "\n",
    "\n",
    "\n",
    "lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=100)\n",
    "t  = lsi.print_topics(1)\n",
    "for to in t:\n",
    "    print((to))\n",
    "\n",
    "\"\"\"\n",
    "tfidf = models.TfidfModel(corpus) \n",
    "corpus_tfidf = tfidf[corpus]\n",
    "\n",
    "n_topics = 60\n",
    "lda = models.LdaModel(corpus_tfidf, id2word=dictionary, num_topics=n_topics)\n",
    "\n",
    "for i in range(0, n_topics):\n",
    " temp = lda.show_topic(i, 10)\n",
    " terms = []\n",
    " for term in temp:\n",
    "     terms.append(term[1])\n",
    " print(\" \" + str(i) + \": \"+ \", \".join(terms))\n",
    " \"\"\"\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "f = open('testoutput.txt', 'wt')\n",
    "for s in range(len(storyTitle)):\n",
    "    f.write(\"%s\\n\" %storyTitle[s].encode('utf8'))\n",
    "f.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}