Added Student Analysis

siesgstarena · Apr 6, 2020 · feec743 · feec743
1 parent 2587eba
commit feec743
Show file tree

Hide file tree

Showing 65 changed files with 2,825,172 additions and 0 deletions.
diff --git a/student_analysis/.gitignore b/student_analysis/.gitignore
@@ -0,0 +1,2 @@
+__pycache__
+.vscode
diff --git a/student_analysis/README.MD b/student_analysis/README.MD
@@ -0,0 +1,64 @@
+# Student Performance Analysis
+
+## About
+This Flask website is a dashboard for keeping track of personal progress on http://arena.siesgst.ac.in/  
+This project gives a coder analysis on the submissions made by him and recommends Problems to him according to Topics solved/unsolved 
+<br>
+<br>
+<b>For Unsolved Topics</b>
+<br>
+System provides problems in increasing difficulty 
+<br>
+<b>For Solved Topics</b>
+<br>
+System provides top 5 problems of Higher  difficulty which has not been solved yet
+
+
+#### This project has been deployed on https://arena-siesgst-dashboard.herokuapp.com/
+
+## Installation
+
+### Install Dependencies
+> <b>pip</b>: pip install -r requirements.txt
+> <br>
+<b>conda</b>:  conda install --yes --file requirements.txt
+
+### Execute Code
+>python main.py
+
+### To view Dashboard (in Browser) goto following link once server starts
+>http://localhost:5000/
+
+#### Layouts are visible properly at 67% zoom in browser
+
+## Dataset
+The Data collected from Databases are stored inside Data folder in the following files
+- users.json
+- submissions.json
+- problems.json
+
+#### Get the Updated Dataset from <http://arena.siesgst.ac.in/labs> and place it 'data' folder
+
+## Results
+![](images/demo.gif)
+
+## Meta
+
+V.Karthikraj 
+- karthikraj.v17@siesgst.ac.in
+- [@LinkedIn](https://www.linkedin.com/in/naidukarthi2193/) 
+
+## Contributing
+
+1. Fork it (<https://github.com/naidukarthi2193/labs/fork>)
+2. Create your feature branch (`git checkout -b feature/fooBar`)
+3. Commit your changes (`git commit -am 'Add some fooBar'`)
+4. Push to the branch (`git push origin feature/fooBar`)
+5. Create a new Pull Request
+
+## TODOs
+- Add Ratings Data and Graphs
+- UI Improvements
+- Responsive design
+- Recommed Blogs according to submissions
+- Add URLs for Recommended Problems
diff --git a/student_analysis/data/README.MD b/student_analysis/data/README.MD
@@ -0,0 +1,76 @@
+## Data Structure for files
+
+### Users
+
+```json
+[
+    {
+        "_id":{"$oid":"5b5c89298114180020d4bcb0"},
+        "notifications":{
+            "updates":true,
+            "activities":true
+            },
+        "name":"User Name",
+        "username":"username",
+        "email":"email@siesgst.ac.in",
+        "about":"about",
+        "ratings":0.0,
+        "createdAt":{"$date":"2018-07-28T15:18:01.849Z"},
+        "updatedAt":{"$date":"2019-01-29T06:23:34.352Z"},
+        "__v":0,
+        "topics":["graphs","dp","geometry","webdev","ml","dl","android","opensource"]
+    }
+]
+```
+
+
+### Submissions
+
+```json
+[
+    {
+        "_id":{"$oid":"5b608157e228ec0020a1fcf5"},
+        "userId":{"$oid":"5b5d756037392f89933e7514"},
+        "problemId":{"$oid":"5b5c8cd7276e2200208fed62"},
+        "contestId":{"$oid":"5b5c89ef30db8a0020962414"},
+        "language":"C++14",
+        "fileContent":"fileContentURL",
+        "time":"0s",
+        "memory":"3452KB",
+        "output":"outputURL",
+        "status":"Accepted",
+        "points":0,
+        "createdAt":{"$date":"2018-07-31T15:33:43.362Z"},
+        "updatedAt":{"$date":"2018-11-11T11:51:18.721Z"},
+        "__v":0,
+        "duringContest":true
+    }
+]
+
+```
+
+### Problems
+
+```json
+[
+    {
+        "_id":{"$oid":"5b5c8cd7276e2200208fed62"},
+        "code":"UNI01",
+        "points":0,
+        "name":"Problem Name",
+        "description":"Problem Description",
+        "explainInput":"Input Explain",
+        "explainOutput":"Output Explain",
+        "example":"Input Data Output Data",
+        "explanation":"",
+        "tags":["adhoc"],
+        "outputFile":"outputFileURL",
+        "inputFile":"inputFileURL",
+        "contestCode":"UNIVERSE",
+        "createdAt":{"$date":"2018-07-28T15:33:43.03Z"},
+        "updatedAt":{"$date":"2018-11-11T14:18:53.641Z"},
+        "__v":0,
+        "constraints":"ConstraintData"
+    }
+]
+```
diff --git a/student_analysis/data/problems.json b/student_analysis/data/problems.json
diff --git a/student_analysis/data/submissions.json b/student_analysis/data/submissions.json
diff --git a/student_analysis/data/users.json b/student_analysis/data/users.json
diff --git a/student_analysis/functions.py b/student_analysis/functions.py
@@ -0,0 +1,190 @@
+import json
+import numpy as np
+import pandas as pd
+from math import ceil
+from sklearn.preprocessing import MultiLabelBinarizer
+from nltk.stem import WordNetLemmatizer 
+from plotly.offline import plot
+import plotly.offline as py
+import plotly.graph_objects as go
+import plotly.figure_factory as ff
+import plotly.express as px
+from plotly.graph_objs import *
+import plotly
+from heapq import nlargest 
+# init_notebook_mode(connected=True) 
+
+lemmatizer = WordNetLemmatizer() 
+mlb = MultiLabelBinarizer()
+
+def preprocess_problem(problems):
+    temp1 = list()
+    for x in problems['tags']:
+        temp = list()
+        for a in x:
+            for z in a.split(","):
+                temp.append(lemmatizer.lemmatize(z.lower().strip()))
+        temp1.append(temp)
+    problems["tags"] = temp1
+    problems = problems.join(pd.DataFrame(mlb.fit_transform(problems["tags"]),
+                              columns=mlb.classes_,index=problems.index))
+    problems=problems.drop(['description', 'explainInput',
+           'explainOutput', 'example', 'explanation', 'tags', 'outputFile',
+           'inputFile', 'contestCode', 'createdAt', 'updatedAt', '__v',
+           'constraints', ],axis=1)
+    problems.points = problems.points.apply(lambda x : ceil(x/100))
+    temp = list()
+    for x in problems['_id']:
+        temp.append(x['$oid'])
+    problems['_id']=temp
+    return problems
+
+def get_submissions(email,users,problems,submissions):
+    user_id = users.loc[users['email'] ==  email]['_id']
+    # print(user_id)
+    personal = submissions.loc[submissions['userId'] ==  user_id.values[0]].reset_index(drop=True)
+    personal = personal.drop(['fileContent','output','createdAt','updatedAt', '__v', 'duringContest'],axis=1)
+    for x1 in ['_id','userId','problemId','contestId']:
+        temp = list()
+        for x in personal[x1]:
+            temp.append(x['$oid'])
+        personal[x1]=temp
+    return personal 
+
+def get_graphs(personal,users,problems,submissions):
+    trace0 = go.Pie(values = list(personal.status.value_counts().to_dict().values()),
+                    labels  = list(personal.status.value_counts().to_dict().keys()),
+                   title="Verdicts"
+                   )
+    data_verdict = [trace0]
+    json_verdict = json.dumps(data_verdict,cls=plotly.utils.PlotlyJSONEncoder)
+    personal_accepted = personal.loc[personal['status']=='Accepted']
+    personal_merged = pd.merge(personal_accepted,preprocess_problem(problems),
+        left_on="problemId",
+         right_on="_id",
+         how="left",
+         copy=True
+        )
+    a = personal_merged.drop(['_id_x', 'userId', 'problemId', 'contestId', 'language', 'time',
+           'memory', 'status', 'points_x', '_id_y', 'code', 'name'],axis=1)
+    te=list()
+    for diff in set(a.points_y):
+        new = a.loc[a['points_y']==diff]
+        temp = list()  
+        for x in list(new.drop(['points_y'],axis=1).columns):
+            try:
+                temp.append(len(new)-new[x].value_counts().to_dict()[0])
+            except:
+                temp.append(new[x].value_counts().to_dict()[1])
+        d = str("Difficulty " + str(diff))
+        trace0 = go.Bar(x =list( new.drop(['points_y'],axis=1).columns ),y =temp,name=d)
+        te.append(trace0)
+    json_topics = json.dumps(te,cls=plotly.utils.PlotlyJSONEncoder)
+
+    return personal_merged,json_verdict,json_topics
+
+def get_topic_ratings(personal,users,problems,submissions):
+    personal_merged = pd.merge(personal,preprocess_problem(problems),
+        left_on="problemId",
+         right_on="_id",
+         how="left",
+         copy=True
+        )
+    personal_topics = personal_merged.drop(['_id_x', 'userId', 'problemId', 'contestId', 'language', 'time',
+            'memory', 'status', 'points_x', '_id_y', 'code', 'points_y', 'name'],axis=1)
+    l = dict()
+    unexplored=list()
+    for x in personal_topics.columns:
+        count = personal_topics[x].loc[personal_topics[x]==1].count()
+        if count!=0:
+            l[x]= count
+        else:
+            unexplored.append(x)
+    srt = sorted(l, key=l.get)
+    b=np.array_split(np.asarray(srt) , 2)
+    return unexplored,list(b[0]),list(b[1])
+
+def n_high_submission(personal,problems,users,submissions):
+    personal_submissions = pd.merge(personal,preprocess_problem(problems),
+        left_on="problemId",
+         right_on="_id",
+         how="left",
+         copy=True
+        )
+
+    submit_types = ['Compilation Error','Time Limit Exceeded', 'Runtime Error', 'Wrong Answer',  'Accepted']
+    nHighest = nlargest(10, personal_submissions.code.value_counts().to_dict(),
+                            key = personal_submissions.code.value_counts().to_dict().get) 
+    l= dict()
+    figures = list()
+    for sub in submit_types:
+        for val in nHighest: 
+            submit_personal = personal_submissions.loc[personal_submissions["status"]==sub]
+
+            l[val] = submit_personal.code.value_counts().to_dict().get(val)
+            trace0 = go.Bar( x = list(l.keys()),y=list(l.values()),name=sub)
+        figures.append(trace0)
+    json_n_highest_submissions = json.dumps(figures,cls=plotly.utils.PlotlyJSONEncoder)
+    return nHighest,json_n_highest_submissions
+
+def get_user_details(users,email,problems,submissions):
+    try:
+        user_id = users.loc[users['email'] ==  email]['_id']
+        data=list()
+        user_info=users.loc[users['email']==email].to_dict()
+        data.append( users.loc[users['email'] ==  email]['_id'])
+        data.append( list(user_info['name'].values())[0])
+        data.append( list(user_info['username'].values())[0])
+        data.append( list(user_info['email'].values())[0])
+        data.append( list(user_info['topics'].values())[0])
+        data.append( set(submissions.loc[submissions['userId'] ==  user_id.values[0]].reset_index(drop=True)['language']))
+        data.append( list(user_info['ratings'].values())[0])
+        return data
+    except:
+        return 0
+
+def get_recommendation(problems,unexplored,good,personal_merged,best):
+    p = preprocess_problem(problems)
+    unexplored_d=dict()
+    if len(unexplored) > 0:
+        for tops in unexplored:
+            sums =  list(p.loc[p[tops]==1]['code'].to_dict().values())
+            l=dict()
+
+            for su in sums:
+                l[su] = list(p.loc[p['code']==su]['points'].to_dict().values())[0]
+            srt = sorted(l, key=l.get)
+            unexplored_d[tops]=srt[0:5]
+    practice_d=dict()
+    if len(good)>0:
+        for tops in good:
+            # print(tops)
+            sums =  list(p.loc[p[tops]==1]['code'].to_dict().values())
+            l=dict()
+
+            for su in sums:
+                l[su] = list(p.loc[p['code']==su]['points'].to_dict().values())[0]
+            srt = sorted(l, key=l.get,reverse=True)
+            srt = set(srt) - set(personal_merged['code'])
+            srt = list(srt)
+            if len(srt)>0:
+                practice_d[tops]=srt[0:5]
+
+
+    best_d=dict()
+    if len(good)>0:
+        for tops in good:
+            # print(tops)
+            sums =  list(p.loc[p[tops]==1]['code'].to_dict().values())
+            l=dict()
+
+            for su in sums:
+                l[su] = list(p.loc[p['code']==su]['points'].to_dict().values())[0]
+            srt = sorted(l, key=l.get,reverse=True)
+            srt = set(srt) - set(personal_merged['code'])
+            srt = list(srt)
+            if len(srt)>0:
+                best_d[tops]=srt[0:5]
+
+    return unexplored_d,practice_d,best_d
+
diff --git a/student_analysis/images/demo.gif b/student_analysis/images/demo.gif
diff --git a/student_analysis/main.py b/student_analysis/main.py
@@ -0,0 +1,56 @@
+from flask import Flask, render_template,request, url_for, redirect
+import plotly
+import plotly.graph_objs as go
+import pandas as pd
+import numpy as np
+import json
+import functions
+
+with open('data/users.json', 'r', errors='ignore') as f:
+        data = json.load(f)
+users = pd.DataFrame(data)
+with open('data/problems.json', 'r', errors='ignore') as f:
+        data = json.load(f)
+problems = pd.DataFrame(data)
+
+with open('data/submissions.json', 'r', errors='ignore') as f:
+        data = json.load(f)
+submissions = pd.DataFrame(data)
+
+app = Flask(__name__)
+
+@app.route('/')
+def index():
+    return render_template('email.html')
+
+@app.route('/email', methods = ['GET', 'POST'])
+def email():
+    email = str(request.form['email'])
+    user_info = functions.get_user_details(users,email,problems,submissions)
+    if user_info==0:
+            status= "Not Found in DB"
+            return render_template("email.html",status=status)
+    personal = functions.get_submissions(email,users,problems,submissions)
+    num_sub=len(personal)
+    personal_merged,json_verdict,json_topics = functions.get_graphs(personal,users,problems,submissions)
+    bad,good,best = functions.get_topic_ratings(personal,users,problems,submissions)
+    nHighest,json_n_highest_submissions = functions.n_high_submission(personal,problems,users,submissions)
+
+    unexplored_d,practice_d ,best_d= functions.get_recommendation(problems,bad,good,personal_merged,best) 
+
+    return render_template("index.html",
+        json_verdict=json_verdict,
+        json_topics=json_topics,
+        good=good,bad=bad,best=best,
+        user_info=user_info,
+        json_n_highest_submissions=json_n_highest_submissions,
+        num_sub=num_sub,
+        nHighest=nHighest,
+        unexplored_d=unexplored_d,
+        practice_d=practice_d,
+        best_d=best_d
+        )
+
+
+if __name__ == '__main__':
+    app.run(debug=True)
diff --git a/student_analysis/nltk_data/corpora/wordnet.zip b/student_analysis/nltk_data/corpora/wordnet.zip