analysis/course_analytics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests import get\n",
    "from json import load, loads\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import all courses name and ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "courses_dict = load(open('data/courses/all_courses_dict.json'))\n",
    "list(courses_dict.keys())"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `GET` Analytics Page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = 'Data Science masters'\n",
    "\n",
    "url = f'https://learn.pwskills.com/course-analytics/{name.replace(\" \", \"-\")}/{courses_dict[name]}'\n",
    "r = get(url)\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(r.text, 'html.parser')\n",
    "script = soup.find('script', {'id': '__NEXT_DATA__'})\n",
    "data = loads(script.text)    # type: ignore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = data['props']['pageProps']['analytics']['users']\n",
    "submissions = data['props']['pageProps']['analytics']['submissions']\n",
    "quiz_analytics = data['props']['pageProps']['quizAnalytics']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.DataFrame(submissions)\n",
    "df2 = pd.DataFrame(users)\n",
    "df3 = pd.DataFrame(quiz_analytics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1.shape, df2.shape, df3.shape"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concatenate\n",
    "\n",
    "Concatenate `df1` and `df2` along with `axis=1` because they are in same order."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df1, df2], axis=1).drop(columns=['_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['img'].isnull()==1]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import `sections` data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import LiveCourse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "live_course = {\n",
    "    \"paramLength\": 0,\n",
    "    \"sections\": data['props']['pageProps']['sections'],\n",
    "    \"courseName\": name,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lc = LiveCourse(**live_course).merged_df(courses_dict[name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_of_assignment = lc.query('type==\"assignment\"').shape[0]\n",
    "no_of_quiz = lc.query('type==\"quiz\"').shape[0]\n",
    "\n",
    "resource_type_count = lc['type'].value_counts().to_dict()\n",
    "\n",
    "total_assignment_points = int(lc['totalPointsInAssignment'].sum())\n",
    "total_quiz_points = int(lc['totalQuestionsInQuiz'].sum())\n",
    "total_video_duration = int(lc['duration'].sum()/3600)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resource_type_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_assignment_points, total_quiz_points, total_video_duration"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Course Analytics Analysis"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[['firstName', 'lastName', 'assignmentsMarkedCount', 'totalAssignmentsScore']].head(3)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Under 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[['firstName', 'lastName', 'assignmentsMarkedCount', 'totalAssignmentsScore']].loc[3:]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top 10 Students in Quiz submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3[['totalPoints', 'firstName', 'lastName']]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}