screentime scraper v1 done

UncleGravity · Apr 8, 2023 · b2417ef · b2417ef
1 parent d912fd8
commit b2417ef
Show file tree

Hide file tree

Showing 5 changed files with 314 additions and 1 deletion.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -72,6 +72,7 @@ services:
     ports:
     - ${SCRAPER_HEALTHKIT_PORT}:${SCRAPER_HEALTHKIT_PORT}
     - ${SCRAPER_MAPS_PORT}:${SCRAPER_MAPS_PORT}
+    - ${SCRAPER_SCREENTIME_PORT}:${SCRAPER_SCREENTIME_PORT}
 
     # Overrides default command so things don't shut down after the process ends.
     # command: sleep infinity

diff --git a/superscraper/package-lock.json b/superscraper/package-lock.json
diff --git a/superscraper/scrapers/_utils/db.js b/superscraper/scrapers/_utils/db.js
@@ -70,7 +70,7 @@ async function createTable(schema, tableName, columns, uniqueColumns) {
     try {
         // Create Table
         await pool.query(createTableQuery);
-        console.log(createTableQuery);
+        // console.log(createTableQuery);
 
         // Convert table to hypertable
         await pool.query(`SELECT create_hypertable('${schema}.${tableName}','time');`);

diff --git a/superscraper/scrapers/screentime/package.json b/superscraper/scrapers/screentime/package.json
@@ -0,0 +1,18 @@
+{
+    "name": "superscraper-screentime",
+    "version": "1.0.0",
+    "description": "",
+    "scripts": {
+      "start": "node screentime-scraper.js",
+      "clean": "rm -rf node_modules",
+      "test": "echo \"Error: no test specified\" && exit 1"
+    },
+    "author": "",
+    "license": "ISC",
+    "dependencies": {
+      "dotenv": "^16.0.3",
+      "express": "^4.18.2",
+      "pg": "^8.9.0"
+    }
+  }
+
diff --git a/superscraper/scrapers/screentime/screentime-scraper.js b/superscraper/scrapers/screentime/screentime-scraper.js
@@ -0,0 +1,280 @@
+const express = require('express');
+const app = express();
+const db = require('../_utils/db.js');
+
+const schema = 'app_usage';
+// const tableName = 'raw_screen_time_data';
+// const uniqueColumns = ['time', 'appname', 'deviceid'];
+
+app.use(express.json({ limit: '10mb' }));
+const serverPort = process.env.SCRAPER_SCREENTIME_PORT;
+
+app.post('/test', (req, res) => {
+    console.log(JSON.stringify(req.body, null, 2));
+    res.json({ result: 'ok' });
+});
+
+app.post('/screentime', (req, res) => {
+      handleScreenTimeEndpoint(req, res);
+});
+
+app.get('/', async (req, res) => {
+    handleRootEndpoint(req, res);
+});
+
+// Start server
+app.listen(serverPort, () => {
+    console.log(`Server is running on port ${serverPort}`);
+});
+
+// Endpoints handling functions
+async function handleRootEndpoint(req, res) {
+    res.status(200).send('Hello from the Screen Time app usage server!');
+}
+
+async function handleScreenTimeEndpoint(req, res) {
+    console.log(JSON.stringify(req.body, null, 2));
+
+    /* Expected request body:
+    combined_data = {
+        'screenTimeData': screentime_data,
+        'windowEventData': window_data,
+        'webEventData': web_data,
+        'vscodeEventData': vscode_data,
+        'utc_offset': get_utc_offset() / 3600
+    }
+    */
+
+    await saveScreenTimeData(req.body.screenTimeData);
+    await saveScreenTimeWebData(req.body.screenTimeWebData);
+    await saveWindowEventData(req.body.windowEventData);
+    await saveWebEventData(req.body.webEventData);
+    await saveVscodeEventData(req.body.vscodeEventData);
+
+    res.json({ result: 'ok' });
+}
+
+async function saveScreenTimeData(screenTimeData) {
+    const tableName = 'apple_screen_time_data';
+    const uniqueColumns = ['time', 'app_name', 'device_id'];
+
+    const data = [];
+    screenTimeData.forEach((entry) => {
+        data.push([
+            {
+                name: 'time',
+                value: entry.time_start,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'time_end',
+                value: entry.time_end,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'app_name',
+                value: entry.app_name,
+                type: 'TEXT',
+            },
+            {
+                name: 'device_id',
+                value: entry.device_id,
+                type: 'TEXT',
+            },
+            {
+                name: 'device_name',
+                value: entry.device_name,
+                type: 'TEXT',
+            },
+            {
+                name: 'gmt_offset',
+                value: entry.gmt_offset,
+                type: 'INTEGER',
+            },
+        ]);
+    });
+
+    try {
+        // Save data to the database
+        await db.saveData(schema, tableName, data, uniqueColumns);
+    } catch (error) {
+        console.error('Error inserting screen time data', error);
+    }
+}
+
+async function saveScreenTimeWebData(screenTimeWebData) {
+    const tableName = 'apple_screen_time_web_data';
+    const uniqueColumns = ['time', 'device_id'];
+
+    const data = [];
+    screenTimeWebData.forEach((entry) => {
+        data.push([
+            {
+                name: 'time',
+                value: entry.time_start,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'time_end',
+                value: entry.time_end,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'app_name',
+                value: entry.app_name,
+                type: 'TEXT',
+            },
+            {
+                name: 'device_id',
+                value: entry.device_id,
+                type: 'TEXT',
+            },
+            {
+                name: 'device_name',
+                value: entry.device_name,
+                type: 'TEXT',
+            },
+            {
+                name: 'gmt_offset',
+                value: entry.gmt_offset,
+                type: 'INTEGER',
+            },
+        ]);
+    });
+
+    try {
+        // Save data to the database
+        await db.saveData(schema, tableName, data, uniqueColumns);
+    } catch (error) {
+        console.error('Error inserting screen time data', error);
+    }
+}
+
+async function saveWindowEventData(windowEventData) {
+    const tableName = 'aw_window_event_data';
+    const uniqueColumns = ['time', 'app_name'];
+
+    const data = [];
+    windowEventData.forEach((entry) => {
+        data.push([
+            {
+                name: 'time',
+                value: entry.time_start,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'time_end',
+                value: entry.time_end,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'duration',
+                value: entry.duration,
+                type: 'DOUBLE PRECISION',
+            },
+            {
+                name: 'app_name',
+                value: entry.app_name,
+                type: 'TEXT',
+            },
+        ]);
+    });
+
+    try {
+        // Save data to the database
+        await db.saveData(schema, tableName, data, uniqueColumns);
+    } catch (error) {
+        console.error('Error inserting window event data', error);
+    }
+}
+
+async function saveWebEventData(webEventData) {
+    const tableName = 'aw_web_event_data';
+    const uniqueColumns = ['time', 'url'];
+
+    const data = [];
+    webEventData.forEach((entry) => {
+        data.push([
+            {
+                name: 'time',
+                value: entry.time_start,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'time_end',
+                value: entry.time_end,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'duration',
+                value: entry.duration,
+                type: 'DOUBLE PRECISION',
+            },
+            {
+                name: 'url',
+                value: entry.url,
+                type: 'TEXT',
+            },
+            {
+                name: 'audible',
+                value: entry.audible,
+                type: 'BOOLEAN',
+            },
+        ]);
+    });
+
+    try {
+        // Save data to the database
+        await db.saveData(schema, tableName, data, uniqueColumns);
+    } catch (error) {
+        console.error('Error inserting web event data', error);
+    }
+}
+
+async function saveVscodeEventData(vscodeEventData) {
+    const tableName = 'aw_vscode_event_data';
+    const uniqueColumns = ['time', 'file'];
+
+    const data = [];
+    vscodeEventData.forEach((entry) => {
+        data.push([
+            {
+                name: 'time',
+                value: entry.time_start,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'time_end',
+                value: entry.time_end,
+                type: 'TIMESTAMP WITH TIME ZONE',
+            },
+            {
+                name: 'duration',
+                value: entry.duration,
+                type: 'DOUBLE PRECISION',
+            },
+            {
+                name: 'language',
+                value: entry.language,
+                type: 'TEXT',
+            },
+            {
+                name: 'project',
+                value: entry.project,
+                type: 'TEXT',
+            },
+            {
+                name: 'file',
+                value: entry.file,
+                type: 'TEXT',
+            },
+        ]);
+    });
+
+    try {
+        // Save data to the database
+        await db.saveData(schema, tableName, data, uniqueColumns);
+    } catch (error) {
+        console.error('Error inserting vscode event data', error);
+    }
+}