added ios map scraper endpoint

UncleGravity · Jun 22, 2023 · d6e5305 · d6e5305
1 parent d61a720
commit d6e5305
Showing 1 changed file with 84 additions and 2 deletions.
diff --git a/superscraper/scrapers/maps/maps-scraper.js b/superscraper/scrapers/maps/maps-scraper.js
@@ -1,3 +1,5 @@
+// This script reads location data from the Overland app (https://github.com/aaronpk/Overland-iOS) and from my scraper app, and saves it to the database.
+
 const express = require('express');
 const app = express();
 const db = require('../_utils/db.js');
@@ -9,20 +11,33 @@ const uniqueColumns = ['time', 'coordinates'];
 app.use(express.json({ limit: '10mb' }));
 const serverPort = process.env.SCRAPER_MAPS_PORT;
 
+////////////////////////////////////////////////////
+// Routes
+
+// TODO create new file called server.js in teh _utils folder that runs the express server. 
+// There should only be one function exposed, add a new route to the server, and then call the function from the scraper script. 
+// This will make it easier to add new scrapers in the future. This might be dumb. make sure it's not dumb.
+
 app.post('/maps', (req, res) => {
   handleMapsEndpoint(req, res);
 });
 
+app.post('/maps', (req, res) => {
+  handleOverlandEndpoint(req, res);
+});
+
 app.get('/', async (req, res) => {
   handleRootEndpoint(req, res);
 });
 
 // Start server
 app.listen(serverPort, () => {
-  console.log(`Server is running on port ${serverPort}`);
+  console.log(`Maps Scraper server listening on port ${serverPort}`);
 });
 
-// Endpoints handling functions
+////////////////////////////////////////////////////
+// Endpoint handlers
+
 async function handleRootEndpoint(req, res) {
   res.status(200).send('Hello from the Maps scraper!');
 }
@@ -31,6 +46,73 @@ async function handleMapsEndpoint(req, res) {
   console.log(JSON.stringify(req.body, null, 2));
   res.json({ result: 'ok' });
 
+  const locations = req.body.location;
+
+  const data = [];
+  locations.forEach((location) => {
+    const coordinates = `POINT(${location.coords.longitude} ${location.coords.latitude})`;
+    data.push([
+      {
+        name: 'time',
+        value: location.timestamp,
+        type: 'TIMESTAMP WITH TIME ZONE',
+      },
+      {
+        name: 'coordinates',
+        value: coordinates,
+        type: 'GEOGRAPHY(POINT, 4326)',
+      },
+      {
+        name: 'altitude',
+        value: location.coords.altitude,
+        type: 'DOUBLE PRECISION',
+      },
+      {
+        name: 'horizontal_accuracy',
+        value: location.coords.accuracy,
+        type: 'INTEGER',
+      },
+      {
+        name: 'altitude_accuracy',
+        value: location.coords.altitude_accuracy,
+        type: 'DOUBLE PRECISION',
+      },
+      {
+        name: 'speed',
+        value: location.coords.speed,
+        type: 'DOUBLE PRECISION',
+      },
+      {
+        name: 'is_moving',
+        value: location.is_moving,
+        type: 'BOOLEAN',
+      },
+      {
+        name: 'activity_type',
+        value: location.activity.type,
+        type: 'TEXT',
+      },
+      {
+        name: 'activity_confidence',
+        value: location.activity.confidence,
+        type: 'INTEGER',
+      },
+    ]);
+  });
+
+  try {
+    await db.saveData(schema, tableName, data, uniqueColumns);
+  } catch (error) {
+    console.error('Error inserting location data', error);
+  }
+}
+
+async function handleOverlandEndpoint(req, res) {
+  console.log(JSON.stringify(req.body, null, 2));
+  res.json({ result: 'ok' });
+
+  console.log(req.body.location.coords)
+
   // parse location json data
   const locations = req.body.locations;