diff --git a/superscraper/package-lock.json b/superscraper/package-lock.json index 4ca3cd3..4058702 100644 --- a/superscraper/package-lock.json +++ b/superscraper/package-lock.json @@ -12,6 +12,37 @@ "scrapers/*" ] }, + "node_modules/@turf/boolean-point-in-polygon": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@turf/boolean-point-in-polygon/-/boolean-point-in-polygon-6.5.0.tgz", + "integrity": "sha512-DtSuVFB26SI+hj0SjrvXowGTUCHlgevPAIsukssW6BG5MlNSBQAo70wpICBNJL6RjukXg8d2eXaAWuD/CqL00A==", + "dependencies": { + "@turf/helpers": "^6.5.0", + "@turf/invariant": "^6.5.0" + }, + "funding": { + "url": "https://opencollective.com/turf" + } + }, + "node_modules/@turf/helpers": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@turf/helpers/-/helpers-6.5.0.tgz", + "integrity": "sha512-VbI1dV5bLFzohYYdgqwikdMVpe7pJ9X3E+dlr425wa2/sMJqYDhTO++ec38/pcPvPE6oD9WEEeU3Xu3gza+VPw==", + "funding": { + "url": "https://opencollective.com/turf" + } + }, + "node_modules/@turf/invariant": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@turf/invariant/-/invariant-6.5.0.tgz", + "integrity": "sha512-Wv8PRNCtPD31UVbdJE/KVAWKe7l6US+lJItRR/HOEW3eh+U/JwRCSUl/KZ7bmjM/C+zLNoreM2TU6OoLACs4eg==", + "dependencies": { + "@turf/helpers": "^6.5.0" + }, + "funding": { + "url": "https://opencollective.com/turf" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -29,6 +60,11 @@ "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" }, + "node_modules/array-source": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/array-source/-/array-source-0.0.4.tgz", + "integrity": "sha512-frNdc+zBn80vipY+GdcJkLEbMWj3xmzArYApmUGxoiV8uAu/ygcs9icPdsGdA26h0MkHUMW6EN2piIvVx+M5Mw==" + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -67,6 +103,11 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, "node_modules/buffer-writer": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", @@ -106,6 +147,25 @@ "node": ">= 0.8" } }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -293,6 +353,14 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/file-source": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/file-source/-/file-source-0.6.1.tgz", + "integrity": "sha512-1R1KneL7eTXmXfKxC10V/9NeGOdbsAXJ+lQ//fvvcHUgtaZcZDWNJNblxAoVOyV1cj45pOtUrR3vZTBwqcW8XA==", + "dependencies": { + "stream-source": "0.3" + } + }, "node_modules/finalhandler": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", @@ -379,6 +447,35 @@ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, + "node_modules/geo-tz": { + "version": "7.0.7", + "resolved": "https://registry.npmjs.org/geo-tz/-/geo-tz-7.0.7.tgz", + "integrity": "sha512-Aq0sRSO1y4w62D5muRqzDmN4SWfFYnt703BLiqiHAvunlwsJs4qd3Fkl1pKSUa0bwuBmPFxIA8M1E+ilg2PSjw==", + "dependencies": { + "@turf/boolean-point-in-polygon": "^6.5.0", + "@turf/helpers": "^6.5.0", + "geobuf": "^3.0.2", + "pbf": "^3.2.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/geobuf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/geobuf/-/geobuf-3.0.2.tgz", + "integrity": "sha512-ASgKwEAQQRnyNFHNvpd5uAwstbVYmiTW0Caw3fBb509tNTqXyAAPMyFs5NNihsLZhLxU1j/kjFhkhLWA9djuVg==", + "dependencies": { + "concat-stream": "^2.0.0", + "pbf": "^3.2.1", + "shapefile": "~0.6.6" + }, + "bin": { + "geobuf2json": "bin/geobuf2json", + "json2geobuf": "bin/json2geobuf", + "shp2geobuf": "bin/shp2geobuf" + } + }, "node_modules/get-intrinsic": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", @@ -452,6 +549,25 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -609,11 +725,32 @@ "node": ">= 0.8" } }, + "node_modules/path-source": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/path-source/-/path-source-0.1.3.tgz", + "integrity": "sha512-dWRHm5mIw5kw0cs3QZLNmpUWty48f5+5v9nWD2dw3Y0Hf+s01Ag8iJEWV0Sm0kocE8kK27DrIowha03e1YR+Qw==", + "dependencies": { + "array-source": "0.0", + "file-source": "0.6" + } + }, "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==" }, + "node_modules/pbf": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/pbf/-/pbf-3.2.1.tgz", + "integrity": "sha512-ClrV7pNOn7rtmoQVF4TS1vyU0WhYRnP92fzbfF75jAIwpnzdJXf8iTd4CMEqO4yUenH6NDqLiwjqlh6QgZzgLQ==", + "dependencies": { + "ieee754": "^1.1.12", + "resolve-protobuf-schema": "^2.1.0" + }, + "bin": { + "pbf": "bin/pbf" + } + }, "node_modules/pg": { "version": "8.11.1", "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.1.tgz", @@ -732,6 +869,11 @@ "node": ">=0.10.0" } }, + "node_modules/protocol-buffers-schema": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.0.tgz", + "integrity": "sha512-TdDRD+/QNdrCGCE7v8340QyuXd4kIWIgapsE2+n/SaGiSSbomYl4TjHlvIoCWRpE7wFt02EpB35VVA2ImcBVqw==" + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -785,6 +927,27 @@ "node": ">= 0.8" } }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/resolve-protobuf-schema": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz", + "integrity": "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==", + "dependencies": { + "protocol-buffers-schema": "^3.3.1" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -856,6 +1019,23 @@ "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, + "node_modules/shapefile": { + "version": "0.6.6", + "resolved": "https://registry.npmjs.org/shapefile/-/shapefile-0.6.6.tgz", + "integrity": "sha512-rLGSWeK2ufzCVx05wYd+xrWnOOdSV7xNUW5/XFgx3Bc02hBkpMlrd2F1dDII7/jhWzv0MSyBFh5uJIy9hLdfuw==", + "dependencies": { + "array-source": "0.0", + "commander": "2", + "path-source": "0.1", + "slice-source": "0.4", + "stream-source": "0.3", + "text-encoding": "^0.6.4" + }, + "bin": { + "dbf2json": "bin/dbf2json", + "shp2json": "bin/shp2json" + } + }, "node_modules/side-channel": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", @@ -869,6 +1049,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/slice-source": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/slice-source/-/slice-source-0.4.1.tgz", + "integrity": "sha512-YiuPbxpCj4hD9Qs06hGAz/OZhQ0eDuALN0lRWJez0eD/RevzKqGdUx1IOMUnXgpr+sXZLq3g8ERwbAH0bCb8vg==" + }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -885,6 +1070,19 @@ "node": ">= 0.8" } }, + "node_modules/stream-source": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/stream-source/-/stream-source-0.3.5.tgz", + "integrity": "sha512-ZuEDP9sgjiAwUVoDModftG0JtYiLUV8K4ljYD1VyUMRWtbVf92474o4kuuul43iZ8t/hRuiDAx1dIJSvirrK/g==" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/superscraper-awair": { "resolved": "scrapers/awair", "link": true @@ -905,6 +1103,12 @@ "resolved": "scrapers/screentime", "link": true }, + "node_modules/text-encoding": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz", + "integrity": "sha512-hJnc6Qg3dWoOMkqP53F0dzRIgtmsAge09kxUIqGrEUS4qr5rWLckGYaQAVr+opBrIMRErGgy6f5aPnyPpyGRfg==", + "deprecated": "no longer maintained" + }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -925,6 +1129,11 @@ "node": ">= 0.6" } }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -933,6 +1142,11 @@ "node": ">= 0.8" } }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", @@ -997,6 +1211,7 @@ "axios": "^1.3.3", "dotenv": "^16.0.3", "express": "^4.18.2", + "geo-tz": "^7.0.7", "pg": "^8.9.0" } }, diff --git a/superscraper/scrapers/maps/maps-scraper.js b/superscraper/scrapers/maps/maps-scraper.js index e2cafd2..9f6f6c4 100644 --- a/superscraper/scrapers/maps/maps-scraper.js +++ b/superscraper/scrapers/maps/maps-scraper.js @@ -1,8 +1,7 @@ -// This script reads location data from the Overland app (https://github.com/aaronpk/Overland-iOS) and from my scraper app, and saves it to the database. - const express = require('express'); const app = express(); const db = require('../_utils/db.js'); +const geoTz = require('geo-tz'); const schema = 'maps'; const tableName = 'raw_visited_places'; @@ -43,7 +42,7 @@ async function handleRootEndpoint(req, res) { } async function handleMapsEndpoint(req, res) { - console.log(JSON.stringify(req.body, null, 2)); + // console.log(JSON.stringify(req.body, null, 2)); const locations = req.body.location; /* @@ -63,12 +62,19 @@ async function handleMapsEndpoint(req, res) { const data = []; locations.forEach((location) => { const coordinates = `POINT(${location.coords.longitude} ${location.coords.latitude})`; + let tz = geoTz.find(location.coords.latitude, location.coords.longitude)[0] || ''; // Get local timezone with geo-tz + data.push([ { name: 'time', value: location.timestamp, type: 'TIMESTAMP WITH TIME ZONE', }, + { + name: 'timezone', + value: tz, + type: 'TEXT', + }, { name: 'coordinates', value: coordinates, diff --git a/superscraper/scrapers/maps/package.json b/superscraper/scrapers/maps/package.json index 7f2e755..d203f67 100644 --- a/superscraper/scrapers/maps/package.json +++ b/superscraper/scrapers/maps/package.json @@ -3,17 +3,17 @@ "version": "1.0.0", "description": "", "scripts": { - "start": "node maps-scraper.js", - "clean": "rm -rf node_modules", - "test": "echo \"Error: no test specified\" && exit 1" + "start": "node maps-scraper.js", + "clean": "rm -rf node_modules", + "test": "echo \"Error: no test specified\" && exit 1" }, "author": "", "license": "ISC", "dependencies": { - "axios": "^1.3.3", - "dotenv": "^16.0.3", - "express": "^4.18.2", - "pg": "^8.9.0" + "axios": "^1.3.3", + "dotenv": "^16.0.3", + "express": "^4.18.2", + "geo-tz": "^7.0.7", + "pg": "^8.9.0" } - } - \ No newline at end of file +} diff --git a/timescaledb/timescaledb-entrypoint.sh b/timescaledb/timescaledb-entrypoint.sh index a9dea24..b9f5b28 100755 --- a/timescaledb/timescaledb-entrypoint.sh +++ b/timescaledb/timescaledb-entrypoint.sh @@ -22,6 +22,10 @@ psql -U $POSTGRES_USER -d $POSTGRES_DB -c 'CREATE SCHEMA telegraf;' psql -U $POSTGRES_USER -d $POSTGRES_DB -c 'CREATE SCHEMA awair;' psql -U $POSTGRES_USER -d $POSTGRES_DB -c 'CREATE SCHEMA healthkit;' psql -U $POSTGRES_USER -d $POSTGRES_DB -c 'CREATE SCHEMA maps;' +psql -U $POSTGRES_USER -d $POSTGRES_DB -c 'CREATE SCHEMA money;' # push this to server +# psql -U $POSTGRES_USER -d $POSTGRES_DB -c 'CREATE SCHEMA screentime;' +# psql -U $POSTGRES_USER -d $POSTGRES_DB -c 'CREATE SCHEMA oura;' +# psql -U $POSTGRES_USER -d $POSTGRES_DB -c 'CREATE SCHEMA strava;' # Set search path for new user -psql -U $POSTGRES_USER -d $POSTGRES_DB -c "ALTER USER $POSTGRES_USER SET search_path = telegraf,awair,healthkit,maps,public;" \ No newline at end of file +psql -U $POSTGRES_USER -d $POSTGRES_DB -c "ALTER USER $POSTGRES_USER SET search_path = telegraf,awair,healthkit,maps,money,public;" \ No newline at end of file