fixed awair scraper db bug

UncleGravity · Jul 1, 2023 · feb968e · feb968e
1 parent 03ee2fb
commit feb968e
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 23 deletions.
diff --git a/.env.local b/.env.local
@@ -1,3 +1,9 @@
+#################################################################################
+#                                                                               #
+#  This file contains all the environment variables needed to run the project   #
+#  Change the values to match your environment and rename the file to .env      #
+#                                                                               #
+#################################################################################
 # TimescaleDB
 POSTGRES_PORT=5432
 POSTGRES_USER=

diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,7 @@
 node_modules/
 .env
-.DS_Store
+.DS_Store
+.env*
+.flaskenv*
+!.env.project
+!.env.vault
diff --git a/superscraper/scrapers/_utils/db.js b/superscraper/scrapers/_utils/db.js
@@ -55,7 +55,7 @@ async function createTable(schema, tableName, columns, uniqueColumns) {
     // Check if the table exists
     const tableExists = await doesTableExist(schema, tableName);
     if (tableExists) {
-        console.log(`Table ${schema}.${tableName} already exists, skipping table creation.`);
+        // console.log(`Table ${schema}.${tableName} already exists, skipping table creation.`);
         return;
     }
 

diff --git a/superscraper/scrapers/awair/awair-api.js b/superscraper/scrapers/awair/awair-api.js
@@ -160,4 +160,4 @@ async function test() {
 
 // test()
 
-module.exports = { getDevices, getAirData, getDeviceIdList };
+module.exports = { getAirData, getDeviceIdList };
diff --git a/superscraper/scrapers/awair/awair-scraper.js b/superscraper/scrapers/awair/awair-scraper.js
@@ -1,41 +1,64 @@
-const { getAirData, getDevices, getDeviceIdList } = require("./awair-api.js");
+const { getAirData, getDeviceIdList } = require("./awair-api.js");
 const db = require("../_utils/db.js");
 
 const AIRDATA_SCHEMA_NAME = "awair";
 const AIRDATA_TABLE_NAME = "awair_sensor_data";
-const UNIQUE_COLUMNS = ['time','location'];
+const UNIQUE_COLUMNS = ['time','deviceid'];
 
 async function main() {
-
-  // save data for all devices
-  for (const deviceId of await getDeviceIdList()) {
-    const airData = await getAirData(deviceId);
-    const formattedData = airData.map(entry => {
-      const [time, temp, humid, co2, voc, pm25] = entry.split(",");
-      return [
-        { name: "time",     value: time,     type: 'TIMESTAMPTZ' },
-        { name: "deviceid", value: deviceId, type: 'INTEGER' },
-        { name: "temp",     value: temp,     type: 'FLOAT' },
-        { name: "humid",    value: humid,    type: 'FLOAT' },
-        { name: "co2",      value: co2,      type: 'INTEGER' },
-        { name: "voc",      value: voc,      type: 'INTEGER' },
-        { name: "pm25",     value: pm25,     type: 'INTEGER' }
-      ];
+  try {
+    // get list of device IDs
+    const deviceIds = await getDeviceIdList().catch(error => {
+      console.error(`Error retrieving device IDs: ${error}`);
+      return [];
     });
-    await db.saveData(AIRDATA_SCHEMA_NAME, AIRDATA_TABLE_NAME, formattedData, UNIQUE_COLUMNS);
+
+    // process each device
+    for (const deviceId of deviceIds) {
+      // get air data for device
+      const airData = await getAirData(deviceId).catch(error => {
+        console.error(`Error retrieving air data for device ${deviceId}: ${error}`);
+        return [];
+      });
+
+      const formattedData = airData.map(entry => {
+        const [time, temp, humid, co2, voc, pm25] = entry.split(",");
+        return [
+          { name: "time",     value: time,     type: 'TIMESTAMPTZ' },
+          { name: "deviceid", value: deviceId, type: 'INTEGER' },
+          { name: "temp",     value: temp,     type: 'FLOAT' },
+          { name: "humid",    value: humid,    type: 'FLOAT' },
+          { name: "co2",      value: co2,      type: 'INTEGER' },
+          { name: "voc",      value: voc,      type: 'INTEGER' },
+          { name: "pm25",     value: pm25,     type: 'INTEGER' }
+        ];
+      });
+
+      // console.log(formattedData);
+
+      await db.saveData(AIRDATA_SCHEMA_NAME, AIRDATA_TABLE_NAME, formattedData, UNIQUE_COLUMNS)
+      .catch(error => {
+        console.error(`Error saving data to database: ${error}`);
+      });
+    }
+  } catch (error) {
+    console.error(`An unexpected error occurred: ${error}`);
   }
 
   // read 100 rows from table (for testing)
   // for (const deviceId of await getDeviceIdList()) {
     // const rows = await db.readData(AIRDATA_SCHEMA_NAME, AIRDATA_TABLE_NAME, 100);
     // console.log(rows);
   // }
-}
 
+}
 
 // run main function every 5 minutes
 setInterval(main, 240000);
-main();
+
+// initial run
+main().catch(error => console.error(`An error occurred while running the awair main function: ${error}`));
+
 
 /*
Original file line number	Diff line number	Diff line change
Expand Up		@@ -160,4 +160,4 @@ async function test() {

		// test()

		module.exports = { getDevices, getAirData, getDeviceIdList };
		module.exports = { getAirData, getDeviceIdList };