Merge pull request #1 from afrizaloky/convert

Speech recognition wit.ai
ghiscure · Oct 25, 2020 · 0b75a8c · 0b75a8c
2 parents d9a51c6 + 0cacd10
commit 0b75a8c
Show file tree

Hide file tree

Showing 6 changed files with 295 additions and 53 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,7 +2,9 @@
 # Created by https://www.toptal.com/developers/gitignore/api/node
 # Edit at https://www.toptal.com/developers/gitignore?templates=node
 
-
+*.mp3
+*.mp4
+convert.js
 package-lock.json
 ### Node ###
 # Logs

diff --git a/Procfile b/Procfile
@@ -0,0 +1 @@
+web: node index.js
diff --git a/handler/messages/index.js b/handler/messages/index.js
@@ -0,0 +1,125 @@
+require('dotenv').config()
+const request = require('request')
+const {getMessage, getMessagefromAudio} = require('../../lib/witai')
+const PAGE_ACCESS_TOKEN = process.env.PAGE_ACCESS_TOKEN;
+const mqtt = require("mqtt");
+const listen = mqtt.connect("mqtt://test.mosquitto.org");
+const ffmpeg = require ('fluent-ffmpeg')
+const fs = require('fs');
+const fetch = require('node-fetch');
+
+
+module.exports = {
+
+    handleMessage: async function(sender_psid, received_message){
+
+    let response;
+    // Checks if the message contains text
+    if (received_message.text) {
+      // Create the payload for a basic text message, which
+      // will be added to the body of our request to the Send API
+
+      var result = await getMessage(received_message.text)
+      var caps= `turning ${result[0][0]} the ${result[1][0]}`
+      response = {
+        "text": caps
+      }
+      callSendAPI(sender_psid, response);
+
+      var topic = `esp8266/ghiscure/${result[1][0]}` 
+      if(result[0][0]=='on'){
+      listen.publish(topic, "1");
+      console.log(`${result[0][0]}, ${topic}`);
+      }else{
+      listen.publish(topic, "0");
+      console.log(`${result[0][0]}, ${topic}`);
+      }
+
+    } 
+    else if (received_message.attachments[0].type=="audio") {
+      console.log('audio')
+
+      // Get the URL of the message attachment
+      let attachment_url = received_message.attachments[0].payload.url;
+
+
+     var result = await fetch(attachment_url)
+     proc = new ffmpeg({source:result.body})
+     proc.setFfmpegPath('ffmpeg')
+     result = proc.saveToFile('output.mp3',  function(stdout, stderr){
+          return "success"
+     })
+    var mimetype_ = "audio/mpeg3"
+     var readStream = fs.createReadStream("output.mp3")
+     result = await getMessagefromAudio(readStream, mimetype_)
+     console.log(result)
+     var caps= `turning ${result[0][0]} the ${result[1][0]}`
+     response = {
+      "text": caps
+    }
+     callSendAPI(sender_psid, response);
+     if(result[0][0]=='on'){
+       listen.publish(topic, "1");
+       console.log(`${result[0][0]}, ${topic}`);
+       }else{
+       listen.publish(topic, "0");
+       console.log(`${result[0][0]}, ${topic}`);
+       }     
+    }
+  },
+
+  handlePostback: async function(sender_psid, received_postback) {
+    console.log('ok')
+     let response;
+    // Get the payload for the postback
+    let payload = received_postback.payload;
+
+    // Set the response based on the postback payload
+    if (payload === 'yes') {
+      response = { "text": "Thanks!" }
+    } else if (payload === 'no') {
+      response = { "text": "Oops, try sending another image." }
+    }
+    // Send the message to acknowledge the postback
+    callSendAPI(sender_psid, response);
+  }
+
+};
+
+async function callSendAPI(sender_psid, response) {
+  // Construct the message body
+  let request_body = {
+    "recipient": {
+      "id": sender_psid
+    },
+    "message": response
+  }
+
+  // Send the HTTP request to the Messenger Platform
+  request({
+    "uri": "https://graph.facebook.com/v8.0/me/messages",
+    "qs": { "access_token": PAGE_ACCESS_TOKEN },
+    "method": "POST",
+    "json": request_body
+  }, (err, res, body) => {
+    if (!err) {
+      console.log('message sent!')
+    } else {
+      console.error("Unable to send message:" + err);
+    }
+  }); 
+}
+
+function convert(input, output, callback) {
+  ffmpeg(input)
+      .output(output)
+      .on('end', function() {                    
+          console.log('conversion ended');
+          callback(null);
+      }).on('error', function(err){
+          console.log('error: ', e.code, e.msg);
+          callback(err);
+      }).run();
+}
+
+// curl -XPOST 'https://api.wit.ai/speech?v=20200513' \-i -L \-H "Authorization: Token" \-H "Content-Type: audio/mpeg3" \--data-binary "@./input/turn.mp3"
diff --git a/index.js b/index.js
@@ -1,65 +1,106 @@
-'use strict';
 
+/**
+ * Copyright 2017-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * Messenger Platform Quick Start Tutorial
+ *
+ * This is the completed code for the Messenger Platform quick start tutorial
+ *
+ * https://developers.facebook.com/docs/messenger-platform/getting-started/quick-start/
+ *
+ * To run this code, you must do the following:
+ *
+ * 1. Deploy this code to a server running Node.js
+ * 2. Run `npm install`
+ * 3. Update the VERIFY_TOKEN
+ * 4. Add your PAGE_ACCESS_TOKEN to your environment vars
+ *
+ */
+
+require('dotenv').config()
+
+'use strict';
 // Imports dependencies and set up http server
-const
+const 
   express = require('express'),
-  bodyParser = require('body-parser'),
-  app = express().use(bodyParser.json()); // creates express http server
+  body_parser = require('body-parser'),
+  app = express().use(body_parser.json()); // creates express http server
+
+const {handleMessage, handlePostback} = require('./handler/messages')
 
 
-// Creates the endpoint for our webhook 
+// Accepts POST requests at /webhook endpoint
 app.post('/webhook', (req, res) => {  
-
-    let body = req.body;
-
-    // Checks this is an event from a page subscription
-    if (body.object === 'page') {
-
-      // Iterates over each entry - there may be multiple if batched
-      body.entry.forEach(function(entry) {
-
-        // Gets the message. entry.messaging is an array, but 
-        // will only ever contain one message, so we get index 0
-        let webhook_event = entry.messaging[0];
-        console.log(webhook_event);
-      });
-
-      // Returns a '200 OK' response to all requests
-      res.status(200).send('EVENT_RECEIVED');
-    } else {
-      // Returns a '404 Not Found' if event is not from a page subscription
-      res.sendStatus(404);
-    }
-
-  });
 
-  // Adds support for GET requests to our webhook
-app.get('/webhook', (req, res) => {
+  // Parse the request body from the POST
+  let body = req.body;
 
-    // Your verify token. Should be a random string.
-    let VERIFY_TOKEN = "verysecret"
-
-    // Parse the query params
-    let mode = req.query['hub.mode'];
-    let token = req.query['hub.verify_token'];
-    let challenge = req.query['hub.challenge'];
+  // Check the webhook event is from a Page subscription
+  if (body.object === 'page') {
+
+    body.entry.forEach(function(entry) {
+
+      // Gets the body of the webhook event
+      let webhook_event = entry.messaging[0];
+      console.log(webhook_event);
+
+
+      // Get the sender PSID
+      let sender_psid = webhook_event.sender.id;
+      console.log('Sender ID: ' + sender_psid);
+
+      // Check if the event is a message or postback and
+      // pass the event to the appropriate handler function
+      if (webhook_event.message) {
+        handleMessage(sender_psid, webhook_event.message);        
+      } else if (webhook_event.postback) {
+
+        handlePostback(sender_psid, webhook_event.postback);
+      }
 
-    // Checks if a token and mode is in the query string of the request
-    if (mode && token) {
+    });
+    // Return a '200 OK' response to all events
+    res.status(200).send('EVENT_RECEIVED');
+
+  } else {
+    // Return a '404 Not Found' if event is not from a page subscription
+    res.sendStatus(404);
+  }
+
+});
+
+// Accepts GET requests at the /webhook endpoint
+app.get('/webhook', (req, res) => {
+
+  /** UPDATE YOUR VERIFY TOKEN **/
+  const VERIFY_TOKEN = process.env.VERIFY_TOKEN;
+
+
+  // Parse params from the webhook verification request
+  let mode = req.query['hub.mode'];
+  let token = req.query['hub.verify_token'];
+  let challenge = req.query['hub.challenge'];
 
-      // Checks the mode and token sent is correct
-      if (mode === 'subscribe' && token === VERIFY_TOKEN) {
-
-        // Responds with the challenge token from the request
-        console.log('WEBHOOK_VERIFIED');
-        res.status(200).send(challenge);
+  // Check if a token and mode were sent
+  if (mode && token) {
+
+    // Check the mode and token sent are correct
+    if (mode === 'subscribe' && token === VERIFY_TOKEN) {
 
-      } else {
-        // Responds with '403 Forbidden' if verify tokens do not match
-        res.sendStatus(403);      
-      }
+      // Respond with 200 OK and challenge token from the request
+      console.log('WEBHOOK_VERIFIED');
+      res.status(200).send(challenge);
+
+    } else {
+      // Responds with '403 Forbidden' if verify tokens do not match
+      res.sendStatus(403);      
     }
-  });
+  }
+});
 
 // Sets server port and logs message on success
-app.listen(process.env.PORT || 1337, () => console.log('webhook is listening'));
+var PORT = process.env.PORT || 1337
+app.listen(PORT, () => console.log(`webhook is listening on port ${PORT}`));
diff --git a/lib/witai.js b/lib/witai.js
@@ -0,0 +1,68 @@
+var fetch = require('node-fetch');
+require('dotenv').config()
+
+module.exports = {
+
+    getMessage: async function(query){
+      var url =`https://api.wit.ai/message?v=20201020&q=${encodeURI(query)}` 
+
+      var response = await fetch(url, {
+          headers: {
+              'Authorization': process.env.witai_token
+          }
+      })
+      var json_data = await response.json()
+      try {
+
+        // Get Command Value
+        var cmd_value =json_data.traits.wit$on_off[0].value
+        var cmd_confidence = json_data.traits.wit$on_off[0].confidence
+
+        // Get Object Value
+        var object_value = json_data.entities['object:object'][0].value
+        var object_confidence = json_data.entities['object:object'][0].confidence
+
+        console.log(cmd_value, cmd_confidence)
+        console.log(object_value, object_confidence)  
+        return [[cmd_value,cmd_confidence],[object_value,object_confidence]]   
+
+      } catch (error) {
+        console.log(error)
+
+      }
+
+    },
+
+    getMessagefromAudio: async function(bin_data, mimetype_){
+      var options = {
+          method: 'POST',
+          headers: {
+              'Authorization': process.env.witai_token,
+              'Content-Type': mimetype_
+          },
+          encoding: null,
+          body: bin_data
+        }
+      var url =`https://api.wit.ai/speech?v=20200513` 
+      try {
+        var response = await fetch(url, options)
+        var json_data = await response.json()
+
+        // Get Command Value
+        var cmd_value =json_data.traits.wit$on_off[0].value
+        var cmd_confidence = json_data.traits.wit$on_off[0].confidence
+
+        // Get Object Value
+        var object_value = json_data.entities['object:object'][0].value
+        var object_confidence = json_data.entities['object:object'][0].confidence
+
+        console.log(cmd_value, cmd_confidence)
+        console.log(object_value, object_confidence)  
+        return [[cmd_value,cmd_confidence],[object_value,object_confidence]]     
+      } catch (error) {
+        console.log(error)
+
+      }
+    }
+
+  };
diff --git a/package.json b/package.json
@@ -4,13 +4,18 @@
   "description": "",
   "main": "index.js",
   "scripts": {
+    "start": "node index.js",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "author": "",
   "license": "ISC",
   "dependencies": {
     "body-parser": "^1.19.0",
+    "dotenv": "^8.2.0",
     "express": "^4.17.1",
-    "dotenv": "^8.2.0"
+    "fluent-ffmpeg": "^2.1.2",
+    "mqtt": "^4.1.0",
+    "node-fetch": "^2.6.1",
+    "request": "^2.88.2"
   }
 }