diff --git a/CMakeLists.txt b/CMakeLists.txt index 1613ddb..411470c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,7 +28,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON) # add_compile_options(-Wall -Wpedantic -Werror) # endif() -# include(FetchContent) +include(FetchContent) # FetchContent_Declare( # Catch2 @@ -37,16 +37,16 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON) # FetchContent_MakeAvailable(Catch2) -# FetchContent_Declare(json -# GIT_REPOSITORY https://github.com/ArthurSonzogni/nlohmann_json_cmake_fetchcontent.git -# GIT_TAG v3.7.3) +FetchContent_Declare(json + GIT_REPOSITORY https://github.com/ArthurSonzogni/nlohmann_json_cmake_fetchcontent.git + GIT_TAG v3.7.3) -# FetchContent_GetProperties(json) -# FetchContent_MakeAvailable(json) -# if(NOT json_POPULATED) -# FetchContent_Populate(json) -# add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR} EXCLUDE_FROM_ALL) -# endif() +FetchContent_GetProperties(json) +FetchContent_MakeAvailable(json) +if (NOT json_POPULATED) + FetchContent_Populate(json) + add_subdirectory(${json_SOURCE_DIR} ${json_BINARY_DIR} EXCLUDE_FROM_ALL) +endif() # find_package( Boost 1.54 REQUIRED COMPONENTS system filesystem ) # list( APPEND CINDER_LIBS_DEPENDS ${Boost_LIBRARIES} ) @@ -67,8 +67,6 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON) # endif() # endif() -# LIBRARIES ( opencv_core opencv_highgui opencv_imgproc opencv_videoio ) - find_library(TENSORFLOW_LIB tensorflow HINT /Users/rustomichhaporia/GitHub/libtensorflow-cpu-darwin-x86_64-2.3.1/lib) set(CMAKE_CXX_STANDARD 17) @@ -115,6 +113,7 @@ ci_make_app( CINDER_PATH ${CINDER_PATH} SOURCES apps/maskade_app.cpp ${CORE_SOURCE_FILES} ${VISUALIZER_SOURCE_FILES} INCLUDES include # blocks/Cinder-OpenCV-master/include + LIBRARIES nlohmann_json::nlohmann_json # BLOCKS Cinder-OpenCV-master ) diff --git a/apps/maskade_app.cpp b/apps/maskade_app.cpp index 0210e71..e98a6c5 100644 --- a/apps/maskade_app.cpp +++ b/apps/maskade_app.cpp @@ -1,6 +1,4 @@ #include "maskade_classifier.hpp" +#include "cinder/app/RendererGl.h" -using ci::app::RendererGl; -using maskade::MaskadeClassifier; - -CINDER_APP(MaskadeClassifier, RendererGl) +CINDER_APP(maskade::MaskadeClassifier, ci::app::RendererGl) diff --git a/config/config.json b/config/config.json new file mode 100644 index 0000000..be337b7 --- /dev/null +++ b/config/config.json @@ -0,0 +1,6 @@ +{ + "model_image_width": 224, + "model_image_height": 224, + "font": "IBM Plex Mono", + "font_color": "0xFFBFBF" +} \ No newline at end of file diff --git a/include/CinderOpenCV.hpp b/include/CinderOpenCV.hpp new file mode 100644 index 0000000..2d97f1f --- /dev/null +++ b/include/CinderOpenCV.hpp @@ -0,0 +1,197 @@ +// This file is borrowed from the Cinder-OpenCV Cinderblock +// https://github.com/cinder/Cinder-OpenCV3/blob/master/include/CinderOpenCV.h + +#pragma once + +#include "opencv2/opencv.hpp" + +#include "cinder/Cinder.h" +#include "cinder/ImageIo.h" + +namespace cinder { + +class ImageTargetCvMat : public ImageTarget { + public: + static std::shared_ptr createRef( cv::Mat *mat ) { return std::shared_ptr( new ImageTargetCvMat( mat ) ); } + + virtual bool hasAlpha() const { return mMat->channels() == 4; } + virtual void* getRowPointer( int32_t row ) { return reinterpret_cast( reinterpret_cast(mMat->data) + row * mMat->step ); } + + protected: + ImageTargetCvMat( cv::Mat *mat ); + + cv::Mat *mMat; +}; + +class ImageSourceCvMat : public ImageSource { + public: + ImageSourceCvMat( const cv::Mat &mat ) + : ImageSource() + { + mWidth = mat.cols; + mHeight = mat.rows; + if( (mat.channels() == 3) || (mat.channels() == 4) ) { + setColorModel( ImageIo::CM_RGB ); + if( mat.channels() == 4 ) + setChannelOrder( ImageIo::BGRA ); + else + setChannelOrder( ImageIo::BGR ); + } + else if( mat.channels() == 1 ) { + setColorModel( ImageIo::CM_GRAY ); + setChannelOrder( ImageIo::Y ); + } + + switch( mat.depth() ) { + case CV_8U: setDataType( ImageIo::UINT8 ); break; + case CV_16U: setDataType( ImageIo::UINT16 ); break; + case CV_32F: setDataType( ImageIo::FLOAT32 ); break; + default: + throw ImageIoExceptionIllegalDataType(); + } + + mRowBytes = (int32_t)mat.step; + mData = reinterpret_cast( mat.data ); + } + + void load( ImageTargetRef target ) { + // get a pointer to the ImageSource function appropriate for handling our data configuration + ImageSource::RowFunc func = setupRowFunc( target ); + + const uint8_t *data = mData; + for( int32_t row = 0; row < mHeight; ++row ) { + ((*this).*func)( target, row, data ); + data += mRowBytes; + } + } + + const uint8_t *mData; + int32_t mRowBytes; +}; + +////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ImageTargetCvMat +inline ImageTargetCvMat::ImageTargetCvMat( cv::Mat *mat ) + : ImageTarget(), mMat( mat ) +{ + switch( mat->depth() ) { + case CV_8U: setDataType( ImageIo::UINT8 ); break; + case CV_16U: setDataType( ImageIo::UINT16 ); break; + case CV_32F: setDataType( ImageIo::FLOAT32 ); break; + default: + throw ImageIoExceptionIllegalDataType(); + } + + switch( mat->channels() ) { + case 1: + setColorModel( ImageIo::CM_GRAY ); + setChannelOrder( ImageIo::Y ); + break; + case 3: + setColorModel( ImageIo::CM_RGB ); + setChannelOrder( ImageIo::BGR ); + break; + case 4: + setColorModel( ImageIo::CM_RGB ); + setChannelOrder( ImageIo::BGRA ); + break; + default: + throw ImageIoExceptionIllegalColorModel(); + break; + } +} + +inline cv::Mat toOcv( ci::ImageSourceRef sourceRef, int type = -1 ) +{ + if( type == -1 ) { + int depth = CV_8U; + if( sourceRef->getDataType() == ImageIo::UINT16 ) + depth = CV_16U; + else if( ( sourceRef->getDataType() == ImageIo::FLOAT32 ) || ( sourceRef->getDataType() == ImageIo::FLOAT16 ) ) + depth = CV_32F; + int channels = ImageIo::channelOrderNumChannels( sourceRef->getChannelOrder() ); + type = CV_MAKETYPE( depth, channels ); + } + + cv::Mat result( sourceRef->getHeight(), sourceRef->getWidth(), type ); + ImageTargetRef target = ImageTargetCvMat::createRef( &result ); + sourceRef->load( target ); + return result; +} + +inline cv::Mat toOcvRef( Channel8u &channel ) +{ + return cv::Mat( channel.getHeight(), channel.getWidth(), CV_MAKETYPE( CV_8U, 1 ), channel.getData(), channel.getRowBytes() ); +} + +inline cv::Mat toOcvRef( Channel16u &channel ) +{ + return cv::Mat( channel.getHeight(), channel.getWidth(), CV_MAKETYPE( CV_16U, 1 ), channel.getData(), channel.getRowBytes() ); +} + +inline cv::Mat toOcvRef( Channel32f &channel ) +{ + return cv::Mat( channel.getHeight(), channel.getWidth(), CV_MAKETYPE( CV_32F, 1 ), channel.getData(), channel.getRowBytes() ); +} + +inline cv::Mat toOcvRef( Surface8u &surface ) +{ + return cv::Mat( surface.getHeight(), surface.getWidth(), CV_MAKETYPE( CV_8U, surface.hasAlpha()?4:3), surface.getData(), surface.getRowBytes() ); +} + +inline cv::Mat toOcvRef( Surface16u &surface ) +{ + return cv::Mat( surface.getHeight(), surface.getWidth(), CV_MAKETYPE( CV_16U, surface.hasAlpha()?4:3), surface.getData(), surface.getRowBytes() ); +} + +inline cv::Mat toOcvRef( Surface32f &surface ) +{ + return cv::Mat( surface.getHeight(), surface.getWidth(), CV_MAKETYPE( CV_32F, surface.hasAlpha()?4:3), surface.getData(), surface.getRowBytes() ); +} + +inline ImageSourceRef fromOcv( cv::Mat &mat ) +{ + return ImageSourceRef( new ImageSourceCvMat( mat ) ); +} + +inline ImageSourceRef fromOcv( cv::UMat &umat ) +{ + return ImageSourceRef( new ImageSourceCvMat( umat.getMat( cv::ACCESS_READ ) ) ); +} + +inline cv::Scalar toOcv( const Color &color ) +{ + return CV_RGB( color.r * 255, color.g * 255, color.b * 255 ); +} + +inline vec2 fromOcv( const cv::Point2f &point ) +{ + return vec2( point.x, point.y ); +} + +inline cv::Point2f toOcv( const vec2 &point ) +{ + return cv::Point2f( point.x, point.y ); +} + +inline ivec2 fromOcv( const cv::Point &point ) +{ + return ivec2( point.x, point.y ); +} + +inline cv::Point toOcv( const ivec2 &point ) +{ + return cv::Point( point.x, point.y ); +} + +inline cv::Rect toOcv( const ci::Area &r ) +{ + return cv::Rect( r.x1, r.y1, r.getWidth(), r.getHeight() ); +} + +inline ci::Area fromOcv( const cv::Rect &r ) +{ + return Area( r.x, r.y, r.x + r.width, r.y + r.height ); +} + +} // namespace cinder \ No newline at end of file diff --git a/include/maskade_classifier.hpp b/include/maskade_classifier.hpp index 8520319..d4eb49b 100644 --- a/include/maskade_classifier.hpp +++ b/include/maskade_classifier.hpp @@ -1,53 +1,94 @@ #pragma once +// This header MUST be included first (do not reorder includes) +#include "opencv2/opencv.hpp" + #include "cinder/app/App.h" -#include "cinder/app/RendererGl.h" #include "cinder/gl/gl.h" +#include "cppflow/cppflow.h" -#include -#include -#include "cppflow/cppflow.h" -#include "cppflow/model.h" -#include "cppflow/ops.h" -#include "opencv2/core.hpp" -#include "opencv2/core/core.hpp" -#include "opencv2/core/mat.hpp" -#include "opencv2/highgui.hpp" -#include "opencv2/highgui/highgui.hpp" -#include "opencv2/imgcodecs/imgcodecs.hpp" -#include "opencv2/imgproc.hpp" -#include "opencv2/imgproc/imgproc.hpp" -#include "opencv2/video/video.hpp" -#include "opencv2/videoio.hpp" -using cv::FONT_HERSHEY_COMPLEX; -using cv::LINE_AA; -using cv::Mat; -using cv::Scalar; -using cv::VideoCapture; -using cv::waitKey; -using std::cout; -using std::endl; - -namespace maskade { +namespace maskade { class MaskadeClassifier : public ci::app::App { - public: + public: + /** + * @brief Default constructor that initializes the model using the model path. + * + */ MaskadeClassifier(); - void Run(); + + /** + * @brief Sets up configuration file variables and opens camera for Cinder. + * + */ void setup() override; - void mouseDown(ci::app::MouseEvent event) override; + + /** + * @brief Reads in a new image from the camera every time the Cinder app + * updates. + * + */ void update() override; + + /** + * @brief Calls the appropriate functions to draw the camera feed and + * classification on the Cinder app. + * + */ void draw() override; + private: - -}; + /** + * @brief Opens the laptop camera for video access using OpenCV. + * + */ + void OpenCamera(); + + /** + * @brief Conducts appropriate image transformations and draws the current + * image as a texture on the Cinder app. + * + */ + void DrawImage(); -} + /** + * @brief Passes the image into the TensorFlow model to get a prediction of + * the user's mask status. + * + */ + float CalculatePrediction(); + + /** + * @brief Draws the appropriate message to the Cinder app based on the model's + * prediction. + * + */ + void DrawPrediction(int prediction_class); + + // Relative path to the configuration JSON + std::string config_path_ = "../../../../../../config/config.json"; + // Relative path to the cached model file + std::string model_path_ = + "../../../../../../assets/converted_savedmodel/model.savedmodel"; + // The cppflow wrapper object for a TensorFlow model + cppflow::model model_; + // The width of the image to be input to the model + int model_image_width_; + // The height of the image to be input to the model + int model_image_height_; + // The OpenCV object that allows Cinder to read in camera footage + cv::VideoCapture capture_; + // The OpenCV matrix object representing the current image being processed + cv::Mat image_; + // The string name of the font for printing text + std::string font_name_; + // The Cinder color of the font + ci::ColorT font_color_; + // The width of the window + size_t window_width_; + // The height of the window + size_t window_height_; +}; -// void drawText(Mat& image) { -// putText(image, "Hello OpenCV", cv::Point(20, 50), FONT_HERSHEY_COMPLEX, -// 1, // font face and scale -// Scalar(255, 255, 255), // white -// 1, LINE_AA); // line thickness and type -// } \ No newline at end of file +} // namespace maskade diff --git a/src/maskade_classifier.cpp b/src/maskade_classifier.cpp index 68fe5ea..f15a82d 100644 --- a/src/maskade_classifier.cpp +++ b/src/maskade_classifier.cpp @@ -1,140 +1,140 @@ #include "maskade_classifier.hpp" +#include "CinderOpenCV.hpp" +#include "cinder/app/RendererGl.h" +#include "nlohmann/json.hpp" +#include "opencv2/core/mat.hpp" + namespace maskade { -MaskadeClassifier::MaskadeClassifier() { - +MaskadeClassifier::MaskadeClassifier() : model_(model_path_) { } -void MaskadeClassifier::Run() { - // Read in a sample image (hopefully, this will later be from the camera feed) - cppflow::tensor input = cppflow::decode_jpeg(cppflow::read_file( - std::string("/Users/rustomichhaporia/GitHub/Cinder/my-projects/" - "final-project-rustom-ichhaporia/assets/photo3.jpeg"))); - - std::cout << input; - // Cast the datatype of the input, expand dimensions, and change size to match - // the image size of the model - input = cppflow::cast(input, TF_UINT8, TF_FLOAT); - input = input / 255.f; - input = cppflow::expand_dims(input, 0); - std::cout << input.shape(); - auto il = {224, 224}; - input = cppflow::resize_bilinear(input, cppflow::tensor(il)); - std::cout << input.shape(); - - // Load in the saved model built online with Google's Teachable Machines - // project https://teachablemachine.withgoogle.com/train - cppflow::model model( - "/Users/rustomichhaporia/GitHub/Cinder/my-projects/" - "final-project-rustom-ichhaporia/assets/converted_savedmodel/" - "model.savedmodel"); - - // Print list of possible operations on the Tensor model - // std::vector ops = model.get_operations(); - // for (auto item : ops) { - // std::cout << item << std::endl << std::endl; - // } - - // Output the prediction from the model - auto output = model(input); - std::cout << output; - - // return 0; - - // The code below connects OpenCV binaries built locally to the laptop's - // camera feed Some code is taken from online OpenCV examples for proof of - // concept This can only be done in superuser mode on VS code - - int IMG_SIZE = 224; - - std::cout << output << std::endl; - - cout << "Built with OpenCV " << CV_VERSION << endl; - cv::Mat image; - VideoCapture capture; - capture.open(0); - if (capture.isOpened()) { - cout << "Capture is opened" << endl; - for (;;) { - // for (size_t i = 0; i < 1; ++i) { - capture >> image; - // cv::flip(image, image, 1); - // image = image(cv::Rect(540, 360, IMG_SIZE, IMG_SIZE)); - - std::cout << image.size; - image.convertTo(image, CV_32F); - cv::cvtColor(image, image, cv::COLOR_BGR2RGB); - image /= 255.f; - - // Image dimensions - int rows = image.rows; - int cols = image.cols; - int channels = image.channels(); - int total = image.total(); - - // Assign to vector for 3 channel image - // Souce: https://stackoverflow.com/a/56600115/2076973 - Mat flat = image.reshape(1, image.total() * channels); - - std::vector img_data(IMG_SIZE * IMG_SIZE * 3); - img_data = image.isContinuous() ? flat : flat.clone(); - cppflow::tensor tensor(img_data, {1, rows, cols, channels}); - std::cout << tensor.dtype(); - // tensor = tensor/255.f; - auto dims = {224, 224}; - tensor = cppflow::resize_bilinear(tensor, cppflow::tensor(dims)); - - auto output_2 = model(tensor); - - auto argmax = cppflow::arg_max(output_2, 1).get_data(); - - if (argmax[0] == 0) { - putText(image, "PUT ON YOUR MASK", cv::Point(20, 50), - FONT_HERSHEY_COMPLEX, - 1, // font face and scale - Scalar(255, 255, 255), // white - 1, LINE_AA); // line thickness and type - } else { - putText(image, "THANKS FOR WEARING YOUR MASK!", cv::Point(20, 50), - FONT_HERSHEY_COMPLEX, - 1, // font face and scale - Scalar(255, 255, 255), // white - 1, LINE_AA); // line thickness and type - } - imshow("Sample", image); - - std::cout << "This is the prediction" << cppflow::arg_max(output_2, 1); - - if (image.empty()) - break; - // drawText(image); - if (waitKey(10) >= 0) - break; - } - } else { - cout << "No capture" << endl; - image = Mat::zeros(480, 640, CV_8UC1); - // drawText(image); - imshow("Sample", image); - waitKey(0); - } +void MaskadeClassifier::setup() { + // Reads config variables from JSON file + std::ifstream input(config_path_); + nlohmann::json config; + input >> config; + + // Set font variables + font_name_ = std::string(config["font"]); + font_color_ = ci::ColorT().hex( + uint32_t(std::stoull(std::string(config["font_color"]), 0, 16))); + + // Set dimensions for model input + model_image_width_ = config["model_image_width"].get(); + model_image_height_ = config["model_image_height"].get(); + + // Begin collecting video + OpenCamera(); } -void MaskadeClassifier::setup() { +void MaskadeClassifier::update() { + // Read in the most recent snapshot from video feed to current image + capture_ >> image_; +} + +void MaskadeClassifier::draw() { + // Executes the drawing and calculation heartbeat functions + DrawImage(); + + int prediction = CalculatePrediction(); + + DrawPrediction(prediction); } -void MaskadeClassifier::mouseDown(ci::app::MouseEvent event) { +void MaskadeClassifier::OpenCamera() { + // Opens the default video feed accessible by OpenCV if possible + capture_.open(0); + if (capture_.isOpened()) { + std::cout << "Video capture is opened." << std::endl; + // Read in first image to set window size + update(); + + // Set window size to match the size of the video feed + window_width_ = image_.cols; + window_height_ = image_.rows; + ci::app::setWindowSize(window_width_, window_height_); + } + + else { + // Prints error message if camera cannot be opened + std::cout << "No video capture is available. You may need to enable " + "Superuser permissions." + << std::endl; + // Display empty image + auto image = cv::Mat::zeros(1000, 1000, CV_8UC1); + cv::imshow("", image); + cv::waitKey(0); + } } -void MaskadeClassifier::update() { +void MaskadeClassifier::DrawImage() { + // Converts image data to OpenCV data type + image_.convertTo(image_, CV_32F); + // Normalizes the RGB values of the data in the image + image_ /= 255.0; + // Creates Cinder texture from OpenCV Mat + ci::gl::TextureRef texture = ci::gl::Texture::create(ci::fromOcv(image_)); + // Draw texture on the Cinder app + ci::gl::draw(texture); } -void MaskadeClassifier::draw() { - Run(); +float MaskadeClassifier::CalculatePrediction() { + // Switches the color schema of the image form BGR (openCV native format) to + // RGB (TensorFlow native format) + cv::cvtColor(image_, image_, cv::COLOR_BGR2RGB); + + // Assign to vector for 3 channel image + // Souce: https://stackoverflow.com/a/56600115/2076973 + cv::Mat flat_mat = image_.reshape(1, image_.total() * image_.channels()); + + // Create a vector of floats representing the colors of each pixel (flattened + // image matrix) + std::vector flat_data(image_.total() * image_.channels()); + flat_data = image_.isContinuous() ? flat_mat : flat_mat.clone(); + + // Create a TensorFlow tensor with the appropriate shape + cppflow::tensor input_tensor( + flat_data, {1, image_.rows, image_.cols, image_.channels()}); + + // Resize input tensor to fit the dimensions of image that the model is + // expecting (change from video dimensions to model dimensions) + input_tensor = cppflow::resize_bilinear( + input_tensor, cppflow::tensor({model_image_width_, model_image_height_})); + + // Get output probabilities from the model + auto output = model_(input_tensor); + + // Select the class with the highest likelihood + auto argmax = cppflow::arg_max(output, 1).get_data(); + + return argmax[0]; +} + +void MaskadeClassifier::DrawPrediction(int prediction_class) { + // Draw background box so text can more easily be seen + ci::Rectf text_box(glm::vec2(window_width_ * 1 / 10, window_height_ * 8 / 10), + glm::vec2(window_width_ * 9 / 10, window_height_)); + + // Dark grey box + ci::gl::color(ci::ColorAT().hex(0xAB303030)); + ci::gl::drawSolidRoundedRect(text_box, 15); + + // Reset brush for drawing images + ci::gl::color(ci::ColorT().hex(0xffffff)); + + // Determine message based on calculated classification + std::string output_line = (prediction_class == 0) + ? "Hey, your mask isn't on!" + : "Thank you for wearing your mask!"; + + ci::gl::drawStringCentered( + output_line, glm::vec2(window_width_ / 2, window_height_ * 9 / 10), + font_color_, ci::Font(font_name_, window_height_ / 20)); } } // namespace maskade \ No newline at end of file diff --git a/test/test_maskade_classifier.cpp b/test/test_maskade_classifier.cpp new file mode 100644 index 0000000..b12765e --- /dev/null +++ b/test/test_maskade_classifier.cpp @@ -0,0 +1,32 @@ +// // Read in a sample image (hopefully, this will later be from the camera feed) +// cppflow::tensor input = cppflow::decode_jpeg(cppflow::read_file( +// std::string("/Users/rustomichhaporia/GitHub/Cinder/my-projects/" +// "final-project-rustom-ichhaporia/assets/photo3.jpeg"))); + +// std::cout << input; +// // Cast the datatype of the input, expand dimensions, and change size to match +// // the image size of the model +// input = cppflow::cast(input, TF_UINT8, TF_FLOAT); +// input = input / 255.f; +// input = cppflow::expand_dims(input, 0); +// std::cout << input.shape(); +// auto il = {224, 224}; +// input = cppflow::resize_bilinear(input, cppflow::tensor(il)); +// std::cout << input.shape(); + +// // Load in the saved model built online with Google's Teachable Machines +// // project https://teachablemachine.withgoogle.com/train +// cppflow::model model( +// "/Users/rustomichhaporia/GitHub/Cinder/my-projects/" +// "final-project-rustom-ichhaporia/assets/converted_savedmodel/" +// "model.savedmodel"); + +// // Print list of possible operations on the Tensor model +// // std::vector ops = model.get_operations(); +// // for (auto item : ops) { +// // std::cout << item << std::endl << std::endl; +// // } + +// // Output the prediction from the model +// auto output = model(input); +// std::cout << output; \ No newline at end of file