Move annotation detection to separate thread

diff --git a/core/annotationmodel.cpp b/core/annotationmodel.cpp
index edeaf74d5a471537654674785a71a343ecb43cf7..a1bf838e8f7addab9f9b80cfe2319c333a0599c9 100644 (file)
--- a/core/annotationmodel.cpp
+++ b/core/annotationmodel.cpp
@@ -10,7 +10,7 @@ void AnnotationModel::setAnnotations(
 }
 
 QHash<int, QByteArray> AnnotationModel::roleNames() const {
-static QHash<int, QByteArray> roles{
+  static QHash<int, QByteArray> roles{
     {Qt::DisplayRole, "text"},
     {AnnotationX, "annotationX"},
     {AnnotationY, "annotationY"},

diff --git a/featureplugins/feature_annotations/CMakeLists.txt b/featureplugins/feature_annotations/CMakeLists.txt
index 8a8d5ca9cca9b5388ca0b8b1eb5b89ea9dc9db60..cb2db02ff4c0112ca0f9a8df9464b6e5cc341546 100644 (file)
--- a/featureplugins/feature_annotations/CMakeLists.txt
+++ b/featureplugins/feature_annotations/CMakeLists.txt
@@ -4,6 +4,7 @@ project(feature_annotations)
 find_package(Qt5 ${QT_MIN_VERSION} CONFIG REQUIRED COMPONENTS
     Core
     Gui
+    Concurrent
 )
 
 find_package(dlib CONFIG REQUIRED)
@@ -17,6 +18,7 @@ endif()
 set(feature_annotations_LIBS
     Qt5::Core
     Qt5::Gui
+    Qt5::Concurrent
     ${DLIB_LIBRARY_NAME}
     pluginapi
 )

diff --git a/featureplugins/feature_annotations/featureannotations.cpp b/featureplugins/feature_annotations/featureannotations.cpp
index 73314fe508d8a4174184398bfefd268ab7aebf93..729c57159582fd41156a58dd585f67503a54191f 100644 (file)
--- a/featureplugins/feature_annotations/featureannotations.cpp
+++ b/featureplugins/feature_annotations/featureannotations.cpp
@@ -1,39 +1,58 @@
 #include "featureannotations.h"
 
-#include <QLoggingCategory>
 #include <QImage>
+#include <QLoggingCategory>
 
 Q_LOGGING_CATEGORY(annotationsCategory, "Annotations")
+Q_LOGGING_CATEGORY(annotationsVerboseCategory, "Annotations.verbose")
 
 #include <dlib/dnn.h>
 #include <dlib/image_saver/image_saver.h>
 
+#include <QMetaObject>
+#include <QtConcurrent>
+
 // random windows.h defines
 #undef interface
 
 using namespace std;
 using namespace dlib;
 
+template <long num_filters, typename SUBNET>
+using con5d = con<num_filters, 5, 5, 2, 2, SUBNET>;
+template <long num_filters, typename SUBNET>
+using con5 = con<num_filters, 5, 5, 1, 1, SUBNET>;
 
-template <long num_filters, typename SUBNET> using con5d = con<num_filters,5,5,2,2,SUBNET>;
-template <long num_filters, typename SUBNET> using con5  = con<num_filters,5,5,1,1,SUBNET>;
+template <typename SUBNET>
+using downsampler_r = relu<affine<
+    con5d<32, relu<affine<con5d<32, relu<affine<con5d<16, SUBNET>>>>>>>>>;
+template <typename SUBNET> using rcon5_r = relu<affine<con5<45, SUBNET>>>;
 
-template <typename SUBNET> using downsampler_r  = relu<affine<con5d<32, relu<affine<con5d<32, relu<affine<con5d<16,SUBNET>>>>>>>>>;
-template <typename SUBNET> using rcon5_r  = relu<affine<con5<45,SUBNET>>>;
-
-using detection_net_type = loss_mmod<con<1,9,9,1,1,rcon5_r<rcon5_r<rcon5_r<downsampler_r<input_rgb_image_pyramid<pyramid_down<6>>>>>>>>;
+using detection_net_type =
+    loss_mmod<con<1, 9, 9, 1, 1,
+                  rcon5_r<rcon5_r<rcon5_r<downsampler_r<
+                      input_rgb_image_pyramid<pyramid_down<6>>>>>>>>;
 
 namespace {
 detection_net_type net;
-}
+bool valid = false;
+} // namespace
 
 FeatureAnnoations::FeatureAnnoations(QObject *parent) : QObject{parent} {
-  deserialize("C:/_C/anime_face_recognition/mmod_network.dat") >> net;
+  try {
+    deserialize("C:/_C/anime_face_recognition/mmod_network.dat") >> net;
+    valid = true;
+  } catch (const std::exception &ex) {
+    qCWarning(annotationsCategory)
+        << "Failed to read neural network. Error:" << ex.what();
+  }
 }
 
 FeaturePluginInstance *
 FeatureAnnoations::createInstance(QObject *instance,
-                                   PlayerFeaturePlauginInterface *interface) {
+                                  PlayerFeaturePlauginInterface *interface) {
+  if (!valid)
+    throw std::exception{"Network failed to initialize"};
   return new FeatureAnnoationsInstance(instance, interface);
 }
 
@@ -42,53 +61,80 @@ FeatureAnnoationsInstance::FeatureAnnoationsInstance(
     : FeaturePluginInstance{instance, player} {
   qCDebug(annotationsCategory) << "Registering with instance" << instance;
 
-  connect(instance, SIGNAL(frameChanged(const QImage &)),
-          this, SLOT(onFrameChanged(const QImage &)));
+  connect(instance, SIGNAL(frameChanged(const QImage &)), this,
+          SLOT(onFrameChanged(const QImage &)));
 
+  connect(&m_watcher, SIGNAL(finished()), this, SLOT(onResultReady()));
 
   m_fpsTimer.setInterval(1000);
   connect(&m_fpsTimer, &QTimer::timeout, [&]() {
-    qCDebug(annotationsCategory) << "Did" << m_fpsCounter
-                                 << "frames per seoncd";
+    qCDebug(annotationsCategory)
+        << "Did" << m_fpsCounter << "frames per seoncd";
     m_fpsCounter = 0;
   });
   m_fpsTimer.start();
 }
 
-void FeatureAnnoationsInstance::onFrameChanged(const QImage &image)
-{
-  qCDebug(annotationsCategory) << "Frame changed! " << image.size() << image.format();
-  if (image.size().isEmpty()) return;
-  try {
-    matrix<rgb_pixel> img{image.size().height(), image.size().width()};
-
-    const auto *imageData = reinterpret_cast<const QRgb *>(image.bits());
-    auto size = img.size();
-
-    std::transform(imageData, imageData + size, img.begin(),
-                   [](const QRgb &d) {
-                     return rgb_pixel{static_cast<unsigned char>(qRed(d)),
-                                      static_cast<unsigned char>(qGreen(d)),
-                                      static_cast<unsigned char>(qBlue(d))};
-                   });
-
-    //dlib::save_bmp(img, "testimage.bmp");
-
-    auto dets = net(img);
-    PlayerFeaturePlauginInterface::AnnotationList al;
-    for (auto&& d : dets)
-      al << PlayerFeaturePlauginInterface::Annotation{
-              static_cast<double>(d.rect.left()) / image.size().width(),
-              static_cast<double>(d.rect.top()) / image.size().height(),
-              static_cast<double>(d.rect.right() - d.rect.left()) / image.size().width(),
-              static_cast<double>(d.rect.bottom() - d.rect.top()) / image.size().height(),
-              "red",
-              ""
-            };
-    m_playerInterface->featureSetAnnotations(al);
-    ++m_fpsCounter;
-
-  } catch(const std::exception &ex) {
-    qCDebug(annotationsCategory) << "Exception: " << ex.what();
-  }
+void FeatureAnnoationsInstance::onFrameChanged(const QImage &image) {
+  qCDebug(annotationsVerboseCategory)
+      << "Frame changed! " << image.size() << image.format();
+  if (image.size().isEmpty())
+    return;
+
+  if (m_watcher.isRunning())
+    return;
+
+  qCDebug(annotationsVerboseCategory) << "Starting annotation detection...";
+
+  auto future = QtConcurrent::run(
+      [](QImage image) -> PlayerFeaturePlauginInterface::AnnotationList {
+        try {
+          const int maxHeight{600};
+          if (image.size().height() > maxHeight)
+            image = image.scaledToHeight(maxHeight, Qt::FastTransformation);
+
+          matrix<rgb_pixel> img{image.size().height(), image.size().width()};
+
+          const auto *imageData = reinterpret_cast<const QRgb *>(image.bits());
+          auto size = img.size();
+
+          std::transform(
+              imageData, imageData + size, img.begin(), [](const QRgb &d) {
+                return rgb_pixel{static_cast<unsigned char>(qRed(d)),
+                                 static_cast<unsigned char>(qGreen(d)),
+                                 static_cast<unsigned char>(qBlue(d))};
+              });
+
+          // dlib::save_bmp(img, "testimage.bmp");
+
+          auto dets = net(img);
+          PlayerFeaturePlauginInterface::AnnotationList al;
+          for (auto &&d : dets)
+            al << PlayerFeaturePlauginInterface::Annotation{
+                static_cast<double>(d.rect.left()) / image.size().width(),
+                static_cast<double>(d.rect.top()) / image.size().height(),
+                static_cast<double>(d.rect.right() - d.rect.left()) /
+                    image.size().width(),
+                static_cast<double>(d.rect.bottom() - d.rect.top()) /
+                    image.size().height(),
+                "red",
+                ""};
+
+          qCDebug(annotationsVerboseCategory)
+              << "Found" << al.size() << "annotations";
+          return al;
+
+        } catch (const std::exception &ex) {
+          qCWarning(annotationsCategory) << "Exception: " << ex.what();
+        }
+        return {};
+      },
+      std::move(image));
+
+  m_watcher.setFuture(future);
+}
+
+void FeatureAnnoationsInstance::onResultReady() {
+  ++m_fpsCounter;
+  m_playerInterface->featureSetAnnotations(m_watcher.result());
 }

diff --git a/featureplugins/feature_annotations/featureannotations.h b/featureplugins/feature_annotations/featureannotations.h
index 06e6f337afbf6d3bffc5dc3298cadd78617fc30a..898ad91334645af71acd2ce550f675616e70ce64 100644 (file)
--- a/featureplugins/feature_annotations/featureannotations.h
+++ b/featureplugins/feature_annotations/featureannotations.h
@@ -7,6 +7,7 @@
 #include <QTimer>
 
 #include <QtPlugin>
+#include <QFutureWatcher>
 
 #include "aniplayer/featurepluginbase.h"
 #include "aniplayer/playerfeatureplugininterface.h"
@@ -34,12 +35,15 @@ public:
 private slots:
   void onFrameChanged(const QImage &);
 
+  void onResultReady();
+
 private:
   int m_fid;
   QString m_path;
   double m_duration;
   QTimer m_fpsTimer;
   int m_fpsCounter = 0;
+  QFutureWatcher<PlayerFeaturePlauginInterface::AnnotationList> m_watcher;
 };
 
 #endif // FeatureAnnoations_H