Add annotations feature

This feature is work in progress. Currently it is very slow and should not be enabled by default.

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f9bc421fd74169fe1ea0675f2429e613195ba8d3..be9bc8e32ce2bc00d753337b04c5acad6196eb4e 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -14,6 +14,8 @@ add_feature_info(BackendMpv WITH_BACKEND_MPV "the MPV backend")
 option(WITH_LOCALMYLIST "Build LocalMyList feature plugin" ON)
 add_feature_info(FeatureLocalMyList WITH_LOCALMYLIST "automatically mark files as watched that are in your LocalMyList")
 
+option(WITH_ANNOTATIONS "Build annotations feature plugin" ON)
+add_feature_info(FeatureAnnotations WITH_ANNOTATIONS "annotate certain features in the video")
 
 add_subdirectory(pluginapi)
 add_subdirectory(core)

diff --git a/core/annotationmodel.cpp b/core/annotationmodel.cpp
new file mode 100644 (file)
index 0000000..1fd3682
--- /dev/null
+++ b/core/annotationmodel.cpp
@@ -0,0 +1,51 @@
+#include "annotationmodel.h"
+
+AnnotationModel::AnnotationModel(QObject *parent) : QAbstractListModel{parent} {}
+
+void AnnotationModel::setAnnotations(
+    const PlayerFeaturePlauginInterface::AnnotationList &data) {
+  beginResetModel();
+  m_data = data;
+  endResetModel();
+}
+
+QHash<int, QByteArray> AnnotationModel::roleNames() const {
+static QHash<int, QByteArray> roles{
+    {Qt::DisplayRole, "text"},
+    {AnnotationX, "annotationX"},
+    {AnnotationY, "annotationY"},
+    {AnnotationW, "annotationW"},
+    {AnnotationH, "annotationH"},
+    {AnnotationColor, "annotationColor"},
+    {AnnotationText, "annotationText"},
+  };
+  return roles;
+}
+
+int AnnotationModel::rowCount(const QModelIndex &parent) const {
+  if (parent.isValid())
+    return 0;
+  return m_data.size();
+}
+
+QVariant AnnotationModel::data(const QModelIndex &index, int role) const {
+
+  const PlayerFeaturePlauginInterface::Annotation &annotation =
+      m_data.at(index.row());
+  switch (role) {
+  case Qt::DisplayRole:
+  case AnnotationText:
+    return annotation.text;
+  case AnnotationX:
+    return annotation.x;
+  case AnnotationY:
+    return annotation.y;
+  case AnnotationW:
+    return annotation.w;
+  case AnnotationH:
+    return annotation.h;
+  case AnnotationColor:
+    return annotation.color;
+  }
+  return {};
+}

diff --git a/core/annotationmodel.h b/core/annotationmodel.h
new file mode 100644 (file)
index 0000000..492acad
--- /dev/null
+++ b/core/annotationmodel.h
@@ -0,0 +1,32 @@
+#ifndef ANNOTATIONMODEL_H
+#define ANNOTATIONMODEL_H
+
+#include <QAbstractListModel>
+#include "aniplayer/playerfeatureplugininterface.h"
+
+class AnnotationModel : public QAbstractListModel
+{
+  Q_OBJECT
+public:
+  enum TrackRoles {
+    AnnotationX = Qt::UserRole + 1,
+    AnnotationY,
+    AnnotationW,
+    AnnotationH,
+    AnnotationColor,
+    AnnotationText,
+  };
+
+  explicit AnnotationModel(QObject *parent = nullptr);
+
+  void setAnnotations(const  PlayerFeaturePlauginInterface::AnnotationList &);
+
+  QHash<int, QByteArray> roleNames() const override;
+  int rowCount(const QModelIndex &parent = QModelIndex{}) const override;
+  QVariant data(const QModelIndex &index, int role) const override;
+
+private:
+  PlayerFeaturePlauginInterface::AnnotationList m_data;
+};
+
+#endif // ANNOTATIONMODEL_H

diff --git a/core/instancemanager.cpp b/core/instancemanager.cpp
index c07647d8c0e5c1219e6fd739db1ced1074c40d95..746b2f9da95b669b7b4f92271faf7ec7333099e9 100644 (file)
--- a/core/instancemanager.cpp
+++ b/core/instancemanager.cpp
@@ -41,7 +41,7 @@ InstanceManager::InstanceManager(QObject *parent)
   m_featurePluginManager->setPluginDirectories(featurePluginPaths);
   m_featurePluginManager->setPluginPrefix("feature");
   // TODO feature plugins should be looked up and not preferred.
-  m_featurePluginManager->setPreferredPlugins({"localmylist"});
+  m_featurePluginManager->setPreferredPlugins({"localmylist", "annotations"});
   m_featurePluginManager->loadAll();
 }
 

diff --git a/core/player.cpp b/core/player.cpp
index ac5bfc49c3c4016aa88c41673ed13b410840cb66..919ef655f4f31a92f9f60c42a57873bb064fadae 100644 (file)
--- a/core/player.cpp
+++ b/core/player.cpp
@@ -2,6 +2,7 @@
 
 #include <QLoggingCategory>
 #include <QQmlApplicationEngine>
+#include <QMetaObject>
 
 #ifdef Q_OS_WIN
 #include <Windows.h>
@@ -38,6 +39,9 @@ Player::Player(BackendPluginBase *backendPlugin, QObject *parent)
 
   m_chapterModel = new ChapterModel{this};
   Q_CHECK_PTR(m_chapterModel);
+
+  m_annotationModel = new AnnotationModel{this};
+  Q_CHECK_PTR(m_annotationModel);
 }
 
 Player::~Player() {
@@ -105,6 +109,23 @@ QAbstractItemModel *Player::subtitleTrackModel() const {
 
 QAbstractItemModel *Player::chapterModel() const { return m_chapterModel; }
 
+QAbstractItemModel *Player::annotationModel() const {
+  return m_annotationModel;
+}
+
+bool Player::shouldSaveFramesToImage() const
+{
+  return true;
+}
+
+void Player::newFrame(const QImage &image)
+{
+  m_lastFrame = image;
+  // This is called from VideoElement::synchronize which should do as little as
+  // possible
+  QMetaObject::invokeMethod(this, "onNewFrame");
+}
+
 void Player::load(const QUrl &resource) {
   if (canLoadVideoNow())
     m_backend->open(resource);
@@ -304,11 +325,14 @@ void Player::backendCurrentSubtitleTrackChanged(Player::TrackIndex track) {
   emit currentSubtitleTrackChanged(track);
 }
 
-void Player::featureShowStatusMessage(const QString &message)
-{
+void Player::featureShowStatusMessage(const QString &message) {
   emit statusMessageRequested(message);
 }
 
+void Player::featureSetAnnotations(const AnnotationList &annotations) {
+  m_annotationModel->setAnnotations(annotations);
+}
+
 void Player::reqisterQmlTypes() {
   qRegisterMetaType<TimeStamp>("TimeStamp");
   qRegisterMetaType<TrackIndex>("StreamIndex");
@@ -316,6 +340,10 @@ void Player::reqisterQmlTypes() {
   qmlRegisterType<Player>("org.aptx.aniplayer", 1, 0, "Player");
 }
 
+void Player::onNewFrame() {
+  emit frameChanged(m_lastFrame);
+}
+
 bool Player::canLoadVideoNow() const {
   return m_backendInstanceReady && m_renderer && m_rendererReady;
 }

diff --git a/core/player.h b/core/player.h
index 08248417974cd22867bd8145974a74b79e9aee4f..4c2b4cb3bdcc80c510082e1b533f63285bd49e38 100644 (file)
--- a/core/player.h
+++ b/core/player.h
@@ -5,11 +5,13 @@
 #include <QObject>
 #include <QQmlEngine>
 #include <QUrl>
+#include <QImage>
 
 #include "aniplayer/backendpluginbase.h"
 #include "aniplayer/playerfeatureplugininterface.h"
 #include "chaptermodel.h"
 #include "trackmodel.h"
+#include "annotationmodel.h"
 
 class Player : public QObject,
                public PlayerPluginInterface,
@@ -51,6 +53,8 @@ class Player : public QObject,
                  NOTIFY subtitleTrackModelChanged)
   Q_PROPERTY(QAbstractItemModel *chapterModel READ chapterModel NOTIFY
                  chapterModelChanged)
+  Q_PROPERTY(QAbstractItemModel *annotationModel READ annotationModel NOTIFY
+                 annotationModelChanged)
 
 public:
   using TrackIndex = int;
@@ -99,6 +103,10 @@ public:
   QAbstractItemModel *audioTrackModel() const;
   QAbstractItemModel *subtitleTrackModel() const;
   QAbstractItemModel *chapterModel() const;
+  QAbstractItemModel *annotationModel() const;
+
+  bool shouldSaveFramesToImage() const;
+  void newFrame(const QImage &);
 
 signals:
   void stateChanged(PlayState state);
@@ -125,9 +133,12 @@ signals:
   void audioTrackModelChanged(QAbstractItemModel *audioTrackModel);
   void subtitleTrackModelChanged(QAbstractItemModel *subtitleTrackModel);
   void chapterModelChanged(QAbstractItemModel *chapterModel);
+  void annotationModelChanged(QAbstractItemModel *annotationModel);
 
   void statusMessageRequested(const QString &message);
 
+  void frameChanged(const QImage &frame);
+
 public slots:
   // Basic Play state
   void load(const QUrl &resource);
@@ -175,10 +186,14 @@ protected:
   void backendCurrentSubtitleTrackChanged(TrackIndex) override;
 
   void featureShowStatusMessage(const QString &message) override;
+  void featureSetAnnotations(const AnnotationList &) override;
 
 public:
   static void reqisterQmlTypes();
 
+private slots:
+  void onNewFrame();
+
 private:
   bool canLoadVideoNow() const;
   void loadNextFile();
@@ -202,6 +217,9 @@ private:
   TrackModel *m_audioTrackModel;
   TrackModel *m_subtitleTrackModel;
   ChapterModel *m_chapterModel;
+  AnnotationModel *m_annotationModel;
+  QImage m_lastFrame;
+  bool m_saveToImage = true;
   bool m_muted = false;
   bool m_backendInstanceReady = false;
   bool m_rendererReady = false;

diff --git a/core/videoelement.cpp b/core/videoelement.cpp
index fcf7d3daafc897b64fb12ef44ffb828fa5a8fd0d..e7f0a6a2c4e713e2ec0951363c0c50a400b3dee5 100644 (file)
--- a/core/videoelement.cpp
+++ b/core/videoelement.cpp
@@ -2,6 +2,7 @@
 #include "player.h"
 
 #include <QLoggingCategory>
+#include <QOpenGLFramebufferObject>
 
 Q_LOGGING_CATEGORY(videoElementCategory, "VideoElement")
 
@@ -49,10 +50,19 @@ VideoElement::Renderer::~Renderer() { delete m_renderer; }
 void VideoElement::Renderer::render() {
   if (m_renderer)
     m_renderer->render(framebufferObject());
+  if (m_savetoImage)
+    m_image = framebufferObject()->toImage(false);
 }
 
 void VideoElement::Renderer::synchronize(QQuickFramebufferObject *object) {
-  auto ve = static_cast<VideoElement *>(object);
+  const auto ve = static_cast<VideoElement *>(object);
+
+  if (ve->source()) {
+    m_savetoImage = ve->source()->shouldSaveFramesToImage();
+    if (m_savetoImage)
+      ve->source()->newFrame(m_image);
+  }
+
   if (!ve->rendererUpdateRequired)
     return;
   ve->rendererUpdateRequired = false;

diff --git a/core/videoelement.h b/core/videoelement.h
index 1fb7d4ad78ee74a0daea8fd27381659d29f5fcba..55cf5bbc470c8f62ace322db9be81cf84423f401 100644 (file)
--- a/core/videoelement.h
+++ b/core/videoelement.h
@@ -6,6 +6,8 @@
 #include "player.h"
 #include "aniplayer/playerplugininterface.h"
 
+#include <QImage>
+
 class VideoElement : public QQuickFramebufferObject,
                      public VideoUpdateInterface {
   Q_OBJECT
@@ -19,6 +21,8 @@ class VideoElement : public QQuickFramebufferObject,
 
   private:
     VideoRendererBase *m_renderer = nullptr;
+    bool m_savetoImage = false;
+    QImage m_image;
   };
 
   Player *m_source = nullptr;

diff --git a/featureplugins/CMakeLists.txt b/featureplugins/CMakeLists.txt
index acee97d7a483dd6737ca335bed8964fc4c8edc51..e9e49b1ca948bc884fa95cc50a2718313318ef56 100644 (file)
--- a/featureplugins/CMakeLists.txt
+++ b/featureplugins/CMakeLists.txt
@@ -5,3 +5,7 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/featureplugins)
 if (WITH_LOCALMYLIST)
     add_subdirectory(feature_localmylist)
 endif()
+
+if (WITH_ANNOTATIONS)
+    add_subdirectory(feature_annotations)
+endif()

diff --git a/featureplugins/feature_annotations/CMakeLists.txt b/featureplugins/feature_annotations/CMakeLists.txt
new file mode 100644 (file)
index 0000000..7164c38
--- /dev/null
+++ b/featureplugins/feature_annotations/CMakeLists.txt
@@ -0,0 +1,54 @@
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+project(feature_annotations)
+
+find_package(Qt5 ${QT_MIN_VERSION} CONFIG REQUIRED COMPONENTS
+    Core
+    Gui
+)
+
+find_package(dlib CONFIG REQUIRED)
+
+set(feature_annotations_LIBS
+    Qt5::Core
+    Qt5::Gui
+    dlib::dlib
+    pluginapi
+)
+
+set(feature_annotations_SOURCES
+    featureannotations.cpp
+)
+
+set(feature_annotations_HEADERS
+    featureannotations.h
+    feature_annotations_global.h
+)
+
+set(CMAKE_AUTOMOC ON)
+set(CMAKE_AUTOUIC ON)
+set(CMAKE_AUTORCC ON)
+
+add_library(feature_annotations MODULE
+    ${feature_annotations_SOURCES}
+    ${feature_annotations_HEADERS}
+)
+
+set_property(TARGET feature_annotations PROPERTY CXX_STANDARD 14)
+set_property(TARGET feature_annotations PROPERTY OUTPUT_NAME "feature_annotations")
+target_link_libraries(feature_annotations ${feature_annotations_LIBS})
+
+add_definitions(-DFEATURE_ANNOTATIONS_LIBRARY)
+
+if(WIN32)
+    set(INSTALL_DESTINATION "aniplayer/uiplugins")
+else()
+    set(INSTALL_DESTINATION "lib${LIB_SUFFIX}/aniplayer/uiplugins")
+endif()
+
+install(TARGETS feature_annotations
+    LIBRARY DESTINATION ${INSTALL_DESTINATION}
+    ARCHIVE DESTINATION ${INSTALL_DESTINATION}
+)
+install(FILES $<TARGET_PDB_FILE:feature_annotations>
+    DESTINATION ${INSTALL_DESTINATION} OPTIONAL
+)

diff --git a/featureplugins/feature_annotations/feature_annotations.json b/featureplugins/feature_annotations/feature_annotations.json
new file mode 100644 (file)
index 0000000..e69de29

diff --git a/featureplugins/feature_annotations/feature_annotations_global.h b/featureplugins/feature_annotations/feature_annotations_global.h
new file mode 100644 (file)
index 0000000..338bb53
--- /dev/null
+++ b/featureplugins/feature_annotations/feature_annotations_global.h
@@ -0,0 +1,12 @@
+#ifndef FEATURE_ANNOTATIONS_GLOBAL_H
+#define FEATURE_ANNOTATIONS_GLOBAL_H
+
+#include <QtCore/qglobal.h>
+
+#if defined(FEATURE_ANNOTATIONS_LIBRARY)
+#  define FEATURE_ANNOTATIONSSHARED_EXPORT Q_DECL_EXPORT
+#else
+#  define FEATURE_ANNOTATIONSSHARED_EXPORT Q_DECL_IMPORT
+#endif
+
+#endif // FEATURE_ANNOTATIONS_GLOBAL_H

diff --git a/featureplugins/feature_annotations/featureannotations.cpp b/featureplugins/feature_annotations/featureannotations.cpp
new file mode 100644 (file)
index 0000000..1fb0aa4
--- /dev/null
+++ b/featureplugins/feature_annotations/featureannotations.cpp
@@ -0,0 +1,94 @@
+#include "featureannotations.h"
+
+#include <QLoggingCategory>
+#include <QImage>
+
+Q_LOGGING_CATEGORY(annotationsCategory, "Annotations")
+
+#include <dlib/dnn.h>
+#include <dlib/image_saver/image_saver.h>
+
+// random windows.h defines
+#undef interface
+
+using namespace std;
+using namespace dlib;
+
+
+template <long num_filters, typename SUBNET> using con5d = con<num_filters,5,5,2,2,SUBNET>;
+template <long num_filters, typename SUBNET> using con5  = con<num_filters,5,5,1,1,SUBNET>;
+
+template <typename SUBNET> using downsampler_r  = relu<affine<con5d<32, relu<affine<con5d<32, relu<affine<con5d<16,SUBNET>>>>>>>>>;
+template <typename SUBNET> using rcon5_r  = relu<affine<con5<45,SUBNET>>>;
+
+using detection_net_type = loss_mmod<con<1,9,9,1,1,rcon5_r<rcon5_r<rcon5_r<downsampler_r<input_rgb_image_pyramid<pyramid_down<6>>>>>>>>;
+
+namespace {
+detection_net_type net;
+}
+
+FeatureAnnoations::FeatureAnnoations(QObject *parent) : QObject{parent} {
+  deserialize("mmod_network.dat") >> net;
+}
+
+FeaturePluginInstance *
+FeatureAnnoations::createInstance(QObject *instance,
+                                   PlayerFeaturePlauginInterface *interface) {
+  return new FeatureAnnoationsInstance(instance, interface);
+}
+
+FeatureAnnoationsInstance::FeatureAnnoationsInstance(
+    QObject *instance, PlayerFeaturePlauginInterface *player, QObject *)
+    : FeaturePluginInstance{instance, player} {
+  qCDebug(annotationsCategory) << "Registering with instance" << instance;
+
+  connect(instance, SIGNAL(frameChanged(const QImage &)),
+          this, SLOT(onFrameChanged(const QImage &)));
+
+
+  m_fpsTimer.setInterval(1000);
+  connect(&m_fpsTimer, &QTimer::timeout, [&]() {
+    qCDebug(annotationsCategory) << "Did" << m_fpsCounter
+                                 << "frames per seoncd";
+    m_fpsCounter = 0;
+  });
+  m_fpsTimer.start();
+}
+
+void FeatureAnnoationsInstance::onFrameChanged(const QImage &image)
+{
+  qCDebug(annotationsCategory) << "Frame changed! " << image.size() << image.format();
+  if (image.size().isEmpty()) return;
+  try {
+    matrix<rgb_pixel> img{image.size().height(), image.size().width()};
+
+    const auto *imageData = reinterpret_cast<const QRgb *>(image.bits());
+    auto size = img.size();
+
+    std::transform(imageData, imageData + size, img.begin(),
+                   [](const QRgb &d) {
+                     return rgb_pixel{static_cast<unsigned char>(qRed(d)),
+                                      static_cast<unsigned char>(qGreen(d)),
+                                      static_cast<unsigned char>(qBlue(d))};
+                   });
+
+    //dlib::save_bmp(img, "testimage.bmp");
+
+    auto dets = net(img);
+    PlayerFeaturePlauginInterface::AnnotationList al;
+    for (auto&& d : dets)
+      al << PlayerFeaturePlauginInterface::Annotation{
+              static_cast<double>(d.rect.left()) / image.size().width(),
+              static_cast<double>(d.rect.top()) / image.size().height(),
+              static_cast<double>(d.rect.right() - d.rect.left()) / image.size().width(),
+              static_cast<double>(d.rect.bottom() - d.rect.top()) / image.size().height(),
+              "red",
+              ""
+            };
+    m_playerInterface->featureSetAnnotations(al);
+    ++m_fpsCounter;
+
+  } catch(const std::exception &ex) {
+    qCDebug(annotationsCategory) << "Exception: " << ex.what();
+  }
+}

diff --git a/featureplugins/feature_annotations/featureannotations.h b/featureplugins/feature_annotations/featureannotations.h
new file mode 100644 (file)
index 0000000..06e6f33
--- /dev/null
+++ b/featureplugins/feature_annotations/featureannotations.h
@@ -0,0 +1,45 @@
+#ifndef FeatureAnnoations_H
+#define FeatureAnnoations_H
+
+#include <QHash>
+#include <QObject>
+#include <QUrl>
+#include <QTimer>
+
+#include <QtPlugin>
+
+#include "aniplayer/featurepluginbase.h"
+#include "aniplayer/playerfeatureplugininterface.h"
+#include "feature_annotations_global.h"
+
+class FEATURE_ANNOTATIONSSHARED_EXPORT FeatureAnnoations
+    : public QObject,
+      public FeaturePluginBase {
+  Q_OBJECT
+  Q_PLUGIN_METADATA(IID ANIPLAYER_FEATURE_PLUGIN_INTERFACE_IID FILE
+                    "feature_annotations.json")
+  Q_INTERFACES(FeaturePluginBase)
+public:
+  FeatureAnnoations(QObject *parent = nullptr);
+
+  FeaturePluginInstance *
+  createInstance(QObject *instance, PlayerFeaturePlauginInterface *) override;
+};
+
+class FeatureAnnoationsInstance : public QObject, public FeaturePluginInstance {
+  Q_OBJECT
+public:
+  FeatureAnnoationsInstance(QObject *instance, PlayerFeaturePlauginInterface *, QObject *parent = nullptr);
+
+private slots:
+  void onFrameChanged(const QImage &);
+
+private:
+  int m_fid;
+  QString m_path;
+  double m_duration;
+  QTimer m_fpsTimer;
+  int m_fpsCounter = 0;
+};
+
+#endif // FeatureAnnoations_H

diff --git a/uiplugins/ui_desktop_qml_default/qml/main.qml b/uiplugins/ui_desktop_qml_default/qml/main.qml
index d1bb4c672c123320b7d99b197096bae17f13773f..c393808bb88816bdf06ac088513c6f4957896da4 100644 (file)
--- a/uiplugins/ui_desktop_qml_default/qml/main.qml
+++ b/uiplugins/ui_desktop_qml_default/qml/main.qml
@@ -48,6 +48,23 @@ Window {
         anchors.fill: parent
     }
 
+    Item {
+        id: annotationView
+        anchors.fill: parent
+        Repeater {
+
+            model: player.annotationModel
+            Rectangle {
+                x: annotationX * annotationView.width
+                y: annotationY * annotationView.height
+                width: annotationW * annotationView.width
+                height: annotationH * annotationView.height
+                border.color: annotationColor
+                color: "transparent"
+            }
+        }
+    }
+
     MouseArea {
         anchors.fill: parent
         acceptedButtons: Qt.AllButtons