From: APTX Date: Thu, 23 Nov 2017 15:56:58 +0000 (+0900) Subject: Add annotations feature X-Git-Url: https://gitweb.aptx.org/?a=commitdiff_plain;h=59826fcbc58bc873a00c921856b2af71bd9c6373;p=aniplayer.git Add annotations feature This feature is work in progress. Currently it is very slow and should not be enabled by default. --- diff --git a/CMakeLists.txt b/CMakeLists.txt index f9bc421..be9bc8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,8 @@ add_feature_info(BackendMpv WITH_BACKEND_MPV "the MPV backend") option(WITH_LOCALMYLIST "Build LocalMyList feature plugin" ON) add_feature_info(FeatureLocalMyList WITH_LOCALMYLIST "automatically mark files as watched that are in your LocalMyList") +option(WITH_ANNOTATIONS "Build annotations feature plugin" ON) +add_feature_info(FeatureAnnotations WITH_ANNOTATIONS "annotate certain features in the video") add_subdirectory(pluginapi) add_subdirectory(core) diff --git a/core/annotationmodel.cpp b/core/annotationmodel.cpp new file mode 100644 index 0000000..1fd3682 --- /dev/null +++ b/core/annotationmodel.cpp @@ -0,0 +1,51 @@ +#include "annotationmodel.h" + +AnnotationModel::AnnotationModel(QObject *parent) : QAbstractListModel{parent} {} + +void AnnotationModel::setAnnotations( + const PlayerFeaturePlauginInterface::AnnotationList &data) { + beginResetModel(); + m_data = data; + endResetModel(); +} + +QHash AnnotationModel::roleNames() const { +static QHash roles{ + {Qt::DisplayRole, "text"}, + {AnnotationX, "annotationX"}, + {AnnotationY, "annotationY"}, + {AnnotationW, "annotationW"}, + {AnnotationH, "annotationH"}, + {AnnotationColor, "annotationColor"}, + {AnnotationText, "annotationText"}, + }; + return roles; +} + +int AnnotationModel::rowCount(const QModelIndex &parent) const { + if (parent.isValid()) + return 0; + return m_data.size(); +} + +QVariant AnnotationModel::data(const QModelIndex &index, int role) const { + + const PlayerFeaturePlauginInterface::Annotation &annotation = + m_data.at(index.row()); + switch (role) { + case Qt::DisplayRole: + case AnnotationText: + return annotation.text; + case AnnotationX: + return annotation.x; + case AnnotationY: + return annotation.y; + case AnnotationW: + return annotation.w; + case AnnotationH: + return annotation.h; + case AnnotationColor: + return annotation.color; + } + return {}; +} diff --git a/core/annotationmodel.h b/core/annotationmodel.h new file mode 100644 index 0000000..492acad --- /dev/null +++ b/core/annotationmodel.h @@ -0,0 +1,32 @@ +#ifndef ANNOTATIONMODEL_H +#define ANNOTATIONMODEL_H + +#include +#include "aniplayer/playerfeatureplugininterface.h" + +class AnnotationModel : public QAbstractListModel +{ + Q_OBJECT +public: + enum TrackRoles { + AnnotationX = Qt::UserRole + 1, + AnnotationY, + AnnotationW, + AnnotationH, + AnnotationColor, + AnnotationText, + }; + + explicit AnnotationModel(QObject *parent = nullptr); + + void setAnnotations(const PlayerFeaturePlauginInterface::AnnotationList &); + + QHash roleNames() const override; + int rowCount(const QModelIndex &parent = QModelIndex{}) const override; + QVariant data(const QModelIndex &index, int role) const override; + +private: + PlayerFeaturePlauginInterface::AnnotationList m_data; +}; + +#endif // ANNOTATIONMODEL_H diff --git a/core/instancemanager.cpp b/core/instancemanager.cpp index c07647d..746b2f9 100644 --- a/core/instancemanager.cpp +++ b/core/instancemanager.cpp @@ -41,7 +41,7 @@ InstanceManager::InstanceManager(QObject *parent) m_featurePluginManager->setPluginDirectories(featurePluginPaths); m_featurePluginManager->setPluginPrefix("feature"); // TODO feature plugins should be looked up and not preferred. - m_featurePluginManager->setPreferredPlugins({"localmylist"}); + m_featurePluginManager->setPreferredPlugins({"localmylist", "annotations"}); m_featurePluginManager->loadAll(); } diff --git a/core/player.cpp b/core/player.cpp index ac5bfc4..919ef65 100644 --- a/core/player.cpp +++ b/core/player.cpp @@ -2,6 +2,7 @@ #include #include +#include #ifdef Q_OS_WIN #include @@ -38,6 +39,9 @@ Player::Player(BackendPluginBase *backendPlugin, QObject *parent) m_chapterModel = new ChapterModel{this}; Q_CHECK_PTR(m_chapterModel); + + m_annotationModel = new AnnotationModel{this}; + Q_CHECK_PTR(m_annotationModel); } Player::~Player() { @@ -105,6 +109,23 @@ QAbstractItemModel *Player::subtitleTrackModel() const { QAbstractItemModel *Player::chapterModel() const { return m_chapterModel; } +QAbstractItemModel *Player::annotationModel() const { + return m_annotationModel; +} + +bool Player::shouldSaveFramesToImage() const +{ + return true; +} + +void Player::newFrame(const QImage &image) +{ + m_lastFrame = image; + // This is called from VideoElement::synchronize which should do as little as + // possible + QMetaObject::invokeMethod(this, "onNewFrame"); +} + void Player::load(const QUrl &resource) { if (canLoadVideoNow()) m_backend->open(resource); @@ -304,11 +325,14 @@ void Player::backendCurrentSubtitleTrackChanged(Player::TrackIndex track) { emit currentSubtitleTrackChanged(track); } -void Player::featureShowStatusMessage(const QString &message) -{ +void Player::featureShowStatusMessage(const QString &message) { emit statusMessageRequested(message); } +void Player::featureSetAnnotations(const AnnotationList &annotations) { + m_annotationModel->setAnnotations(annotations); +} + void Player::reqisterQmlTypes() { qRegisterMetaType("TimeStamp"); qRegisterMetaType("StreamIndex"); @@ -316,6 +340,10 @@ void Player::reqisterQmlTypes() { qmlRegisterType("org.aptx.aniplayer", 1, 0, "Player"); } +void Player::onNewFrame() { + emit frameChanged(m_lastFrame); +} + bool Player::canLoadVideoNow() const { return m_backendInstanceReady && m_renderer && m_rendererReady; } diff --git a/core/player.h b/core/player.h index 0824841..4c2b4cb 100644 --- a/core/player.h +++ b/core/player.h @@ -5,11 +5,13 @@ #include #include #include +#include #include "aniplayer/backendpluginbase.h" #include "aniplayer/playerfeatureplugininterface.h" #include "chaptermodel.h" #include "trackmodel.h" +#include "annotationmodel.h" class Player : public QObject, public PlayerPluginInterface, @@ -51,6 +53,8 @@ class Player : public QObject, NOTIFY subtitleTrackModelChanged) Q_PROPERTY(QAbstractItemModel *chapterModel READ chapterModel NOTIFY chapterModelChanged) + Q_PROPERTY(QAbstractItemModel *annotationModel READ annotationModel NOTIFY + annotationModelChanged) public: using TrackIndex = int; @@ -99,6 +103,10 @@ public: QAbstractItemModel *audioTrackModel() const; QAbstractItemModel *subtitleTrackModel() const; QAbstractItemModel *chapterModel() const; + QAbstractItemModel *annotationModel() const; + + bool shouldSaveFramesToImage() const; + void newFrame(const QImage &); signals: void stateChanged(PlayState state); @@ -125,9 +133,12 @@ signals: void audioTrackModelChanged(QAbstractItemModel *audioTrackModel); void subtitleTrackModelChanged(QAbstractItemModel *subtitleTrackModel); void chapterModelChanged(QAbstractItemModel *chapterModel); + void annotationModelChanged(QAbstractItemModel *annotationModel); void statusMessageRequested(const QString &message); + void frameChanged(const QImage &frame); + public slots: // Basic Play state void load(const QUrl &resource); @@ -175,10 +186,14 @@ protected: void backendCurrentSubtitleTrackChanged(TrackIndex) override; void featureShowStatusMessage(const QString &message) override; + void featureSetAnnotations(const AnnotationList &) override; public: static void reqisterQmlTypes(); +private slots: + void onNewFrame(); + private: bool canLoadVideoNow() const; void loadNextFile(); @@ -202,6 +217,9 @@ private: TrackModel *m_audioTrackModel; TrackModel *m_subtitleTrackModel; ChapterModel *m_chapterModel; + AnnotationModel *m_annotationModel; + QImage m_lastFrame; + bool m_saveToImage = true; bool m_muted = false; bool m_backendInstanceReady = false; bool m_rendererReady = false; diff --git a/core/videoelement.cpp b/core/videoelement.cpp index fcf7d3d..e7f0a6a 100644 --- a/core/videoelement.cpp +++ b/core/videoelement.cpp @@ -2,6 +2,7 @@ #include "player.h" #include +#include Q_LOGGING_CATEGORY(videoElementCategory, "VideoElement") @@ -49,10 +50,19 @@ VideoElement::Renderer::~Renderer() { delete m_renderer; } void VideoElement::Renderer::render() { if (m_renderer) m_renderer->render(framebufferObject()); + if (m_savetoImage) + m_image = framebufferObject()->toImage(false); } void VideoElement::Renderer::synchronize(QQuickFramebufferObject *object) { - auto ve = static_cast(object); + const auto ve = static_cast(object); + + if (ve->source()) { + m_savetoImage = ve->source()->shouldSaveFramesToImage(); + if (m_savetoImage) + ve->source()->newFrame(m_image); + } + if (!ve->rendererUpdateRequired) return; ve->rendererUpdateRequired = false; diff --git a/core/videoelement.h b/core/videoelement.h index 1fb7d4a..55cf5bb 100644 --- a/core/videoelement.h +++ b/core/videoelement.h @@ -6,6 +6,8 @@ #include "player.h" #include "aniplayer/playerplugininterface.h" +#include + class VideoElement : public QQuickFramebufferObject, public VideoUpdateInterface { Q_OBJECT @@ -19,6 +21,8 @@ class VideoElement : public QQuickFramebufferObject, private: VideoRendererBase *m_renderer = nullptr; + bool m_savetoImage = false; + QImage m_image; }; Player *m_source = nullptr; diff --git a/featureplugins/CMakeLists.txt b/featureplugins/CMakeLists.txt index acee97d..e9e49b1 100644 --- a/featureplugins/CMakeLists.txt +++ b/featureplugins/CMakeLists.txt @@ -5,3 +5,7 @@ set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/featureplugins) if (WITH_LOCALMYLIST) add_subdirectory(feature_localmylist) endif() + +if (WITH_ANNOTATIONS) + add_subdirectory(feature_annotations) +endif() diff --git a/featureplugins/feature_annotations/CMakeLists.txt b/featureplugins/feature_annotations/CMakeLists.txt new file mode 100644 index 0000000..7164c38 --- /dev/null +++ b/featureplugins/feature_annotations/CMakeLists.txt @@ -0,0 +1,54 @@ +set(CMAKE_INCLUDE_CURRENT_DIR ON) +project(feature_annotations) + +find_package(Qt5 ${QT_MIN_VERSION} CONFIG REQUIRED COMPONENTS + Core + Gui +) + +find_package(dlib CONFIG REQUIRED) + +set(feature_annotations_LIBS + Qt5::Core + Qt5::Gui + dlib::dlib + pluginapi +) + +set(feature_annotations_SOURCES + featureannotations.cpp +) + +set(feature_annotations_HEADERS + featureannotations.h + feature_annotations_global.h +) + +set(CMAKE_AUTOMOC ON) +set(CMAKE_AUTOUIC ON) +set(CMAKE_AUTORCC ON) + +add_library(feature_annotations MODULE + ${feature_annotations_SOURCES} + ${feature_annotations_HEADERS} +) + +set_property(TARGET feature_annotations PROPERTY CXX_STANDARD 14) +set_property(TARGET feature_annotations PROPERTY OUTPUT_NAME "feature_annotations") +target_link_libraries(feature_annotations ${feature_annotations_LIBS}) + +add_definitions(-DFEATURE_ANNOTATIONS_LIBRARY) + +if(WIN32) + set(INSTALL_DESTINATION "aniplayer/uiplugins") +else() + set(INSTALL_DESTINATION "lib${LIB_SUFFIX}/aniplayer/uiplugins") +endif() + +install(TARGETS feature_annotations + LIBRARY DESTINATION ${INSTALL_DESTINATION} + ARCHIVE DESTINATION ${INSTALL_DESTINATION} +) +install(FILES $ + DESTINATION ${INSTALL_DESTINATION} OPTIONAL +) diff --git a/featureplugins/feature_annotations/feature_annotations.json b/featureplugins/feature_annotations/feature_annotations.json new file mode 100644 index 0000000..e69de29 diff --git a/featureplugins/feature_annotations/feature_annotations_global.h b/featureplugins/feature_annotations/feature_annotations_global.h new file mode 100644 index 0000000..338bb53 --- /dev/null +++ b/featureplugins/feature_annotations/feature_annotations_global.h @@ -0,0 +1,12 @@ +#ifndef FEATURE_ANNOTATIONS_GLOBAL_H +#define FEATURE_ANNOTATIONS_GLOBAL_H + +#include + +#if defined(FEATURE_ANNOTATIONS_LIBRARY) +# define FEATURE_ANNOTATIONSSHARED_EXPORT Q_DECL_EXPORT +#else +# define FEATURE_ANNOTATIONSSHARED_EXPORT Q_DECL_IMPORT +#endif + +#endif // FEATURE_ANNOTATIONS_GLOBAL_H diff --git a/featureplugins/feature_annotations/featureannotations.cpp b/featureplugins/feature_annotations/featureannotations.cpp new file mode 100644 index 0000000..1fb0aa4 --- /dev/null +++ b/featureplugins/feature_annotations/featureannotations.cpp @@ -0,0 +1,94 @@ +#include "featureannotations.h" + +#include +#include + +Q_LOGGING_CATEGORY(annotationsCategory, "Annotations") + +#include +#include + +// random windows.h defines +#undef interface + +using namespace std; +using namespace dlib; + + +template using con5d = con; +template using con5 = con; + +template using downsampler_r = relu>>>>>>>>; +template using rcon5_r = relu>>; + +using detection_net_type = loss_mmod>>>>>>>; + +namespace { +detection_net_type net; +} + +FeatureAnnoations::FeatureAnnoations(QObject *parent) : QObject{parent} { + deserialize("mmod_network.dat") >> net; +} + +FeaturePluginInstance * +FeatureAnnoations::createInstance(QObject *instance, + PlayerFeaturePlauginInterface *interface) { + return new FeatureAnnoationsInstance(instance, interface); +} + +FeatureAnnoationsInstance::FeatureAnnoationsInstance( + QObject *instance, PlayerFeaturePlauginInterface *player, QObject *) + : FeaturePluginInstance{instance, player} { + qCDebug(annotationsCategory) << "Registering with instance" << instance; + + connect(instance, SIGNAL(frameChanged(const QImage &)), + this, SLOT(onFrameChanged(const QImage &))); + + + m_fpsTimer.setInterval(1000); + connect(&m_fpsTimer, &QTimer::timeout, [&]() { + qCDebug(annotationsCategory) << "Did" << m_fpsCounter + << "frames per seoncd"; + m_fpsCounter = 0; + }); + m_fpsTimer.start(); +} + +void FeatureAnnoationsInstance::onFrameChanged(const QImage &image) +{ + qCDebug(annotationsCategory) << "Frame changed! " << image.size() << image.format(); + if (image.size().isEmpty()) return; + try { + matrix img{image.size().height(), image.size().width()}; + + const auto *imageData = reinterpret_cast(image.bits()); + auto size = img.size(); + + std::transform(imageData, imageData + size, img.begin(), + [](const QRgb &d) { + return rgb_pixel{static_cast(qRed(d)), + static_cast(qGreen(d)), + static_cast(qBlue(d))}; + }); + + //dlib::save_bmp(img, "testimage.bmp"); + + auto dets = net(img); + PlayerFeaturePlauginInterface::AnnotationList al; + for (auto&& d : dets) + al << PlayerFeaturePlauginInterface::Annotation{ + static_cast(d.rect.left()) / image.size().width(), + static_cast(d.rect.top()) / image.size().height(), + static_cast(d.rect.right() - d.rect.left()) / image.size().width(), + static_cast(d.rect.bottom() - d.rect.top()) / image.size().height(), + "red", + "" + }; + m_playerInterface->featureSetAnnotations(al); + ++m_fpsCounter; + + } catch(const std::exception &ex) { + qCDebug(annotationsCategory) << "Exception: " << ex.what(); + } +} diff --git a/featureplugins/feature_annotations/featureannotations.h b/featureplugins/feature_annotations/featureannotations.h new file mode 100644 index 0000000..06e6f33 --- /dev/null +++ b/featureplugins/feature_annotations/featureannotations.h @@ -0,0 +1,45 @@ +#ifndef FeatureAnnoations_H +#define FeatureAnnoations_H + +#include +#include +#include +#include + +#include + +#include "aniplayer/featurepluginbase.h" +#include "aniplayer/playerfeatureplugininterface.h" +#include "feature_annotations_global.h" + +class FEATURE_ANNOTATIONSSHARED_EXPORT FeatureAnnoations + : public QObject, + public FeaturePluginBase { + Q_OBJECT + Q_PLUGIN_METADATA(IID ANIPLAYER_FEATURE_PLUGIN_INTERFACE_IID FILE + "feature_annotations.json") + Q_INTERFACES(FeaturePluginBase) +public: + FeatureAnnoations(QObject *parent = nullptr); + + FeaturePluginInstance * + createInstance(QObject *instance, PlayerFeaturePlauginInterface *) override; +}; + +class FeatureAnnoationsInstance : public QObject, public FeaturePluginInstance { + Q_OBJECT +public: + FeatureAnnoationsInstance(QObject *instance, PlayerFeaturePlauginInterface *, QObject *parent = nullptr); + +private slots: + void onFrameChanged(const QImage &); + +private: + int m_fid; + QString m_path; + double m_duration; + QTimer m_fpsTimer; + int m_fpsCounter = 0; +}; + +#endif // FeatureAnnoations_H diff --git a/uiplugins/ui_desktop_qml_default/qml/main.qml b/uiplugins/ui_desktop_qml_default/qml/main.qml index d1bb4c6..c393808 100644 --- a/uiplugins/ui_desktop_qml_default/qml/main.qml +++ b/uiplugins/ui_desktop_qml_default/qml/main.qml @@ -48,6 +48,23 @@ Window { anchors.fill: parent } + Item { + id: annotationView + anchors.fill: parent + Repeater { + + model: player.annotationModel + Rectangle { + x: annotationX * annotationView.width + y: annotationY * annotationView.height + width: annotationW * annotationView.width + height: annotationH * annotationView.height + border.color: annotationColor + color: "transparent" + } + } + } + MouseArea { anchors.fill: parent acceptedButtons: Qt.AllButtons