From f33c03640abfbaaf503f742ee0e5f93b9be19cd9 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Mardelle Date: Wed, 23 Nov 2011 22:41:58 +0100 Subject: [PATCH] Rewrite audio thumbs threading --- src/clipmanager.cpp | 179 +++++++++++++++++++++++++++++++++----------- src/clipmanager.h | 7 +- src/docclipbase.cpp | 36 +-------- src/docclipbase.h | 5 +- src/kthumb.cpp | 152 +------------------------------------ src/kthumb.h | 15 +--- src/projectlist.cpp | 3 +- 7 files changed, 148 insertions(+), 249 deletions(-) diff --git a/src/clipmanager.cpp b/src/clipmanager.cpp index 2f223465..f4d22a5c 100644 --- a/src/clipmanager.cpp +++ b/src/clipmanager.cpp @@ -45,9 +45,9 @@ ClipManager::ClipManager(KdenliveDoc *doc) : QObject(), m_audioThumbsQueue(), m_doc(doc), - m_generatingAudioId(), m_abortThumb(false), - m_closing(false) + m_closing(false), + m_abortAudioThumb(false) { m_clipIdCounter = 1; m_folderIdCounter = 1; @@ -68,15 +68,14 @@ ClipManager::~ClipManager() { m_closing = true; m_abortThumb = true; + m_abortAudioThumb = true; m_thumbsThread.waitForFinished(); + m_audioThumbsThread.waitForFinished(); m_thumbsMutex.lock(); m_requestedThumbs.clear(); - m_thumbsMutex.unlock(); m_audioThumbsQueue.clear(); - m_generatingAudioId.clear(); - m_thumbsMutex.lock(); - m_requestedThumbs.clear(); m_thumbsMutex.unlock(); + qDeleteAll(m_clipList); m_clipList.clear(); #if KDE_IS_VERSION(4,5,0) @@ -87,13 +86,16 @@ ClipManager::~ClipManager() void ClipManager::clear() { m_abortThumb = true; + m_abortAudioThumb = true; m_thumbsThread.waitForFinished(); + m_audioThumbsThread.waitForFinished(); m_thumbsMutex.lock(); m_requestedThumbs.clear(); + m_audioThumbsQueue.clear(); m_thumbsMutex.unlock(); m_abortThumb = false; + m_abortAudioThumb = false; m_folderList.clear(); - m_audioThumbsQueue.clear(); m_modifiedClips.clear(); qDeleteAll(m_clipList); m_clipList.clear(); @@ -165,60 +167,149 @@ void ClipManager::slotGetThumbs() void ClipManager::checkAudioThumbs() { if (!KdenliveSettings::audiothumbnails()) { - if (!m_generatingAudioId.isEmpty()) { - DocClipBase *clip = getClipById(m_generatingAudioId); - if (clip) clip->slotClearAudioCache(); + if (m_audioThumbsThread.isRunning()) { + m_abortAudioThumb = true; + m_thumbsMutex.lock(); + m_audioThumbsQueue.clear(); + m_thumbsMutex.unlock(); + m_audioThumbsThread.waitForFinished(); + m_abortAudioThumb = false; } - m_audioThumbsQueue.clear(); - m_generatingAudioId.clear(); return; } + m_thumbsMutex.lock(); for (int i = 0; i < m_clipList.count(); i++) { - m_audioThumbsQueue.append(m_clipList.at(i)->getId()); + DocClipBase *clip = m_clipList.at(i); + if (clip->clipType() & (AUDIO | AV | PLAYLIST) && !clip->audioThumbCreated()) + m_audioThumbsQueue.append(m_clipList.at(i)->getId()); } - if (m_generatingAudioId.isEmpty()) startAudioThumbsGeneration(); + m_thumbsMutex.unlock(); + if (!m_audioThumbsThread.isRunning()) m_audioThumbsThread = QtConcurrent::run(this, &ClipManager::slotGetAudioThumbs); } void ClipManager::askForAudioThumb(const QString &id) { DocClipBase *clip = getClipById(id); - if (clip && KdenliveSettings::audiothumbnails()) { - m_audioThumbsQueue.append(id); - if (m_generatingAudioId.isEmpty()) startAudioThumbsGeneration(); + if (clip && KdenliveSettings::audiothumbnails() && (clip->clipType() & (AUDIO | AV | PLAYLIST))) { + m_thumbsMutex.lock(); + if (!m_audioThumbsQueue.contains(id)) m_audioThumbsQueue.append(id); + m_thumbsMutex.unlock(); + if (!m_audioThumbsThread.isRunning()) m_audioThumbsThread = QtConcurrent::run(this, &ClipManager::slotGetAudioThumbs); } } -void ClipManager::startAudioThumbsGeneration() +void ClipManager::slotGetAudioThumbs() { - if (!KdenliveSettings::audiothumbnails()) { - m_audioThumbsQueue.clear(); - m_generatingAudioId.clear(); - return; - } - if (!m_audioThumbsQueue.isEmpty()) { - m_generatingAudioId = m_audioThumbsQueue.takeFirst(); - DocClipBase *clip = getClipById(m_generatingAudioId); - if (!clip || !clip->slotGetAudioThumbs()) - endAudioThumbsGeneration(m_generatingAudioId); - } else { - m_generatingAudioId.clear(); - } -} + Mlt::Profile prof((char*) KdenliveSettings::current_profile().toUtf8().constData()); + mlt_audio_format audioFormat = mlt_audio_pcm; + while (!m_abortAudioThumb && !m_audioThumbsQueue.isEmpty()) { + m_thumbsMutex.lock(); + QString clipId = m_audioThumbsQueue.takeFirst(); + m_thumbsMutex.unlock(); + DocClipBase *clip = getClipById(clipId); + if (!clip || clip->audioThumbCreated()) continue; + KUrl url = clip->fileURL(); + QString hash = clip->getClipHash(); + if (hash.isEmpty()) continue; + QString audioPath = projectFolder() + "/thumbs/" + hash + ".thumb"; + double lengthInFrames = clip->duration().frames(m_doc->fps()); + //FIXME: should this be hardcoded?? + int channels = 2; + int frequency = 48000; + int arrayWidth = 20; + double frame = 0.0; + audioByteArray storeIn; + QFile f(audioPath); + if (QFileInfo(audioPath).size() > 0 && f.open(QIODevice::ReadOnly)) { + const QByteArray channelarray = f.readAll(); + f.close(); + if (channelarray.size() != arrayWidth*(frame + lengthInFrames) * channels) { + kDebug() << "--- BROKEN THUMB FOR: " << url.fileName() << " ---------------------- "; + f.remove(); + continue; + } + kDebug() << "reading audio thumbs from file"; + + int h1 = arrayWidth * channels; + int h2 = (int) frame * h1; + int h3; + for (int z = (int) frame; z < (int)(frame + lengthInFrames) && !m_abortAudioThumb; z++) { + h3 = 0; + for (int c = 0; c < channels; c++) { + QByteArray audioArray(arrayWidth, '\x00'); + for (int i = 0; i < arrayWidth; i++) { + audioArray[i] = channelarray.at(h2 + h3 + i); + } + h3 += arrayWidth; + storeIn[z][c] = audioArray; + } + h2 += h1; + } + if (!m_abortAudioThumb) clip->updateAudioThumbnail(storeIn); + continue; + } + + if (!f.open(QIODevice::WriteOnly)) { + kDebug() << "++++++++ ERROR WRITING TO FILE: " << audioPath; + kDebug() << "++++++++ DISABLING AUDIO THUMBS"; + m_thumbsMutex.lock(); + m_audioThumbsQueue.clear(); + m_thumbsMutex.unlock(); + KdenliveSettings::setAudiothumbnails(false); + break; + } -void ClipManager::endAudioThumbsGeneration(const QString &requestedId) -{ - if (!KdenliveSettings::audiothumbnails()) { - m_audioThumbsQueue.clear(); - m_generatingAudioId.clear(); - return; - } - if (!m_audioThumbsQueue.isEmpty()) { - if (m_generatingAudioId == requestedId) { - startAudioThumbsGeneration(); + Mlt::Producer producer(prof, url.path().toUtf8().constData()); + if (!producer.is_valid()) { + kDebug() << "++++++++ INVALID CLIP: " << url.path(); + continue; + } + + if (KdenliveSettings::normaliseaudiothumbs()) { + Mlt::Filter m_convert(prof, "volume"); + m_convert.set("gain", "normalise"); + producer.attach(m_convert); + } + + int last_val = 0; + int val = 0; + double framesPerSecond = mlt_producer_get_fps(producer.get_producer()); + Mlt::Frame *mlt_frame; + + for (int z = (int) frame; z < (int)(frame + lengthInFrames) && producer.is_valid() && !m_abortAudioThumb; z++) { + val = (int)((z - frame) / (frame + lengthInFrames) * 100.0); + if (last_val != val && val > 1) { + setThumbsProgress(i18n("Creating thumbnail for %1", url.fileName()), val); + last_val = val; + } + producer.seek(z); + mlt_frame = producer.get_frame(); + if (mlt_frame && mlt_frame->is_valid()) { + int samples = mlt_sample_calculator(framesPerSecond, frequency, mlt_frame_get_position(mlt_frame->get_frame())); + qint16* pcm = static_cast(mlt_frame->get_audio(audioFormat, frequency, channels, samples)); + for (int c = 0; c < channels; c++) { + QByteArray audioArray; + audioArray.resize(arrayWidth); + for (int i = 0; i < audioArray.size(); i++) { + audioArray[i] = ((*(pcm + c + i * samples / audioArray.size())) >> 9) + 127 / 2 ; + } + f.write(audioArray); + storeIn[z][c] = audioArray; + } + } else { + f.write(QByteArray(arrayWidth, '\x00')); + } + delete mlt_frame; + } + f.close(); + //TODO: post 8.2.1, change text to AUDIO thumbnails + setThumbsProgress(i18n("Creating thumbnail for %1", url.fileName()), -1); + if (m_abortAudioThumb) { + f.remove(); + } else { + clip->updateAudioThumbnail(storeIn); } - } else { - m_generatingAudioId.clear(); } } diff --git a/src/clipmanager.h b/src/clipmanager.h index 7f1622e3..008560f5 100644 --- a/src/clipmanager.h +++ b/src/clipmanager.h @@ -98,8 +98,6 @@ Q_OBJECT public: int getFreeClipId(); int getFreeFolderId(); int lastClipId() const; - void startAudioThumbsGeneration(); - void endAudioThumbsGeneration(const QString &requestedId); void askForAudioThumb(const QString &id); QString projectFolder() const; void clearUnusedProducers(); @@ -129,6 +127,7 @@ private slots: /** Check the list of externally modified clips, and process them if they were not modified in the last 1500 milliseconds */ void slotProcessModifiedClips(); void slotGetThumbs(); + void slotGetAudioThumbs(); private: // Private attributes /** the list of clips in the document */ @@ -141,7 +140,6 @@ private: // Private attributes KdenliveDoc *m_doc; int m_clipIdCounter; int m_folderIdCounter; - QString m_generatingAudioId; KDirWatch m_fileWatcher; /** Timer used to reload clips when they have been externally modified */ QTimer m_modifiedTimer; @@ -155,6 +153,9 @@ private: // Private attributes bool m_abortThumb; /** @brief We are about to delete the clip producer, stop processing thumbs. */ bool m_closing; + QFuture m_audioThumbsThread; + /** @brief If true, abort processing of audio thumbs. */ + bool m_abortAudioThumb; signals: void reloadClip(const QString &); diff --git a/src/docclipbase.cpp b/src/docclipbase.cpp index 7b97cedb..2b36b8bb 100644 --- a/src/docclipbase.cpp +++ b/src/docclipbase.cpp @@ -48,7 +48,6 @@ DocClipBase::DocClipBase(ClipManager *clipManager, QDomElement xml, const QStrin m_videoOnlyProducer(NULL), m_snapMarkers(QList < CommentedTime >()), m_duration(), - m_audioTimer(NULL), m_thumbProd(NULL), m_audioThumbCreated(false), m_id(id), @@ -88,16 +87,12 @@ DocClipBase::DocClipBase(ClipManager *clipManager, QDomElement xml, const QStrin if (!m_properties.contains("name")) m_properties.insert("name", url.fileName()); m_thumbProd = new KThumb(clipManager, url, m_id, m_properties.value("file_hash")); - if (m_clipType == AV || m_clipType == AUDIO || m_clipType == PLAYLIST) slotCreateAudioTimer(); + if (m_clipType & (AV | AUDIO | PLAYLIST)) getAudioThumbs(); } DocClipBase::~DocClipBase() { delete m_thumbProd; - if (m_audioTimer) { - m_audioTimer->stop(); - delete m_audioTimer; - } qDeleteAll(m_toDeleteProducers); m_toDeleteProducers.clear(); qDeleteAll(m_baseTrackProducers); @@ -120,22 +115,9 @@ QPoint DocClipBase::zone() const return zone; } -void DocClipBase::slotCreateAudioTimer() -{ - connect(m_thumbProd, SIGNAL(audioThumbReady(const audioByteArray&)), this , SLOT(updateAudioThumbnail(const audioByteArray&))); - m_audioTimer = new QTimer(this); - connect(m_audioTimer, SIGNAL(timeout()), this, SLOT(slotGetAudioThumbs())); -} - -void DocClipBase::askForAudioThumbs() -{ - if (m_thumbProd && m_audioTimer) m_thumbProd->askForAudioThumbs(getId()); -} void DocClipBase::slotClearAudioCache() { - if (m_thumbProd) m_thumbProd->stopAudioThumbs(); - if (m_audioTimer != NULL) m_audioTimer->stop(); m_audioFrameCache.clear(); m_audioThumbCreated = false; } @@ -173,10 +155,7 @@ const CLIPTYPE & DocClipBase::clipType() const void DocClipBase::setClipType(CLIPTYPE type) { m_clipType = type; - m_properties.insert("type", QString::number((int) type)); - if (m_thumbProd && m_audioTimer == NULL && (m_clipType == AV || m_clipType == AUDIO || m_clipType == PLAYLIST)) - slotCreateAudioTimer(); } KUrl DocClipBase::fileURL() const @@ -1107,20 +1086,13 @@ QMap DocClipBase::properties() const return m_properties; } -bool DocClipBase::slotGetAudioThumbs() +bool DocClipBase::getAudioThumbs() { - if (m_thumbProd == NULL || isPlaceHolder()) return false; - if (!KdenliveSettings::audiothumbnails() || m_audioTimer == NULL) { - if (m_audioTimer != NULL) m_audioTimer->stop(); - return false; - } + if (m_thumbProd == NULL || isPlaceHolder() || !KdenliveSettings::audiothumbnails()) return false; if (m_audioThumbCreated) { - m_audioTimer->stop(); return false; } - m_audioTimer->start(1500); - double lengthInFrames = duration().frames(KdenliveSettings::project_fps()); - m_thumbProd->getAudioThumbs(2, 0, lengthInFrames /*must be number of frames*/, 20); + QTimer::singleShot(800, m_thumbProd, SLOT(slotCreateAudioThumbs())); return true; } diff --git a/src/docclipbase.h b/src/docclipbase.h index f84f8d84..f4f31ce4 100644 --- a/src/docclipbase.h +++ b/src/docclipbase.h @@ -176,7 +176,6 @@ Q_OBJECT public: /** Free cache data */ void slotClearAudioCache(); - void askForAudioThumbs(); QString getClipHash() const; void refreshThumbUrl(); const char *producerProperty(const char *name) const; @@ -209,6 +208,7 @@ Q_OBJECT public: void reloadThumbProducer(); void cleanupProducers(); bool isClean() const; + bool getAudioThumbs(); private: // Private attributes @@ -228,7 +228,6 @@ private: // Private attributes QPixmap m_thumbnail; GenTime m_duration; - QTimer *m_audioTimer; KThumb *m_thumbProd; bool m_audioThumbCreated; @@ -251,7 +250,6 @@ private: // Private attributes QMutex m_replaceMutex; /** Create connections for audio thumbnails */ - void slotCreateAudioTimer(); void slotRefreshProducer(); void setProducerProperty(const char *name, int data); void setProducerProperty(const char *name, double data); @@ -264,7 +262,6 @@ private: // Private attributes public slots: void updateAudioThumbnail(const audioByteArray& data); - bool slotGetAudioThumbs(); QList < CommentedTime > commentedSnapMarkers() const; GenTime findNextSnapMarker(const GenTime & currTime); GenTime findPreviousSnapMarker(const GenTime & currTime); diff --git a/src/kthumb.cpp b/src/kthumb.cpp index c6189ae7..c7dcfebe 100644 --- a/src/kthumb.cpp +++ b/src/kthumb.cpp @@ -42,15 +42,13 @@ KThumb::KThumb(ClipManager *clipManager, KUrl url, const QString &id, const QString &hash, QObject * parent, const char */*name*/) : QObject(parent), - m_audioThumbProducer(), m_url(url), m_thumbFile(), m_dar(1), m_ratio(1), m_producer(NULL), m_clipManager(clipManager), - m_id(id), - m_stopAudioThumbs(false) + m_id(id) { m_thumbFile = clipManager->projectFolder() + "/thumbs/" + hash + ".thumb"; } @@ -59,11 +57,6 @@ KThumb::~KThumb() { if (m_producer) m_clipManager->stopThumbs(m_id); m_intraFramesQueue.clear(); - if (m_audioThumbProducer.isRunning()) { - m_stopAudioThumbs = true; - m_audioThumbProducer.waitForFinished(); - slotAudioThumbOver(); - } m_intra.waitForFinished(); } @@ -339,151 +332,10 @@ void KThumb::getThumbs(KUrl url, int startframe, int endframe, int width, int he emit thumbReady(endframe, image); } */ -void KThumb::stopAudioThumbs() -{ - if (m_audioThumbProducer.isRunning()) { - m_stopAudioThumbs = true; - m_audioThumbProducer.waitForFinished(); - slotAudioThumbOver(); - } -} - -void KThumb::removeAudioThumb() -{ - if (m_thumbFile.isEmpty()) return; - stopAudioThumbs(); - QFile f(m_thumbFile); - f.remove(); -} - -void KThumb::getAudioThumbs(int channel, double frame, double frameLength, int arrayWidth) -{ - if (channel == 0) { - slotAudioThumbOver(); - return; - } - if (m_audioThumbProducer.isRunning()) { - return; - } - - audioByteArray storeIn; - //FIXME: Hardcoded!!! - m_frequency = 48000; - m_channels = channel; - - QFile f(m_thumbFile); - if (f.open(QIODevice::ReadOnly)) { - const QByteArray channelarray = f.readAll(); - f.close(); - if (channelarray.size() != arrayWidth*(frame + frameLength)*m_channels) { - kDebug() << "--- BROKEN THUMB FOR: " << m_url.fileName() << " ---------------------- " << endl; - f.remove(); - slotAudioThumbOver(); - return; - } - - kDebug() << "reading audio thumbs from file"; - - int h1 = arrayWidth * m_channels; - int h2 = (int) frame * h1; - int h3; - for (int z = (int) frame; z < (int)(frame + frameLength); z++) { - h3 = 0; - for (int c = 0; c < m_channels; c++) { - QByteArray m_array(arrayWidth, '\x00'); - for (int i = 0; i < arrayWidth; i++) { - m_array[i] = channelarray.at(h2 + h3 + i); - } - h3 += arrayWidth; - storeIn[z][c] = m_array; - } - h2 += h1; - } - emit audioThumbReady(storeIn); - slotAudioThumbOver(); - } else { - if (m_audioThumbProducer.isRunning()) return; - m_audioThumbFile.setFileName(m_thumbFile); - m_frame = frame; - m_frameLength = frameLength; - m_arrayWidth = arrayWidth; - m_audioThumbProducer = QtConcurrent::run(this, &KThumb::slotCreateAudioThumbs); - /*m_audioThumbProducer.init(m_url, m_thumbFile, frame, frameLength, m_frequency, m_channels, arrayWidth); - m_audioThumbProducer.start(QThread::LowestPriority);*/ - // kDebug() << "STARTING GENERATE THMB FOR: " < 1) { - m_clipManager->setThumbsProgress(i18n("Creating thumbnail for %1", m_url.fileName()), val); - last_val = val; - } - producer.seek(z); - mlt_frame = producer.get_frame(); - if (mlt_frame && mlt_frame->is_valid()) { - int m_samples = mlt_sample_calculator(framesPerSecond, m_frequency, mlt_frame_get_position(mlt_frame->get_frame())); - qint16* m_pcm = static_cast(mlt_frame->get_audio(m_audioFormat, m_frequency, m_channels, m_samples)); - for (int c = 0; c < m_channels; c++) { - QByteArray m_array; - m_array.resize(m_arrayWidth); - for (int i = 0; i < m_array.size(); i++) { - m_array[i] = ((*(m_pcm + c + i * m_samples / m_array.size())) >> 9) + 127 / 2 ; - } - m_audioThumbFile.write(m_array); - - } - } else { - m_audioThumbFile.write(QByteArray(m_arrayWidth, '\x00')); - } - delete mlt_frame; - } - m_audioThumbFile.close(); - if (m_stopAudioThumbs) { - m_audioThumbFile.remove(); - } else { - slotAudioThumbOver(); - } -} - -void KThumb::slotAudioThumbOver() -{ - m_clipManager->setThumbsProgress(i18n("Creating thumbnail for %1", m_url.fileName()), -1); - m_clipManager->endAudioThumbsGeneration(m_id); -} - -void KThumb::askForAudioThumbs(const QString &id) -{ - m_clipManager->askForAudioThumb(id); + m_clipManager->askForAudioThumb(m_id); } #if KDE_IS_VERSION(4,5,0) diff --git a/src/kthumb.h b/src/kthumb.h index b8bc3d0d..415f3551 100644 --- a/src/kthumb.h +++ b/src/kthumb.h @@ -59,7 +59,6 @@ Q_OBJECT public: KThumb(ClipManager *clipManager, KUrl url, const QString &id, const QString &hash, QObject * parent = 0, const char *name = 0); ~KThumb(); void setProducer(Mlt::Producer *producer); - void askForAudioThumbs(const QString &id); bool hasProducer() const; void clearProducer(); void updateThumbUrl(const QString &hash); @@ -79,9 +78,7 @@ public slots: // static QPixmap getImage(QDomElement xml, int frame, int width, int height); /* void getImage(KUrl url, int frame, int width, int height); void getThumbs(KUrl url, int startframe, int endframe, int width, int height);*/ - void stopAudioThumbs(); - void removeAudioThumb(); - void getAudioThumbs(int channel, double frame, double frameLength, int arrayWidth); + void slotCreateAudioThumbs(); static QPixmap getImage(KUrl url, int frame, int width, int height); static QImage getFrame(Mlt::Producer *producer, int framepos, int frameWidth, int displayWidth, int height); static QImage getFrame(Mlt::Frame *frame, int frameWidth, int displayWidth, int height); @@ -91,15 +88,12 @@ public slots: static uint imageVariance(QImage image); private slots: - void slotAudioThumbOver(); - void slotCreateAudioThumbs(); #if KDE_IS_VERSION(4,5,0) /** @brief Fetch all requested frames. */ void slotGetIntraThumbs(); #endif private: - QFuture m_audioThumbProducer; KUrl m_url; QString m_thumbFile; double m_dar; @@ -109,13 +103,6 @@ private: QString m_id; /** @brief Controls the intra frames thumbnails process (cached thumbnails). */ QFuture m_intra; - QFile m_audioThumbFile; - bool m_stopAudioThumbs; - double m_frame; - double m_frameLength; - int m_frequency; - int m_channels; - int m_arrayWidth; /** @brief List of frame numbers from which we want to extract thumbnails. */ QList m_intraFramesQueue; QMutex m_mutex; diff --git a/src/projectlist.cpp b/src/projectlist.cpp index 2705a62a..5ce8e42d 100644 --- a/src/projectlist.cpp +++ b/src/projectlist.cpp @@ -1131,7 +1131,6 @@ void ProjectList::slotAddClip(DocClipBase *clip, bool getProperties) else if (item->hasProxy() && !item->isProxyRunning()) { slotCreateProxy(clip->getId()); } - clip->askForAudioThumbs(); KUrl url = clip->fileURL(); #ifdef NEPOMUK @@ -1851,7 +1850,7 @@ void ProjectList::slotReplyGetFileProperties(const QString &clipId, Mlt::Produce } item->setProperties(properties, metadata); clip->setProducer(producer, replace); - clip->askForAudioThumbs(); + clip->getAudioThumbs(); // Proxy stuff QString size = properties.value("frame_size"); -- 2.39.2