From a7be14d538d33a1f4ecb88df578af1fdbf042382 Mon Sep 17 00:00:00 2001 From: "Simon A. Eugster" Date: Tue, 14 Feb 2012 23:01:57 +0100 Subject: [PATCH] Auto-align works! 1. Add a clip to the timeline and set it as reference in the context menu 2. Add a new clip to the timeline and select auto-align in the context menu Still needs some refinements when dealing with collisions (undo etc.). --- src/CMakeLists.txt | 1 + src/customtrackview.cpp | 172 +++++++++++++++++++++++------ src/customtrackview.h | 10 ++ src/lib/CMakeLists.txt | 10 ++ src/lib/audio/CMakeLists.txt | 10 ++ src/lib/audio/audioCorrelation.cpp | 1 + src/lib/audio/audioEnvelope.cpp | 7 ++ src/mainwindow.cpp | 26 ++++- src/mainwindow.h | 2 + 9 files changed, 206 insertions(+), 33 deletions(-) create mode 100644 src/lib/CMakeLists.txt create mode 100644 src/lib/audio/CMakeLists.txt diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6b3197b8..fa0e56c6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -83,6 +83,7 @@ add_subdirectory(beziercurve) add_subdirectory(colorcorrection) add_subdirectory(colorscopes) add_subdirectory(commands) +add_subdirectory(lib) add_subdirectory(projecttree) add_subdirectory(utils) add_subdirectory(databackup) diff --git a/src/customtrackview.cpp b/src/customtrackview.cpp index cc75b20f..7f30a686 100644 --- a/src/customtrackview.cpp +++ b/src/customtrackview.cpp @@ -62,6 +62,9 @@ #include "commands/razorgroupcommand.h" #include "profilesdialog.h" +#include "lib/audio/audioEnvelope.h" +#include "lib/audio/audioCorrelation.h" + #include #include #include @@ -98,38 +101,41 @@ bool sortGuidesList(const Guide *g1 , const Guide *g2) // const int duration = animate ? 1500 : 1; CustomTrackView::CustomTrackView(KdenliveDoc *doc, CustomTrackScene* projectscene, QWidget *parent) : - QGraphicsView(projectscene, parent), - m_tracksHeight(KdenliveSettings::trackheight()), - m_projectDuration(0), - m_cursorPos(0), - m_document(doc), - m_scene(projectscene), - m_cursorLine(NULL), - m_operationMode(NONE), - m_moveOpMode(NONE), - m_dragItem(NULL), - m_dragGuide(NULL), - m_visualTip(NULL), - m_animation(NULL), - m_clickPoint(), - m_autoScroll(KdenliveSettings::autoscroll()), - m_pasteEffectsAction(NULL), - m_ungroupAction(NULL), - m_scrollOffset(0), - m_clipDrag(false), - m_findIndex(0), - m_tool(SELECTTOOL), - m_copiedItems(), - m_menuPosition(), - m_blockRefresh(false), - m_selectionGroup(NULL), - m_selectedTrack(0), - m_controlModifier(false) -{ - if (doc) + QGraphicsView(projectscene, parent), + m_tracksHeight(KdenliveSettings::trackheight()), + m_projectDuration(0), + m_cursorPos(0), + m_document(doc), + m_scene(projectscene), + m_cursorLine(NULL), + m_operationMode(NONE), + m_moveOpMode(NONE), + m_dragItem(NULL), + m_dragGuide(NULL), + m_visualTip(NULL), + m_animation(NULL), + m_clickPoint(), + m_autoScroll(KdenliveSettings::autoscroll()), + m_pasteEffectsAction(NULL), + m_ungroupAction(NULL), + m_scrollOffset(0), + m_clipDrag(false), + m_findIndex(0), + m_tool(SELECTTOOL), + m_copiedItems(), + m_menuPosition(), + m_blockRefresh(false), + m_selectionGroup(NULL), + m_selectedTrack(0), + m_audioCorrelator(NULL), + m_audioAlignmentReference(NULL), + m_controlModifier(false) +{ + if (doc) { m_commandStack = doc->commandStack(); - else + } else { m_commandStack = NULL; + } m_ct = 0; setMouseTracking(true); setAcceptDrops(true); @@ -2475,6 +2481,10 @@ void CustomTrackView::dropEvent(QDropEvent * event) } event->setDropAction(Qt::MoveAction); event->accept(); + + /// \todo enable when really working +// alignAudio(); + } else QGraphicsView::dropEvent(event); setFocus(); } @@ -5943,6 +5953,97 @@ void CustomTrackView::splitAudio() } } +void CustomTrackView::setAudioAlignReference() +{ + QList selection = scene()->selectedItems(); + if (selection.isEmpty() || selection.size() > 1) { + emit displayMessage(i18n("You must select exactly one clip for the audio reference."), ErrorMessage); + return; + } + if (m_audioCorrelator != NULL) { + delete m_audioCorrelator; + } + if (selection.at(0)->type() == AVWIDGET) { + ClipItem *clip = static_cast(selection.at(0)); + if (clip->clipType() == AV || clip->clipType() == AUDIO) { + m_audioAlignmentReference = clip; + + AudioEnvelope *envelope = new AudioEnvelope(clip->getProducer(clip->track())); + m_audioCorrelator = new AudioCorrelation(envelope); + + envelope->drawEnvelope().save("kdenlive-audio-reference-envelope.png"); + envelope->dumpInfo(); + + emit displayMessage(i18n("Audio align reference set."), InformationMessage); + } + return; + } + emit displayMessage(i18n("Reference for audio alignment must contain audio data."), ErrorMessage); +} + +void CustomTrackView::alignAudio() +{ + bool referenceOK = true; + if (m_audioCorrelator == NULL) { + referenceOK = false; + } + if (referenceOK) { + if (!scene()->items().contains(m_audioAlignmentReference)) { + // The reference item has been deleted from the timeline (or so) + referenceOK = false; + } + } + if (!referenceOK) { + emit displayMessage(i18n("Audio alignment reference not yet set."), DefaultMessage); + return; + } + + int counter = 0; + QList selection = scene()->selectedItems(); + foreach (QGraphicsItem *item, selection) { + if (item->type() == AVWIDGET) { + ClipItem *clip = static_cast(item); + if (clip->clipType() == AV || clip->clipType() == AUDIO) { + AudioEnvelope *envelope = new AudioEnvelope(clip->getProducer(clip->track())); + int index = m_audioCorrelator->addChild(envelope); + int shift = m_audioCorrelator->getShift(index); + counter++; + + m_audioCorrelator->info(index)->toImage().save("kdenlive-audio-align-cross-correlation.png"); + envelope->drawEnvelope().save("kdenlive-audio-align-envelope.png"); + envelope->dumpInfo(); + + int targetPos = m_audioAlignmentReference->startPos().frames(m_document->fps()) + shift; + qDebug() << "Reference starts at " << m_audioAlignmentReference->startPos().frames(m_document->fps()); + qDebug() << "We will start at " << targetPos; + qDebug() << "to shift by " << shift; + qDebug() << "(eventually)"; + qDebug() << "(maybe)"; + + GenTime add(shift, m_document->fps()); + ItemInfo start = clip->info(); + ItemInfo end = clip->info(); + end.startPos = m_audioAlignmentReference->info().startPos + add; + end.endPos = m_audioAlignmentReference->info().startPos + add + clip->info().cropDuration; + + QUndoCommand *moveCommand = new QUndoCommand(); + moveCommand->setText(i18n("Auto-align clip")); + new MoveClipCommand(this, start, end, true, moveCommand); + moveClip(start, end, true); + updateTrackDuration(clip->track(), moveCommand); + m_commandStack->push(moveCommand); + + } + } + } + + if (counter == 0) { + emit displayMessage(i18n("No audio clips selected."), ErrorMessage); + } else { + emit displayMessage(i18n("Auto-aligned %1 clips.").arg(counter), InformationMessage); + } +} + void CustomTrackView::doSplitAudio(const GenTime &pos, int track, EffectsList effects, bool split) { ClipItem *clip = getClipItemAt(pos, track); @@ -6026,8 +6127,15 @@ void CustomTrackView::doSplitAudio(const GenTime &pos, int track, EffectsList ef deleteClip(clp->info()); clip->setVideoOnly(false); Mlt::Tractor *tractor = m_document->renderer()->lockService(); - if (!m_document->renderer()->mltUpdateClipProducer(tractor, m_document->tracksCount() - info.track, info.startPos.frames(m_document->fps()), clip->baseClip()->getProducer(info.track))) { - emit displayMessage(i18n("Cannot update clip (time: %1, track: %2)", info.startPos.frames(m_document->fps()), info.track), ErrorMessage); + if (!m_document->renderer()->mltUpdateClipProducer( + tractor, + m_document->tracksCount() - info.track, + info.startPos.frames(m_document->fps()), + clip->baseClip()->getProducer(info.track))) { + + emit displayMessage(i18n("Cannot update clip (time: %1, track: %2)", + info.startPos.frames(m_document->fps()), info.track), + ErrorMessage); } m_document->renderer()->unlockService(tractor); diff --git a/src/customtrackview.h b/src/customtrackview.h index c5c62f9a..5d932b5d 100644 --- a/src/customtrackview.h +++ b/src/customtrackview.h @@ -42,6 +42,7 @@ class ClipItem; class AbstractClipItem; class AbstractGroupItem; class Transition; +class AudioCorrelation; class CustomTrackView : public QGraphicsView { @@ -138,6 +139,12 @@ public: /** @brief Creates SplitAudioCommands for selected clips. */ void splitAudio(); + /// Define which clip to take as reference for automatic audio alignment + void setAudioAlignReference(); + + /// Automatically align the currently selected clips to synchronize their audio with the reference's audio + void alignAudio(); + /** @brief Seperates the audio of a clip to a audio track. * @param pos Position of the clip to split * @param track Track of the clip @@ -356,6 +363,9 @@ private: QWaitCondition m_producerNotReady; KStatefulBrush m_activeTrackBrush; + AudioCorrelation *m_audioCorrelator; + ClipItem *m_audioAlignmentReference; + /** stores the state of the control modifier during mouse press. * Will then be used to identify whether we resize a group or only one item of it. */ bool m_controlModifier; diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt new file mode 100644 index 00000000..e3a89b07 --- /dev/null +++ b/src/lib/CMakeLists.txt @@ -0,0 +1,10 @@ + +add_subdirectory(audio) + +# Hack. kdenlive_SRCS is defined in the sub-scope and added to this scope; +# now we again need to add it to the parent scope. + +set(kdenlive_SRCS + ${kdenlive_SRCS} + PARENT_SCOPE +) diff --git a/src/lib/audio/CMakeLists.txt b/src/lib/audio/CMakeLists.txt new file mode 100644 index 00000000..bf5cd2b3 --- /dev/null +++ b/src/lib/audio/CMakeLists.txt @@ -0,0 +1,10 @@ + +set(kdenlive_SRCS + ${kdenlive_SRCS} + lib/audio/audioCorrelation.cpp + lib/audio/audioCorrelationInfo.cpp + lib/audio/audioEnvelope.cpp + lib/audio/audioInfo.cpp + lib/audio/audioStreamInfo.cpp + PARENT_SCOPE +) diff --git a/src/lib/audio/audioCorrelation.cpp b/src/lib/audio/audioCorrelation.cpp index 0a308c1c..71983442 100644 --- a/src/lib/audio/audioCorrelation.cpp +++ b/src/lib/audio/audioCorrelation.cpp @@ -26,6 +26,7 @@ AudioCorrelation::~AudioCorrelation() foreach (AudioEnvelope *envelope, m_children) { delete envelope; } + std::cout << "Envelope deleted." << std::endl; } int AudioCorrelation::addChild(AudioEnvelope *envelope) diff --git a/src/lib/audio/audioEnvelope.cpp b/src/lib/audio/audioEnvelope.cpp index 3968f622..7849372d 100644 --- a/src/lib/audio/audioEnvelope.cpp +++ b/src/lib/audio/audioEnvelope.cpp @@ -71,6 +71,8 @@ void AudioEnvelope::loadEnvelope() QTime t; t.start(); + m_producer->seek(0); + m_producer->set_speed(1.0); // This is necessary, otherwise we don't get any new frames in the 2nd run. for (int i = 0; i < m_envelopeSize; i++) { frame = m_producer->get_frame(i); @@ -89,6 +91,11 @@ void AudioEnvelope::loadEnvelope() if (sum > m_envelopeMax) { m_envelopeMax = sum; } + +// std::cout << position << "|" << m_producer->get_playtime() +// << "-" << m_producer->get_in() << "+" << m_producer->get_out() << " "; + + delete frame; } m_envelopeMean /= m_envelopeSize; std::cout << "Calculating the envelope (" << m_envelopeSize << " frames) took " diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp index bfeab9c5..6ed7c1ec 100644 --- a/src/mainwindow.cpp +++ b/src/mainwindow.cpp @@ -587,6 +587,8 @@ MainWindow::MainWindow(const QString &MltPath, const KUrl & Url, const QString & m_timelineContextClipMenu->addAction(actionCollection()->action("group_clip")); m_timelineContextClipMenu->addAction(actionCollection()->action("ungroup_clip")); m_timelineContextClipMenu->addAction(actionCollection()->action("split_audio")); + m_timelineContextClipMenu->addAction(actionCollection()->action("set_audio_align_ref")); + m_timelineContextClipMenu->addAction(actionCollection()->action("align_audio")); m_timelineContextClipMenu->addSeparator(); m_timelineContextClipMenu->addAction(actionCollection()->action("cut_timeline_clip")); m_timelineContextClipMenu->addAction(actionCollection()->action(KStandardAction::name(KStandardAction::Copy))); @@ -1505,7 +1507,7 @@ void MainWindow::setupActions() collection.addAction("edit_clip_marker", editClipMarker); connect(editClipMarker, SIGNAL(triggered(bool)), this, SLOT(slotEditClipMarker())); - KAction *addMarkerGuideQuickly = new KAction(KIcon("bookmark-new"), i18n("Add Marker/Guide quickly"), this); + KAction* addMarkerGuideQuickly = new KAction(KIcon("bookmark-new"), i18n("Add Marker/Guide quickly"), this); addMarkerGuideQuickly->setShortcut(Qt::Key_Asterisk); collection.addAction("add_marker_guide_quickly", addMarkerGuideQuickly); connect(addMarkerGuideQuickly, SIGNAL(triggered(bool)), this, SLOT(slotAddMarkerGuideQuickly())); @@ -1514,6 +1516,14 @@ void MainWindow::setupActions() collection.addAction("split_audio", splitAudio); connect(splitAudio, SIGNAL(triggered(bool)), this, SLOT(slotSplitAudio())); + KAction* setAudioAlignReference = new KAction(i18n("Set reference for audio alignment"), this); + collection.addAction("set_audio_align_ref", setAudioAlignReference); + connect(setAudioAlignReference, SIGNAL(triggered()), this, SLOT(slotSetAudioAlignReference())); + + KAction* alignAudio = new KAction(i18n("Align audio to reference"), this); + collection.addAction("align_audio", alignAudio); + connect(alignAudio, SIGNAL(triggered()), this, SLOT(slotAlignAudio())); + KAction* audioOnly = new KAction(KIcon("document-new"), i18n("Audio Only"), this); collection.addAction("clip_audio_only", audioOnly); audioOnly->setData("clip_audio_only"); @@ -3797,6 +3807,20 @@ void MainWindow::slotSplitAudio() m_activeTimeline->projectView()->splitAudio(); } +void MainWindow::slotSetAudioAlignReference() +{ + if (m_activeTimeline) { + m_activeTimeline->projectView()->setAudioAlignReference(); + } +} + +void MainWindow::slotAlignAudio() +{ + if (m_activeTimeline) { + m_activeTimeline->projectView()->alignAudio(); + } +} + void MainWindow::slotUpdateClipType(QAction *action) { if (m_activeTimeline) { diff --git a/src/mainwindow.h b/src/mainwindow.h index 4444ef34..0dd27859 100644 --- a/src/mainwindow.h +++ b/src/mainwindow.h @@ -478,6 +478,8 @@ private slots: void slotClipInProjectTree(); //void slotClipToProjectTree(); void slotSplitAudio(); + void slotSetAudioAlignReference(); + void slotAlignAudio(); void slotUpdateClipType(QAction *action); void slotShowTimeline(bool show); void slotMaximizeCurrent(bool show); -- 2.39.2