Auto-align works!

1. Add a clip to the timeline and set it as reference
   in the context menu
2. Add a new clip to the timeline and select auto-align in the context menu

Still needs some refinements when dealing with collisions (undo etc.).

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 6b3197b83d038f5d0fed003b56274688bb9f77f8..fa0e56c6c468d2f25cc1484a3a9dbb02bd3a680f 100644 (file)
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -83,6 +83,7 @@ add_subdirectory(beziercurve)
 add_subdirectory(colorcorrection)
 add_subdirectory(colorscopes)
 add_subdirectory(commands)
+add_subdirectory(lib)
 add_subdirectory(projecttree)
 add_subdirectory(utils)
 add_subdirectory(databackup)

diff --git a/src/customtrackview.cpp b/src/customtrackview.cpp
index cc75b20f515da8153416e123f9eac89b9ccfe2e0..7f30a686af0337ad07225efa7957e71530359cb6 100644 (file)
--- a/src/customtrackview.cpp
+++ b/src/customtrackview.cpp
@@ -62,6 +62,9 @@
 #include "commands/razorgroupcommand.h"
 #include "profilesdialog.h"
 
+#include "lib/audio/audioEnvelope.h"
+#include "lib/audio/audioCorrelation.h"
+
 #include <KDebug>
 #include <KLocale>
 #include <KUrl>
@@ -98,38 +101,41 @@ bool sortGuidesList(const Guide *g1 , const Guide *g2)
 // const int duration = animate ? 1500 : 1;
 
 CustomTrackView::CustomTrackView(KdenliveDoc *doc, CustomTrackScene* projectscene, QWidget *parent) :
-        QGraphicsView(projectscene, parent),
-        m_tracksHeight(KdenliveSettings::trackheight()),
-        m_projectDuration(0),
-        m_cursorPos(0),
-        m_document(doc),
-        m_scene(projectscene),
-        m_cursorLine(NULL),
-        m_operationMode(NONE),
-        m_moveOpMode(NONE),
-        m_dragItem(NULL),
-        m_dragGuide(NULL),
-        m_visualTip(NULL),
-        m_animation(NULL),
-        m_clickPoint(),
-        m_autoScroll(KdenliveSettings::autoscroll()),
-        m_pasteEffectsAction(NULL),
-        m_ungroupAction(NULL),
-        m_scrollOffset(0),
-        m_clipDrag(false),
-        m_findIndex(0),
-        m_tool(SELECTTOOL),
-        m_copiedItems(),
-        m_menuPosition(),
-        m_blockRefresh(false),
-        m_selectionGroup(NULL),
-        m_selectedTrack(0),
-        m_controlModifier(false)
-{
-    if (doc)
+    QGraphicsView(projectscene, parent),
+    m_tracksHeight(KdenliveSettings::trackheight()),
+    m_projectDuration(0),
+    m_cursorPos(0),
+    m_document(doc),
+    m_scene(projectscene),
+    m_cursorLine(NULL),
+    m_operationMode(NONE),
+    m_moveOpMode(NONE),
+    m_dragItem(NULL),
+    m_dragGuide(NULL),
+    m_visualTip(NULL),
+    m_animation(NULL),
+    m_clickPoint(),
+    m_autoScroll(KdenliveSettings::autoscroll()),
+    m_pasteEffectsAction(NULL),
+    m_ungroupAction(NULL),
+    m_scrollOffset(0),
+    m_clipDrag(false),
+    m_findIndex(0),
+    m_tool(SELECTTOOL),
+    m_copiedItems(),
+    m_menuPosition(),
+    m_blockRefresh(false),
+    m_selectionGroup(NULL),
+    m_selectedTrack(0),
+    m_audioCorrelator(NULL),
+    m_audioAlignmentReference(NULL),
+    m_controlModifier(false)
+{
+    if (doc) {
         m_commandStack = doc->commandStack();
-    else
+    } else {
         m_commandStack = NULL;
+    }
     m_ct = 0;
     setMouseTracking(true);
     setAcceptDrops(true);
@@ -2475,6 +2481,10 @@ void CustomTrackView::dropEvent(QDropEvent * event)
         }
         event->setDropAction(Qt::MoveAction);
         event->accept();
+
+        /// \todo enable when really working
+//        alignAudio();
+
     } else QGraphicsView::dropEvent(event);
     setFocus();
 }
@@ -5943,6 +5953,97 @@ void CustomTrackView::splitAudio()
     }
 }
 
+void CustomTrackView::setAudioAlignReference()
+{
+    QList<QGraphicsItem *> selection = scene()->selectedItems();
+    if (selection.isEmpty() || selection.size() > 1) {
+        emit displayMessage(i18n("You must select exactly one clip for the audio reference."), ErrorMessage);
+        return;
+    }
+    if (m_audioCorrelator != NULL) {
+        delete m_audioCorrelator;
+    }
+    if (selection.at(0)->type() == AVWIDGET) {
+        ClipItem *clip = static_cast<ClipItem*>(selection.at(0));
+        if (clip->clipType() == AV || clip->clipType() == AUDIO) {
+            m_audioAlignmentReference = clip;
+
+            AudioEnvelope *envelope = new AudioEnvelope(clip->getProducer(clip->track()));
+            m_audioCorrelator = new AudioCorrelation(envelope);
+
+            envelope->drawEnvelope().save("kdenlive-audio-reference-envelope.png");
+            envelope->dumpInfo();
+
+            emit displayMessage(i18n("Audio align reference set."), InformationMessage);
+        }
+        return;
+    }
+    emit displayMessage(i18n("Reference for audio alignment must contain audio data."), ErrorMessage);
+}
+
+void CustomTrackView::alignAudio()
+{
+    bool referenceOK = true;
+    if (m_audioCorrelator == NULL) {
+        referenceOK = false;
+    }
+    if (referenceOK) {
+        if (!scene()->items().contains(m_audioAlignmentReference)) {
+            // The reference item has been deleted from the timeline (or so)
+            referenceOK = false;
+        }
+    }
+    if (!referenceOK) {
+        emit displayMessage(i18n("Audio alignment reference not yet set."), DefaultMessage);
+        return;
+    }
+
+    int counter = 0;
+    QList<QGraphicsItem *> selection = scene()->selectedItems();
+    foreach (QGraphicsItem *item, selection) {
+        if (item->type() == AVWIDGET) {
+            ClipItem *clip = static_cast<ClipItem*>(item);
+            if (clip->clipType() == AV || clip->clipType() == AUDIO) {
+                AudioEnvelope *envelope = new AudioEnvelope(clip->getProducer(clip->track()));
+                int index = m_audioCorrelator->addChild(envelope);
+                int shift = m_audioCorrelator->getShift(index);
+                counter++;
+
+                m_audioCorrelator->info(index)->toImage().save("kdenlive-audio-align-cross-correlation.png");
+                envelope->drawEnvelope().save("kdenlive-audio-align-envelope.png");
+                envelope->dumpInfo();
+
+                int targetPos = m_audioAlignmentReference->startPos().frames(m_document->fps()) + shift;
+                qDebug() << "Reference starts at " << m_audioAlignmentReference->startPos().frames(m_document->fps());
+                qDebug() << "We will start at " << targetPos;
+                qDebug() << "to shift by " << shift;
+                qDebug() << "(eventually)";
+                qDebug() << "(maybe)";
+
+                GenTime add(shift, m_document->fps());
+                ItemInfo start = clip->info();
+                ItemInfo end = clip->info();
+                end.startPos = m_audioAlignmentReference->info().startPos + add;
+                end.endPos = m_audioAlignmentReference->info().startPos + add + clip->info().cropDuration;
+
+                QUndoCommand *moveCommand = new QUndoCommand();
+                moveCommand->setText(i18n("Auto-align clip"));
+                new MoveClipCommand(this, start, end, true, moveCommand);
+                moveClip(start, end, true);
+                updateTrackDuration(clip->track(), moveCommand);
+                m_commandStack->push(moveCommand);
+
+            }
+        }
+    }
+
+    if (counter == 0) {
+        emit displayMessage(i18n("No audio clips selected."), ErrorMessage);
+    } else {
+        emit displayMessage(i18n("Auto-aligned %1 clips.").arg(counter), InformationMessage);
+    }
+}
+
 void CustomTrackView::doSplitAudio(const GenTime &pos, int track, EffectsList effects, bool split)
 {
     ClipItem *clip = getClipItemAt(pos, track);
@@ -6026,8 +6127,15 @@ void CustomTrackView::doSplitAudio(const GenTime &pos, int track, EffectsList ef
                 deleteClip(clp->info());
                 clip->setVideoOnly(false);
                 Mlt::Tractor *tractor = m_document->renderer()->lockService();
-                if (!m_document->renderer()->mltUpdateClipProducer(tractor, m_document->tracksCount() - info.track, info.startPos.frames(m_document->fps()), clip->baseClip()->getProducer(info.track))) {
-                    emit displayMessage(i18n("Cannot update clip (time: %1, track: %2)", info.startPos.frames(m_document->fps()), info.track), ErrorMessage);
+                if (!m_document->renderer()->mltUpdateClipProducer(
+                            tractor,
+                            m_document->tracksCount() - info.track,
+                            info.startPos.frames(m_document->fps()),
+                            clip->baseClip()->getProducer(info.track))) {
+
+                    emit displayMessage(i18n("Cannot update clip (time: %1, track: %2)",
+                                             info.startPos.frames(m_document->fps()), info.track),
+                                        ErrorMessage);
                 }
                 m_document->renderer()->unlockService(tractor);
 

diff --git a/src/customtrackview.h b/src/customtrackview.h
index c5c62f9a0624b3fbb6c6403cb71ea31e52027abb..5d932b5d69011259e77d80f9914fea26fc154de6 100644 (file)
--- a/src/customtrackview.h
+++ b/src/customtrackview.h
@@ -42,6 +42,7 @@ class ClipItem;
 class AbstractClipItem;
 class AbstractGroupItem;
 class Transition;
+class AudioCorrelation;
 
 class CustomTrackView : public QGraphicsView
 {
@@ -138,6 +139,12 @@ public:
     /** @brief Creates SplitAudioCommands for selected clips. */
     void splitAudio();
 
+    /// Define which clip to take as reference for automatic audio alignment
+    void setAudioAlignReference();
+
+    /// Automatically align the currently selected clips to synchronize their audio with the reference's audio
+    void alignAudio();
+
     /** @brief Seperates the audio of a clip to a audio track.
     * @param pos Position of the clip to split
     * @param track Track of the clip
@@ -356,6 +363,9 @@ private:
     QWaitCondition m_producerNotReady;
     KStatefulBrush m_activeTrackBrush;
 
+    AudioCorrelation *m_audioCorrelator;
+    ClipItem *m_audioAlignmentReference;
+
     /** stores the state of the control modifier during mouse press.
      * Will then be used to identify whether we resize a group or only one item of it. */
     bool m_controlModifier;

diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt
new file mode 100644 (file)
index 0000000..e3a89b0
--- /dev/null
+++ b/src/lib/CMakeLists.txt
@@ -0,0 +1,10 @@
+
+add_subdirectory(audio)
+
+# Hack. kdenlive_SRCS is defined in the sub-scope and added to this scope;
+# now we again need to add it to the parent scope.
+
+set(kdenlive_SRCS
+    ${kdenlive_SRCS}
+    PARENT_SCOPE
+)

diff --git a/src/lib/audio/CMakeLists.txt b/src/lib/audio/CMakeLists.txt
new file mode 100644 (file)
index 0000000..bf5cd2b
--- /dev/null
+++ b/src/lib/audio/CMakeLists.txt
@@ -0,0 +1,10 @@
+
+set(kdenlive_SRCS
+    ${kdenlive_SRCS}
+    lib/audio/audioCorrelation.cpp
+    lib/audio/audioCorrelationInfo.cpp
+    lib/audio/audioEnvelope.cpp
+    lib/audio/audioInfo.cpp
+    lib/audio/audioStreamInfo.cpp
+    PARENT_SCOPE
+)

diff --git a/src/lib/audio/audioCorrelation.cpp b/src/lib/audio/audioCorrelation.cpp
index 0a308c1c8368e1af7bb208aa12a9d54d282e8026..7198344232fbe06cb8fa65d0e9661a704b102417 100644 (file)
--- a/src/lib/audio/audioCorrelation.cpp
+++ b/src/lib/audio/audioCorrelation.cpp
@@ -26,6 +26,7 @@ AudioCorrelation::~AudioCorrelation()
     foreach (AudioEnvelope *envelope, m_children) {
         delete envelope;
     }
+    std::cout << "Envelope deleted." << std::endl;
 }
 
 int AudioCorrelation::addChild(AudioEnvelope *envelope)

diff --git a/src/lib/audio/audioEnvelope.cpp b/src/lib/audio/audioEnvelope.cpp
index 3968f6228818d64c3546af47cea2207b62114d18..7849372d1a5ce067d715d3a0dcfd59d1a297a3c7 100644 (file)
--- a/src/lib/audio/audioEnvelope.cpp
+++ b/src/lib/audio/audioEnvelope.cpp
@@ -71,6 +71,8 @@ void AudioEnvelope::loadEnvelope()
 
     QTime t;
     t.start();
+    m_producer->seek(0);
+    m_producer->set_speed(1.0); // This is necessary, otherwise we don't get any new frames in the 2nd run.
     for (int i = 0; i < m_envelopeSize; i++) {
 
         frame = m_producer->get_frame(i);
@@ -89,6 +91,11 @@ void AudioEnvelope::loadEnvelope()
         if (sum > m_envelopeMax) {
             m_envelopeMax = sum;
         }
+
+//        std::cout << position << "|" << m_producer->get_playtime()
+//                  << "-" << m_producer->get_in() << "+" << m_producer->get_out() << " ";
+
+        delete frame;
     }
     m_envelopeMean /= m_envelopeSize;
     std::cout << "Calculating the envelope (" << m_envelopeSize << " frames) took "

diff --git a/src/mainwindow.cpp b/src/mainwindow.cpp
index bfeab9c585ed333ecbbca7b466146a3b320ccf94..6ed7c1ecc54f3b902f9440bd70b8ef4e2e704722 100644 (file)
--- a/src/mainwindow.cpp
+++ b/src/mainwindow.cpp
@@ -587,6 +587,8 @@ MainWindow::MainWindow(const QString &MltPath, const KUrl & Url, const QString &
     m_timelineContextClipMenu->addAction(actionCollection()->action("group_clip"));
     m_timelineContextClipMenu->addAction(actionCollection()->action("ungroup_clip"));
     m_timelineContextClipMenu->addAction(actionCollection()->action("split_audio"));
+    m_timelineContextClipMenu->addAction(actionCollection()->action("set_audio_align_ref"));
+    m_timelineContextClipMenu->addAction(actionCollection()->action("align_audio"));
     m_timelineContextClipMenu->addSeparator();
     m_timelineContextClipMenu->addAction(actionCollection()->action("cut_timeline_clip"));
     m_timelineContextClipMenu->addAction(actionCollection()->action(KStandardAction::name(KStandardAction::Copy)));
@@ -1505,7 +1507,7 @@ void MainWindow::setupActions()
     collection.addAction("edit_clip_marker", editClipMarker);
     connect(editClipMarker, SIGNAL(triggered(bool)), this, SLOT(slotEditClipMarker()));
 
-    KAction *addMarkerGuideQuickly = new KAction(KIcon("bookmark-new"), i18n("Add Marker/Guide quickly"), this);
+    KAction* addMarkerGuideQuickly = new KAction(KIcon("bookmark-new"), i18n("Add Marker/Guide quickly"), this);
     addMarkerGuideQuickly->setShortcut(Qt::Key_Asterisk);
     collection.addAction("add_marker_guide_quickly", addMarkerGuideQuickly);
     connect(addMarkerGuideQuickly, SIGNAL(triggered(bool)), this, SLOT(slotAddMarkerGuideQuickly()));
@@ -1514,6 +1516,14 @@ void MainWindow::setupActions()
     collection.addAction("split_audio", splitAudio);
     connect(splitAudio, SIGNAL(triggered(bool)), this, SLOT(slotSplitAudio()));
 
+    KAction* setAudioAlignReference = new KAction(i18n("Set reference for audio alignment"), this);
+    collection.addAction("set_audio_align_ref", setAudioAlignReference);
+    connect(setAudioAlignReference, SIGNAL(triggered()), this, SLOT(slotSetAudioAlignReference()));
+
+    KAction* alignAudio = new KAction(i18n("Align audio to reference"), this);
+    collection.addAction("align_audio", alignAudio);
+    connect(alignAudio, SIGNAL(triggered()), this, SLOT(slotAlignAudio()));
+
     KAction* audioOnly = new KAction(KIcon("document-new"), i18n("Audio Only"), this);
     collection.addAction("clip_audio_only", audioOnly);
     audioOnly->setData("clip_audio_only");
@@ -3797,6 +3807,20 @@ void MainWindow::slotSplitAudio()
         m_activeTimeline->projectView()->splitAudio();
 }
 
+void MainWindow::slotSetAudioAlignReference()
+{
+    if (m_activeTimeline) {
+        m_activeTimeline->projectView()->setAudioAlignReference();
+    }
+}
+
+void MainWindow::slotAlignAudio()
+{
+    if (m_activeTimeline) {
+        m_activeTimeline->projectView()->alignAudio();
+    }
+}
+
 void MainWindow::slotUpdateClipType(QAction *action)
 {
     if (m_activeTimeline) {

diff --git a/src/mainwindow.h b/src/mainwindow.h
index 4444ef341b6a1f5b85f2a526eaa142f01f7d4d68..0dd27859a85004af72e1c03d0ff732e1549e22a6 100644 (file)
--- a/src/mainwindow.h
+++ b/src/mainwindow.h
@@ -478,6 +478,8 @@ private slots:
     void slotClipInProjectTree();
     //void slotClipToProjectTree();
     void slotSplitAudio();
+    void slotSetAudioAlignReference();
+    void slotAlignAudio();
     void slotUpdateClipType(QAction *action);
     void slotShowTimeline(bool show);
     void slotMaximizeCurrent(bool show);